got new tib tokenizer going

[sbp.git] / src / edu / berkeley / sbp / Parser.java

diff --git a/src/edu/berkeley/sbp/Parser.java b/src/edu/berkeley/sbp/Parser.java
index 4244d21..1a0d307 100644 (file)
--- a/src/edu/berkeley/sbp/Parser.java
+++ b/src/edu/berkeley/sbp/Parser.java
@@ -36,16 +36,18 @@ public abstract class Parser<T extends Token, R> {
     public Forest<R> parse(Token.Stream<T> input) throws IOException, Failed {
         GSS gss = new GSS();
         Token.Location loc = input.getLocation();
-        GSS.Phase current = gss.new Phase(null, input.next(), loc);
+        GSS.Phase current = gss.new Phase(null, input.next(1), loc);
         current.newNode(null, null, pt.start, true);
+        int count = 1;
         for(;;) {
             loc = input.getLocation();
-            GSS.Phase next = gss.new Phase(current, input.next(), loc);
+            GSS.Phase next = gss.new Phase(current, input.next(count), loc);
             current.reduce();
             Forest forest = current.token==null ? null : shiftedToken((T)current.token, loc);
             current.shift(next, forest);
-            if (current.isDone()) return (Forest<R>)current.finalResult;
+            count = next.hash.size();
             current.checkFailure();
+            if (current.isDone()) return (Forest<R>)current.finalResult;
             current = next;
         }
     }
@@ -128,12 +130,22 @@ public abstract class Parser<T extends Token, R> {
                     if (start0.contains(p.owner()) && p.next()==null)
                         state.accept = true;
 
-                    // FIXME: how does right-nullability interact with follow restrictions?
-                    // all right-nullable rules get a reduction [Johnstone 2000]
                     if (p.isRightNullable(cache)) {
                         Walk.Follow wf = new Walk.Follow(top.empty(), p.owner(), all_elements, cache);
                         Reduction red = new Reduction(p);
-                        state.reductions.put(wf.walk(p.owner()), red);
+
+                        Topology follow = wf.walk(p.owner());
+                        if (p.owner() instanceof Sequence.RewritingSequence &&
+                            (((Sequence.RewritingSequence)p.owner()).tag+"").equals("emailaddr")) {
+                            System.out.println("follow before: " + new edu.berkeley.sbp.misc.CharToken.CharRange(follow));
+                        }
+                        for(Position p2 = p; p2 != null && p2.element() != null; p2 = p2.next())
+                            follow = follow.intersect(new Walk.Follow(top.empty(), p2.element(), all_elements, cache).walk(p2.element()));
+                        if (p.owner() instanceof Sequence.RewritingSequence &&
+                            (((Sequence.RewritingSequence)p.owner()).tag+"").equals("emailaddr")) {
+                            System.out.println("follow after: " + new edu.berkeley.sbp.misc.CharToken.CharRange(follow));
+                        }
+                        state.reductions.put(follow, red);
                         if (wf.includesEof()) state.eofReductions.add(red);
                     }