got new tib tokenizer going

[sbp.git] / src / edu / berkeley / sbp / tib / Tib.java

diff --git a/src/edu/berkeley/sbp/tib/Tib.java b/src/edu/berkeley/sbp/tib/Tib.java
index 29181a0..69ac824 100644 (file)
--- a/src/edu/berkeley/sbp/tib/Tib.java
+++ b/src/edu/berkeley/sbp/tib/Tib.java
@@ -29,19 +29,107 @@ public class Tib implements Token.Stream<CharToken> {
     public Tib(Reader r) throws IOException, Invalid { this(new BufferedReader(r)); }
     public Tib(InputStream is) throws IOException, Invalid { this(new BufferedReader(new InputStreamReader(is))); }
     public Tib(BufferedReader br) throws IOException, Invalid {
-        cur = parse(br);
-        System.out.println("\rparsing: \"" + cur.toString(0, -1) + "\"");
+        this.br = br;
+        istack.add(-1);
+        //cur = parse(br);
+        //System.out.println("\rparsing: \"" + cur.toString(0, -1) + "\"");
     }
 
     private Block cur;
-    private String s = null;
+    private String s = "";
     int pos = 0;
     int spos = 0;
 
-    int _row = 0;
+    int _row = 1;
     int _col = 0;
     public Token.Location getLocation() { return new CharToken.CartesianLocation(_row, _col); }
-    public CharToken next() throws IOException {
+    private BufferedReader br;
+
+    boolean waiting = false;
+    char waitingChar = ' ';
+    boolean indenting = true;
+    int indentation = 0;
+    private ArrayList<Integer> istack = new ArrayList<Integer>();
+    public CharToken next(int numstates) throws IOException {
+        CharToken ret = nextc(numstates);
+        if      (ret==CharToken.left)  System.out.print("\033[31m{\033[0m");
+        else if (ret==CharToken.right) System.out.print("\033[31m}\033[0m");
+        else if (ret==null) return null;
+        else System.out.print(ret.c);
+        return ret;
+    }
+
+    CharToken waitingBrace = null;
+    public CharToken nextc(int numstates) throws IOException {
+        char c;
+        if (waitingBrace != null) {
+            CharToken ret = waitingBrace;
+            waitingBrace = null;
+            return ret;
+        }
+        if (waiting) {
+            waiting = false;
+            c = waitingChar;
+        } else {
+            int i = br.read();
+            if (i==-1) {
+                if (istack.size() > 1) {
+                    istack.remove(istack.size()-1);
+                    return CharToken.right;
+                }
+                return null;
+            }
+            c = (char)i;
+        }
+        if (c=='\n') { _row++; _col=0; }
+        else         _col++;
+        if (indenting) {
+            if (c==' ') { indentation++; return done(c); }
+            if (c=='\n') { indentation = 0; if (blank) return nextc(numstates); blank = true; waiting = true; waitingChar='\n'; return new CharToken('\n'); }
+            int last = istack.size()==0 ? -1 : istack.get(istack.size()-1);
+            if (indentation==last) {
+                if (blank) {
+                    indenting = false;
+                    waitingChar = c;
+                    waiting = true;
+                    waitingBrace = CharToken.left;
+                    return CharToken.right;
+                    //return nextc(numstates);
+                }
+                blank = false;
+                indenting = false;
+                return done(c);
+            }
+            blank = false;
+            waitingChar = c;
+            waiting = true;
+            if (indentation > last) {
+                indenting = false;
+                istack.add(indentation);
+                System.out.print("\033[31m+"+indentation+"+\033[0m");
+                return CharToken.left;
+            } else /*if (indentation < last)*/ {
+                istack.remove(istack.size()-1);
+                System.out.print("\033[31m-"+last+"-\033[0m");
+                blank = true;
+                return CharToken.right;
+            }
+        } else {
+            blank = false;
+            if (c=='\n') { indenting=true; indentation = 0; }
+            return done(c);
+        }
+    }
+    public CharToken done(char c) {
+        switch(c) {
+            case '{': return CharToken.left;
+            case '}': return CharToken.right;
+            default: return new CharToken(c);
+        }
+    }
+    boolean blank = false;
+    /*
+    public CharToken next(int numstates) throws IOException {
         if (cur==null) return null;
         if (s != null) {
             if (spos < s.length()) {
@@ -64,7 +152,7 @@ public class Tib implements Token.Stream<CharToken> {
         if (o instanceof String) {
             spos = 0;
             s = (String)o;
-            return next();
+            return next(numstates);
         }
         if (o instanceof Block) {
             Block b = (Block)o;
@@ -74,13 +162,13 @@ public class Tib implements Token.Stream<CharToken> {
         if (((Block)o).isLiteral()) {
             spos = 0;
             s = ((Block.Literal)o).text();
-            return next();
+            return next(numstates);
         }
         cur = (Block)o;
         pos = 0;
         return CharToken.left;
     }
-
+    */
     public static Block parse(BufferedReader br) throws Invalid, IOException {
         int row=0, col=0;
         try {