- public Token next() throws IOException {
- if (pos >= str.length()) return Atom.EOF;
- return Atom.fromChar(str.charAt(pos++));
+ private Block cur;
+ private String s = null;
+ int pos = 0;
+ int spos = 0;
+
+ int _row = 0;
+ int _col = 0;
+ public CharToken next() throws IOException {
+ if (cur==null) return null;
+ if (s != null) {
+ if (spos < s.length()) {
+ char c = s.charAt(spos++);
+ if (c=='\n') { _row++; _col = 0; }
+ else _col++;
+ return new CharToken(c, _row, _col);
+ }
+ s = null;
+ }
+ if (pos >= cur.size()) {
+ pos = cur.iip+1;
+ cur = cur.parent;
+ if (cur==null) return null;
+ return CharToken.right(_row, _col);
+ }
+ Object o = cur.child(pos++);
+ if (o instanceof String) {
+ spos = 0;
+ s = (String)o;
+ return next();
+ }
+ if (o instanceof Block) {
+ Block b = (Block)o;
+ _row = b.row;
+ _col = b.col;
+ }
+ if (((Block)o).isLiteral()) {
+ spos = 0;
+ s = ((Block.Literal)o).text();
+ return next();
+ }
+ cur = (Block)o;
+ pos = 0;
+ return CharToken.left(_row, _col);