X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=src%2Forg%2Fxwt%2Fjs%2FLexer.java;h=4752c649561208234fb369a6a00eeffe2009053c;hb=2a94571c07db6db966f6bfa32583a72287f4db29;hp=172c5934fe2477fd20d8b4b701f2a39b529130e2;hpb=2b9f73a3d850503353fab448c6bbfe7f70efdfbe;p=org.ibex.core.git diff --git a/src/org/xwt/js/Lexer.java b/src/org/xwt/js/Lexer.java index 172c593..4752c64 100644 --- a/src/org/xwt/js/Lexer.java +++ b/src/org/xwt/js/Lexer.java @@ -1,5 +1,4 @@ -// This file was derived from org.mozilla.javascript.TokenStream; it -// is covered by the NPL 1.1. +// Derived from org.mozilla.javascript.TokenStream [NPL] /** * The contents of this file are subject to the Netscape Public @@ -21,302 +20,149 @@ package org.xwt.js; import java.io.*; -class Lexer { +/** Lexes a stream of characters into a stream of Tokens */ +class Lexer implements Tokens { - private SmartReader in; - private boolean pushedBack = false; - - private int op; - public Number number; - public String string; - - public Lexer(Reader r) throws IOException { in = new SmartReader(r); } - public int peekToken() throws IOException { int ret = peekToken(); pushBackToken(); return ret; } - public void pushBackToken() { if (pushedBack) throw new Error("can't push back twice"); pushedBack = true; } - - // Token Constants ////////////////////////////////////////////////////////// - - public final static int - EOL = 1, // end of line - RETURN = 5, // return - GOTO = 6, // goto - BITOR = 11, // | - ASSIGN_BITOR = 211, // |= - BITXOR = 12, // ^ - ASSIGN_BITXOR= 212, // ^= - BITAND = 13, // & - ASSIGN_BITAND= 213, // &= - EQ = 14, // == - NE = 15, // != - LT = 16, // < - LE = 17, // <= - GT = 18, // > - GE = 19, // >= - LSH = 20, // << - ASSIGN_LSH = 220, // <<= - RSH = 21, // >> - ASSIGN_RSH = 221, // >>= - URSH = 22, // >>> - ASSIGN_URSH = 222, // >>>= - ADD = 23, // + - ASSIGN_ADD = 223, // += - SUB = 24, // - - ASSIGN_SUB = 224, // -= - MUL = 25, // * - ASSIGN_MUL = 225, // *= - DIV = 26, // / - ASSIGN_DIV = 226, // /= - MOD = 27, // % - ASSIGN_MOD = 227, // %= - BITNOT = 28, // ~ - ASSIGN_BITNOT= 228, // ~= - DELPROP = 31, // delete - TYPEOF = 32, // typeof - NAME = 44, // *** identifiers *** - NUMBER = 45, // *** numeric literals *** - STRING = 46, // *** string literals *** - NULL = 49, // null - THIS = 50, // this - FALSE = 51, // false - TRUE = 52, // true - SHEQ = 53, // === - SHNE = 54, // !== - THROW = 62, // throw - IN = 63, // in - INSTANCEOF = 64, // instanceof - TRY = 75, // try - SEMI = 89, // ; - LB = 90, // [ - RB = 91, // ] - LC = 92, // { - RC = 93, // } - LP = 94, // ( - RP = 95, // ) - COMMA = 96, // , - ASSIGN = 97, // = - HOOK = 98, // ? - COLON = 99, // : - OR = 100, // || - AND = 101, // && - INC = 106, // ++ - DEC = 107, // -- - DOT = 108, // . - FUNCTION = 110, // function - IF = 113, // if keyword - ELSE = 114, // else keyword - SWITCH = 115, // switch keyword - CASE = 116, // case keyword - DEFAULT = 117, // default keyword - WHILE = 118, // while keyword - DO = 119, // do keyword - FOR = 120, // for keyword - BREAK = 121, // break keyword - CONTINUE = 122, // continue keyword - VAR = 123, // var keyword - WITH = 124, // with keyword - CATCH = 125, // catch keyword - FINALLY = 126, // finally keyword - RESERVED = 127, // reserved keywords - NOP = 128, // NOP - VOID = 132, // void keyword - MOD_ASSIGN = 133, // %= - BANG = 134, // %= - ASSERT = 150; // assert keyword + /** for debugging */ + public static void main(String[] s) throws Exception { + Lexer l = new Lexer(new InputStreamReader(System.in), "stdin", 0); + int tok = 0; + while((tok = l.getToken()) != -1) System.out.println(codeToString[tok]); + } + /** the token that was just parsed */ + protected int op; - // Predicates /////////////////////////////////////////////////////////////////////// + /** if the token just parsed was a NUMBER, this is the numeric value */ + protected Number number = null; - protected static boolean isJSIdentifier(String s) { - int length = s.length(); - if (length == 0 || !Character.isJavaIdentifierStart(s.charAt(0))) return false; - for (int i=1; i= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')); } private static boolean isDigit(int c) { return (c >= '0' && c <= '9'); } - private static boolean isLineBreak(int c) { return (c == '\n' || c == '\r' || c == 0x2028 || c == 0x2029); } private static int xDigitToInt(int c) { if ('0' <= c && c <= '9') return c - '0'; - if ('a' <= c && c <= 'f') return c - ('a' - 10); - if ('A' <= c && c <= 'F') return c - ('A' - 10); - return -1; - } - public static boolean isWhiteSpace(int c) { - if (c == '\u0020' || c == '\u0009' || c == '\u000C' || c == '\u000B' || c == '\u00A0') return true; - return Character.getType((char)c) == Character.SPACE_SEPARATOR; + else if ('a' <= c && c <= 'f') return c - ('a' - 10); + else if ('A' <= c && c <= 'F') return c - ('A' - 10); + else return -1; } // Token Subtype Handlers ///////////////////////////////////////////////////////// - private int getKeyword(String name) throws IOException { - final int - Id_break = BREAK, - Id_case = CASE, - Id_continue = CONTINUE, - Id_default = DEFAULT, - Id_delete = DELPROP, - Id_do = DO, - Id_else = ELSE, - Id_false = FALSE, - Id_for = FOR, - Id_function = FUNCTION, - Id_if = IF, - Id_in = IN, - Id_null = NULL, - Id_return = RETURN, - Id_switch = SWITCH, - Id_this = THIS, - Id_true = TRUE, - Id_typeof = TYPEOF, - Id_var = VAR, - Id_void = VOID, - Id_while = WHILE, - Id_with = WITH, - - // the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c - Id_abstract = RESERVED, - Id_boolean = RESERVED, - Id_byte = RESERVED, - Id_catch = CATCH, - Id_char = RESERVED, - Id_class = RESERVED, - Id_const = RESERVED, - Id_debugger = RESERVED, - Id_double = RESERVED, - Id_enum = RESERVED, - Id_extends = RESERVED, - Id_final = RESERVED, - Id_finally = FINALLY, - Id_float = RESERVED, - Id_goto = RESERVED, - Id_implements = RESERVED, - Id_instanceof = INSTANCEOF, - Id_int = RESERVED, - Id_interface = RESERVED, - Id_long = RESERVED, - Id_native = RESERVED, - Id_package = RESERVED, - Id_private = RESERVED, - Id_protected = RESERVED, - Id_public = RESERVED, - Id_assert = ASSERT, - Id_short = RESERVED, - Id_static = RESERVED, - Id_super = RESERVED, - Id_synchronized = RESERVED, - Id_throw = THROW, - Id_throws = RESERVED, - Id_transient = RESERVED, - Id_try = TRY, - Id_volatile = RESERVED; - - int id; - String s = name; - L0: { id = -1; String X = null; int c; - L: switch (s.length()) { + private int getKeyword(String s) throws IOException { + char c; + switch (s.length()) { case 2: c=s.charAt(1); - if (c=='f') { if (s.charAt(0)=='i') {id=Id_if; break L0;} } - else if (c=='n') { if (s.charAt(0)=='i') {id=Id_in; break L0;} } - else if (c=='o') { if (s.charAt(0)=='d') {id=Id_do; break L0;} } - break L; + if (c=='f') { if (s.charAt(0)=='i') return IF; } + else if (c=='n') { if (s.charAt(0)=='i') return IN; } + else if (c=='o') { if (s.charAt(0)=='d') return DO; } + break; case 3: switch (s.charAt(0)) { - case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') {id=Id_for; break L0;} break L; - case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') {id=Id_int; break L0;} break L; - case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') - throw new IOException("the new keyword is not permitted in XWT scripts"); - break L; - case 't': if (s.charAt(2)=='y' && s.charAt(1)=='r') {id=Id_try; break L0;} break L; - case 'v': if (s.charAt(2)=='r' && s.charAt(1)=='a') {id=Id_var; break L0;} break L; - } break L; + case 'a': if (s.charAt(2)=='d' && s.charAt(1)=='n') return AND; break; + case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') return FOR; break; + case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') return RESERVED; + case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') return RESERVED; + case 't': if (s.charAt(2)=='y' && s.charAt(1)=='r') return TRY; break; + case 'v': if (s.charAt(2)=='r' && s.charAt(1)=='a') return VAR; break; + } break; case 4: switch (s.charAt(0)) { - case 'b': X="byte";id=Id_byte; break L; + case 'b': return s.equals("byte") ? RESERVED : -1; case 'c': c=s.charAt(3); - if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='a') {id=Id_case; break L0;} } - else if (c=='r') { if (s.charAt(2)=='a' && s.charAt(1)=='h') {id=Id_char; break L0;} } - break L; + if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='a') return CASE; } + else if (c=='r') { if (s.charAt(2)=='a' && s.charAt(1)=='h') return RESERVED; } + return -1; case 'e': c=s.charAt(3); - if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='l') {id=Id_else; break L0;} } - else if (c=='m') { if (s.charAt(2)=='u' && s.charAt(1)=='n') {id=Id_enum; break L0;} } - break L; - case 'g': X="goto";id=Id_goto; break L; - case 'l': X="long";id=Id_long; break L; - case 'n': X="null";id=Id_null; break L; + if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='l') return ELSE; } + else if (c=='m') { if (s.charAt(2)=='u' && s.charAt(1)=='n') return RESERVED; } + return -1; + case 'g': return s.equals("goto") ? RESERVED : -1; + case 'l': return s.equals("long") ? RESERVED : -1; + case 'n': return s.equals("null") ? NULL : -1; case 't': c=s.charAt(3); - if (c=='e') { if (s.charAt(2)=='u' && s.charAt(1)=='r') {id=Id_true; break L0;} } - else if (c=='s') { if (s.charAt(2)=='i' && s.charAt(1)=='h') {id=Id_this; break L0;} } - break L; - case 'v': X="void";id=Id_void; break L; - case 'w': X="with";id=Id_with; break L; - } break L; + if (c=='e') { if (s.charAt(2)=='u' && s.charAt(1)=='r') return TRUE; } + else if (c=='s') { if (s.charAt(2)=='i' && s.charAt(1)=='h') return THIS; } + return -1; + case 'w': if (s.equals("with")) return RESERVED; else return -1; + case 'v': if (s.equals("void")) return RESERVED; else return -1; + } break; case 5: switch (s.charAt(2)) { - case 'a': X="class";id=Id_class; break L; - case 'e': X="break";id=Id_break; break L; - case 'i': X="while";id=Id_while; break L; - case 'l': X="false";id=Id_false; break L; + case 'a': return s.equals("class") ? RESERVED : -1; + case 'e': return s.equals("break") ? BREAK : -1; + case 'i': return s.equals("while") ? WHILE : -1; + case 'l': return s.equals("false") ? FALSE : -1; case 'n': c=s.charAt(0); - if (c=='c') { X="const"; throw new IOException("the const keyword is not permitted in XWT"); } - else if (c=='f') { X="final";id=Id_final; } - break L; - case 'o': c=s.charAt(0); - if (c=='f') { X="float";id=Id_float; } - else if (c=='s') { X="short";id=Id_short; } - break L; - case 'p': X="super";id=Id_super; break L; - case 'r': X="throw";id=Id_throw; break L; - case 't': X="catch";id=Id_catch; break L; - } break L; + if (s.equals("const")) return RESERVED; + else if (s.equals("final")) return RESERVED; + return -1; + case 'o': c=s.charAt(0); + if (c == 'c') return s.equals("float") ? RESERVED : -1; + else if (c=='s') return s.equals("final") ? RESERVED : -1; + break; + case 'p': return s.equals("super") ? RESERVED : -1; + case 'r': return s.equals("throw") ? THROW : -1; + case 't': return s.equals("catch") ? CATCH : -1; + } break; case 6: switch (s.charAt(1)) { - case 'a': X="native";id=Id_native; break L; + case 'a': return s.equals("class") ? RESERVED : -1; case 'e': c=s.charAt(0); - if (c=='d') { X="delete"; throw new IOException("the delete keyword is not permitted in XWT scripts"); } - else if (c=='r') { X="return";id=Id_return; } - break L; - case 'h': X="throws";id=Id_throws; break L; - case 'o': X="double";id=Id_double; break L; - case 's': X="assert";id=Id_assert; break L; - case 'u': X="public";id=Id_public; break L; - case 'w': X="switch";id=Id_switch; break L; - case 'y': X="typeof";id=Id_typeof; break L; - } break L; + if (s.equals("delete")) return RESERVED; + else if (c=='r') return s.equals("return") ? RETURN : -1; + break; + case 'h': return s.equals("throws") ? RESERVED : -1; + case 'o': return s.equals("double") ? RESERVED : -1; + case 's': return s.equals("assert") ? ASSERT : -1; + case 'u': return s.equals("public") ? RESERVED : -1; + case 'w': return s.equals("switch") ? SWITCH : -1; + case 'y': return s.equals("typeof") ? TYPEOF : -1; + } break; case 7: switch (s.charAt(1)) { - case 'a': X="package";id=Id_package; break L; - case 'e': X="default";id=Id_default; break L; - case 'i': X="finally";id=Id_finally; break L; - case 'o': X="boolean";id=Id_boolean; break L; - case 'r': X="private";id=Id_private; break L; - case 'x': X="extends";id=Id_extends; break L; - } break L; + case 'a': return s.equals("package") ? RESERVED : -1; + case 'e': return s.equals("default") ? DEFAULT : -1; + case 'i': return s.equals("finally") ? FINALLY : -1; + case 'o': return s.equals("boolean") ? RESERVED : -1; + case 'r': return s.equals("private") ? RESERVED : -1; + case 'x': return s.equals("extends") ? RESERVED : -1; + } break; case 8: switch (s.charAt(0)) { - case 'a': X="abstract";id=Id_abstract; break L; - case 'c': X="continue";id=Id_continue; break L; - case 'd': X="debugger";id=Id_debugger; break L; - case 'f': X="function";id=Id_function; break L; - case 'v': X="volatile";id=Id_volatile; break L; - } break L; + case 'a': return s.equals("abstract") ? RESERVED : -1; + case 'c': return s.equals("continue") ? CONTINUE : -1; + case 'd': return s.equals("debugger") ? RESERVED : -1; + case 'f': return s.equals("function") ? FUNCTION : -1; + case 'v': return s.equals("volatile") ? RESERVED : -1; + } break; case 9: c=s.charAt(0); - if (c=='i') { X="interface";id=Id_interface; } - else if (c=='p') { X="protected";id=Id_protected; } - else if (c=='t') { X="transient";id=Id_transient; } - break L; + if (c=='i') return s.equals("interface") ? RESERVED : -1; + else if (c=='p') return s.equals("protected") ? RESERVED : -1; + else if (c=='t') return s.equals("transient") ? RESERVED : -1; + break; case 10: c=s.charAt(1); - if (c=='m') { X="implements";id=Id_implements; } - else if (c=='n') { X="instanceof"; throw new IOException("the instanceof keyword is not permitted in XWT scripts"); } - break L; - case 12: X="synchronized";id=Id_synchronized; break L; + if (c=='m') return s.equals("implements") ? RESERVED : -1; + else if (c=='n' && s.equals("instanceof")) return RESERVED; + break; + case 12: return s.equals("synchronized") ? RESERVED : -1; } - if (X!=null && X!=s && !X.equals(s)) id = -1; - } - if (id == -1) { return -1; } - this.op = id >> 8; - return id & 0xff; + return -1; } private int getIdentifier(int c) throws IOException { @@ -325,8 +171,9 @@ class Lexer { in.unread(); String str = in.getString(); int result = getKeyword(str); + if (result == RESERVED) throw new LexerException("The reserved word \"" + str + "\" is not permitted in XWT scripts"); if (result != -1) return result; - this.string = str; + this.string = str.intern(); return NAME; } @@ -350,7 +197,7 @@ class Lexer { if (c == 'e' || c == 'E') { c = in.read(); if (c == '+' || c == '-') c = in.read(); - if (!isDigit(c)) throw new IOException("msg.missing.exponent"); + if (!isDigit(c)) throw new LexerException("float listeral did not have an exponent value"); do { c = in.read(); } while (isDigit(c)); } } @@ -359,13 +206,12 @@ class Lexer { String numString = in.getString(); if (base == 10 && !isInteger) { try { dval = (Double.valueOf(numString)).doubleValue(); } - catch (NumberFormatException ex) { throw new IOException("msg.caught.nfe"); } + catch (NumberFormatException ex) { throw new LexerException("invalid numeric literal: \"" + numString + "\""); } } else { if (isInteger) { longval = Long.parseLong(numString, base); dval = (double)longval; } else { - // FIXME: we're not handling hex/octal fractions... does that matter? dval = Double.parseDouble(numString); longval = (long) dval; if (longval == dval) isInteger = true; @@ -387,7 +233,7 @@ class Lexer { c = in.read(); in.startString(); // start after the first " while(c != quoteChar) { - if (c == '\n' || c == -1) throw new IOException("msg.unterminated.string.lit"); + if (c == '\n' || c == -1) throw new LexerException("unterminated string literal"); if (c == '\\') { if (stringBuf == null) { in.unread(); // Don't include the backslash @@ -401,36 +247,43 @@ class Lexer { case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\u000B'; break; - default: throw new IOException("\\u and \\0 escapes not currently supported -- use XML entities"); + case '\\': c = '\\'; break; + case 'u': { + int v = 0; + for(int i=0; i<4; i++) { + int ci = in.read(); + if (!((ci >= '0' && ci <= '9') || (ci >= 'a' && ci <= 'f') || (ci >= 'A' && ci <= 'F'))) + throw new LexerException("illegal character '" + ((char)c) + "' in \\u unicode escape sequence"); + v = (v << 8) | Integer.parseInt(ci + "", 16); + } + c = (char)v; + break; + } + default: + // just use the character that was escaped + break; } } if (stringBuf != null) stringBuf.append((char) c); c = in.read(); } - if (stringBuf != null) this.string = stringBuf.toString(); + if (stringBuf != null) this.string = stringBuf.toString().intern(); else { in.unread(); // miss the trailing " - this.string = in.getString(); + this.string = in.getString().intern(); in.read(); } return STRING; } - public int getToken() throws IOException { - if (pushedBack) { pushedBack = false; return op; } - return (op = getToken()); - } - public int _getToken() throws IOException { int c; - do { if ((c = in.read()) == '\n') break; } while (isWhiteSpace(c) || c == '\n'); + do { c = in.read(); } while (c == '\u0020' || c == '\u0009' || c == '\u000C' || c == '\u000B' || c == '\n' ); if (c == -1) return -1; - if (c == '\\' && in.peek() == 'u') throw new IOException("\\u and \\0 escapes not currently supported -- use XML entities"); - if (Character.isJavaIdentifierStart((char)c)) return getIdentifier(c); + if (c == '\\' || Character.isJavaIdentifierStart((char)c)) return getIdentifier(c); if (isDigit(c) || (c == '.' && isDigit(in.peek()))) return getNumber(c); if (c == '"' || c == '\'') return getString(c); switch (c) { - case '\n': return EOL; case ';': return SEMI; case '[': return LB; case ']': return RB; @@ -447,19 +300,14 @@ class Lexer { case '&': return in.match('&') ? AND : in.match('=') ? ASSIGN_BITAND : BITAND; case '=': return !in.match('=') ? ASSIGN : in.match('=') ? SHEQ : EQ; case '!': return !in.match('=') ? BANG : in.match('=') ? SHNE : NE; - case '%': return in.match('=') ? MOD_ASSIGN : MOD; + case '%': return in.match('=') ? ASSIGN_MOD : MOD; case '~': return BITNOT; case '+': return in.match('=') ? ASSIGN_ADD : in.match('+') ? INC : ADD; case '-': return in.match('=') ? ASSIGN_SUB: in.match('-') ? DEC : SUB; case '*': return in.match('=') ? ASSIGN_MUL : MUL; - case '<': return !in.match('<') ? - (in.match('=') ? LE : LT) : - in.match('=') ? ASSIGN_LSH : LSH; - case '>': return !in.match('>') ? - (in.match('=') ? GE : GT) : - in.match('>') ? - (in.match('=') ? ASSIGN_URSH : URSH) : - (in.match('=') ? ASSIGN_RSH : RSH); + case '<': return !in.match('<') ? (in.match('=') ? LE : LT) : in.match('=') ? ASSIGN_LSH : LSH; + case '>': return !in.match('>') ? (in.match('=') ? GE : GT) : + in.match('>') ? (in.match('=') ? ASSIGN_URSH : URSH) : (in.match('=') ? ASSIGN_RSH : RSH); case '/': if (in.match('=')) return ASSIGN_DIV; if (in.match('/')) { while ((c = in.read()) != -1 && c != '\n'); in.unread(); return getToken(); } @@ -467,20 +315,28 @@ class Lexer { while ((c = in.read()) != -1 && !(c == '*' && in.match('/'))) { if (c == '\n' || c != '/' || !in.match('*')) continue; if (in.match('/')) return getToken(); - throw new IOException("msg.nested.comment"); + throw new LexerException("nested comments are not permitted"); } - if (c == -1) throw new IOException("msg.unterminated.comment"); + if (c == -1) throw new LexerException("unterminated comment"); return getToken(); // `goto retry' - default: throw new IOException("illegal character: " + c); + default: throw new LexerException("illegal character: \'" + ((char)c) + "\'"); } } - private static class SmartReader { + + // SmartReader //////////////////////////////////////////////////////////////// + + /** a Reader that tracks line numbers and can push back tokens */ + private class SmartReader { PushbackReader reader = null; int lastread = -1; public SmartReader(Reader r) { reader = new PushbackReader(r); } - public void unread() throws IOException { reader.unread(lastread); } + public void unread() throws IOException { unread((char)lastread); } + public void unread(char c) throws IOException { + reader.unread(c); + if (accumulator != null) accumulator.setLength(accumulator.length() - 1); + } public boolean match(char c) throws IOException { if (peek() == c) { reader.read(); return true; } else return false; } public int peek() throws IOException { int peeked = reader.read(); @@ -489,13 +345,18 @@ class Lexer { } public int read() throws IOException { lastread = reader.read(); - if (accumulator != null) accumulator.append(lastread); + if (accumulator != null) accumulator.append((char)lastread); + if (lastread != '\n' && lastread != '\r') col++; + if (lastread == '\n') { line++; col = 0; } return lastread; } - // FIXME: could be much more efficient + // FEATURE: could be much more efficient StringBuffer accumulator = null; - public void startString() { accumulator = new StringBuffer(); } + public void startString() { + accumulator = new StringBuffer(); + accumulator.append((char)lastread); + } public String getString() throws IOException { String ret = accumulator.toString(); accumulator = null; @@ -503,4 +364,53 @@ class Lexer { } } + + // Token PushBack code //////////////////////////////////////////////////////////// + + private int pushBackDepth = 0; + private int[] pushBackInts = new int[10]; + private Object[] pushBackObjects = new Object[10]; + + /** push back a token */ + public void pushBackToken(int op, Object obj) { + if (pushBackDepth >= pushBackInts.length - 1) { + int[] newInts = new int[pushBackInts.length * 2]; + System.arraycopy(pushBackInts, 0, newInts, 0, pushBackInts.length); + pushBackInts = newInts; + Object[] newObjects = new Object[pushBackObjects.length * 2]; + System.arraycopy(pushBackObjects, 0, newObjects, 0, pushBackObjects.length); + pushBackObjects = newObjects; + } + pushBackInts[pushBackDepth] = op; + pushBackObjects[pushBackDepth] = obj; + pushBackDepth++; + } + + /** push back the most recently read token */ + public void pushBackToken() { pushBackToken(op, number != null ? (Object)number : (Object)string); } + + /** read a token but leave it in the stream */ + public int peekToken() throws IOException { + int ret = getToken(); + pushBackToken(); + return ret; + } + + /** read a token */ + public int getToken() throws IOException { + number = null; + string = null; + if (pushBackDepth == 0) return op = _getToken(); + pushBackDepth--; + op = pushBackInts[pushBackDepth]; + if (pushBackObjects[pushBackDepth] != null) { + number = pushBackObjects[pushBackDepth] instanceof Number ? (Number)pushBackObjects[pushBackDepth] : null; + string = pushBackObjects[pushBackDepth] instanceof String ? (String)pushBackObjects[pushBackDepth] : null; + } + return op; + } + + class LexerException extends IOException { + public LexerException(String s) { super(sourceName + ":" + line + "," + col + ": " + s); } + } }