X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;ds=sidebyside;f=src%2Forg%2Fxwt%2Fjs%2FLexer.java;h=3a80f5a1d157faa71f35b8b193b95f6459f39260;hb=17b346d40a6ac856114dfae365e4f106933d14f9;hp=93586edf4b6f86a59f74a75a65d57dd095d42057;hpb=26425831523e621df99c1a75312f9eaf61ed90cc;p=org.ibex.core.git diff --git a/src/org/xwt/js/Lexer.java b/src/org/xwt/js/Lexer.java index 93586ed..3a80f5a 100644 --- a/src/org/xwt/js/Lexer.java +++ b/src/org/xwt/js/Lexer.java @@ -1,5 +1,4 @@ -// This file was derived from org.mozilla.javascript.TokenStream; it -// is covered by the NPL 1.1. +// Derived from org.mozilla.javascript.TokenStream [NPL] /** * The contents of this file are subject to the Netscape Public @@ -30,16 +29,18 @@ class Lexer { System.out.println(codeToString[tok]); } - private SmartReader in; - private boolean pushedBack = false; + public int op; + public Number number = null; + public String string = null; + + public int line = 0; + public int col = 0; - private int op; - public Number number; - public String string; + private SmartReader in; + public String sourceName = "unknown"; public Lexer(Reader r) throws IOException { in = new SmartReader(r); } - public int peekToken() throws IOException { int ret = getToken(); pushBackToken(); return ret; } - public void pushBackToken() { if (pushedBack) throw new Error("can't push back twice"); pushedBack = true; } + // Token Constants ////////////////////////////////////////////////////////// @@ -189,6 +190,7 @@ class Lexer { else if (c=='o') { if (s.charAt(0)=='d') return DO; } break; case 3: switch (s.charAt(0)) { + case 'a': if (s.charAt(2)=='d' && s.charAt(1)=='n') return AND; break; case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') return FOR; break; case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') return RESERVED; break; case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') throw new IOException("the new keyword is not permitted in XWT scripts"); @@ -320,7 +322,6 @@ class Lexer { longval = Long.parseLong(numString, base); dval = (double)longval; } else { - // FIXME: we're not handling hex/octal fractions... does that matter? dval = Double.parseDouble(numString); longval = (long) dval; if (longval == dval) isInteger = true; @@ -356,7 +357,21 @@ class Lexer { case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\u000B'; break; - default: throw new IOException("\\u and \\0 escapes not currently supported -- use XML entities"); + case '\\': c = '\\'; break; + case 'u': { + int v = 0; + for(int i=0; i<4; i++) { + int ci = in.read(); + if (!((ci >= '0' && ci <= '9') || (ci >= 'a' && ci <= 'f') || (ci >= 'A' && ci <= 'F'))) + throw new IOException("illegal character '" + ((char)c) + "' in \\u unicode escape sequence"); + v = (v << 8) | Integer.parseInt(ci + "", 16); + } + c = (char)v; + break; + } + default: + // just use the character that was escaped + break; } } if (stringBuf != null) stringBuf.append((char) c); @@ -371,17 +386,11 @@ class Lexer { return STRING; } - public int getToken() throws IOException { - if (pushedBack) { pushedBack = false; return op; } - return (op = _getToken()); - } - public int _getToken() throws IOException { int c; do { if ((c = in.read()) == '\n') break; } while (isWhiteSpace(c) || c == '\n'); if (c == -1) return -1; - if (c == '\\' && in.peek() == 'u') throw new IOException("\\u and \\0 escapes not currently supported -- use XML entities"); - if (Character.isJavaIdentifierStart((char)c)) return getIdentifier(c); + if (c == '\\' || Character.isJavaIdentifierStart((char)c)) return getIdentifier(c); if (isDigit(c) || (c == '.' && isDigit(in.peek()))) return getNumber(c); if (c == '"' || c == '\'') return getString(c); switch (c) { @@ -426,17 +435,18 @@ class Lexer { } if (c == -1) throw new IOException("msg.unterminated.comment"); return getToken(); // `goto retry' - default: throw new IOException("illegal character: " + c); + default: throw new IOException("illegal character: " + ((char)c)); } } - private static class SmartReader { + private class SmartReader { PushbackReader reader = null; int lastread = -1; public SmartReader(Reader r) { reader = new PushbackReader(r); } - public void unread() throws IOException { - reader.unread(lastread); + public void unread() throws IOException { unread((char)lastread); } + public void unread(char c) throws IOException { + reader.unread(c); if (accumulator != null) accumulator.setLength(accumulator.length() - 1); } public boolean match(char c) throws IOException { if (peek() == c) { reader.read(); return true; } else return false; } @@ -448,10 +458,11 @@ class Lexer { public int read() throws IOException { lastread = reader.read(); if (accumulator != null) accumulator.append((char)lastread); + if (lastread != '\n' && lastread != '\r') col++; return lastread; } - // FIXME: could be much more efficient + // FEATURE: could be much more efficient StringBuffer accumulator = null; public void startString() { accumulator = new StringBuffer(); @@ -464,4 +475,52 @@ class Lexer { } } + + // PushBack Stuff //////////////////////////////////////////////////////////// + + int pushBackDepth = 0; + int[] pushBackInts = new int[10]; + Object[] pushBackObjects = new Object[10]; + + public void pushBackToken(int op, Object obj) { + if (pushBackDepth >= pushBackInts.length - 1) { + int[] newInts = new int[pushBackInts.length * 2]; + System.arraycopy(pushBackInts, 0, newInts, 0, pushBackInts.length); + pushBackInts = newInts; + Object[] newObjects = new Object[pushBackObjects.length * 2]; + System.arraycopy(pushBackObjects, 0, newObjects, 0, pushBackObjects.length); + pushBackObjects = newObjects; + } + pushBackInts[pushBackDepth] = op; + pushBackObjects[pushBackDepth] = obj; + pushBackDepth++; + } + + public void pushBackToken() { pushBackToken(op, number != null ? (Object)number : (Object)string); } + + public int peekToken() throws IOException { + int ret = getToken(); + pushBackToken(); + return ret; + } + + public int getToken() throws IOException { + number = null; + string = null; + if (pushBackDepth > 0) { + pushBackDepth--; + op = pushBackInts[pushBackDepth]; + if (pushBackObjects[pushBackDepth] != null) { + number = pushBackObjects[pushBackDepth] instanceof Number ? (Number)pushBackObjects[pushBackDepth] : null; + string = pushBackObjects[pushBackDepth] instanceof String ? (String)pushBackObjects[pushBackDepth] : null; + } + } else { + do { + op = _getToken(); + if (op == EOL) { line++; col = 0; } + } while (op == EOL); + } + return op; + } + }