-// This file was derived from org.mozilla.javascript.TokenStream; it
-// is covered by the NPL 1.1.
+// Derived from org.mozilla.javascript.TokenStream [NPL]
/**
* The contents of this file are subject to the Netscape Public
System.out.println(codeToString[tok]);
}
- private SmartReader in;
- private boolean pushedBack = false;
+ public int op;
+ public Number number = null;
+ public String string = null;
+
+ public int line = 0;
+ public int col = 0;
- private int op;
- public Number number;
- public String string;
+ private SmartReader in;
+ public String sourceName = "unknown";
public Lexer(Reader r) throws IOException { in = new SmartReader(r); }
- public int peekToken() throws IOException { int ret = getToken(); pushBackToken(); return ret; }
- public void pushBackToken() { if (pushedBack) throw new Error("can't push back twice"); pushedBack = true; }
+
// Token Constants //////////////////////////////////////////////////////////
else if (c=='o') { if (s.charAt(0)=='d') return DO; }
break;
case 3: switch (s.charAt(0)) {
+ case 'a': if (s.charAt(2)=='d' && s.charAt(1)=='n') return AND; break;
case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') return FOR; break;
case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') return RESERVED; break;
case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') throw new IOException("the new keyword is not permitted in XWT scripts");
longval = Long.parseLong(numString, base);
dval = (double)longval;
} else {
- // FIXME: we're not handling hex/octal fractions... does that matter?
dval = Double.parseDouble(numString);
longval = (long) dval;
if (longval == dval) isInteger = true;
case 'r': c = '\r'; break;
case 't': c = '\t'; break;
case 'v': c = '\u000B'; break;
- default: throw new IOException("\\u and \\0 escapes not currently supported -- use XML entities");
+ case '\\': c = '\\'; break;
+ case 'u': {
+ int v = 0;
+ for(int i=0; i<4; i++) {
+ int ci = in.read();
+ if (!((ci >= '0' && ci <= '9') || (ci >= 'a' && ci <= 'f') || (ci >= 'A' && ci <= 'F')))
+ throw new IOException("illegal character '" + ((char)c) + "' in \\u unicode escape sequence");
+ v = (v << 8) | Integer.parseInt(ci + "", 16);
+ }
+ c = (char)v;
+ break;
+ }
+ default:
+ // just use the character that was escaped
+ break;
}
}
if (stringBuf != null) stringBuf.append((char) c);
return STRING;
}
- public int getToken() throws IOException {
- if (pushedBack) { pushedBack = false; return op; }
- return (op = _getToken());
- }
-
public int _getToken() throws IOException {
int c;
do { if ((c = in.read()) == '\n') break; } while (isWhiteSpace(c) || c == '\n');
if (c == -1) return -1;
- if (c == '\\' && in.peek() == 'u') throw new IOException("\\u and \\0 escapes not currently supported -- use XML entities");
- if (Character.isJavaIdentifierStart((char)c)) return getIdentifier(c);
+ if (c == '\\' || Character.isJavaIdentifierStart((char)c)) return getIdentifier(c);
if (isDigit(c) || (c == '.' && isDigit(in.peek()))) return getNumber(c);
if (c == '"' || c == '\'') return getString(c);
switch (c) {
}
if (c == -1) throw new IOException("msg.unterminated.comment");
return getToken(); // `goto retry'
- default: throw new IOException("illegal character: " + c);
+ default: throw new IOException("illegal character: " + ((char)c));
}
}
- private static class SmartReader {
+ private class SmartReader {
PushbackReader reader = null;
int lastread = -1;
public SmartReader(Reader r) { reader = new PushbackReader(r); }
- public void unread() throws IOException {
- reader.unread(lastread);
+ public void unread() throws IOException { unread((char)lastread); }
+ public void unread(char c) throws IOException {
+ reader.unread(c);
if (accumulator != null) accumulator.setLength(accumulator.length() - 1);
}
public boolean match(char c) throws IOException { if (peek() == c) { reader.read(); return true; } else return false; }
public int read() throws IOException {
lastread = reader.read();
if (accumulator != null) accumulator.append((char)lastread);
+ if (lastread != '\n' && lastread != '\r') col++;
return lastread;
}
- // FIXME: could be much more efficient
+ // FEATURE: could be much more efficient
StringBuffer accumulator = null;
public void startString() {
accumulator = new StringBuffer();
}
}
+
+ // PushBack Stuff ////////////////////////////////////////////////////////////
+
+ int pushBackDepth = 0;
+ int[] pushBackInts = new int[10];
+ Object[] pushBackObjects = new Object[10];
+
+ public void pushBackToken(int op, Object obj) {
+ if (pushBackDepth >= pushBackInts.length - 1) {
+ int[] newInts = new int[pushBackInts.length * 2];
+ System.arraycopy(pushBackInts, 0, newInts, 0, pushBackInts.length);
+ pushBackInts = newInts;
+ Object[] newObjects = new Object[pushBackObjects.length * 2];
+ System.arraycopy(pushBackObjects, 0, newObjects, 0, pushBackObjects.length);
+ pushBackObjects = newObjects;
+ }
+ pushBackInts[pushBackDepth] = op;
+ pushBackObjects[pushBackDepth] = obj;
+ pushBackDepth++;
+ }
+
+ public void pushBackToken() { pushBackToken(op, number != null ? (Object)number : (Object)string); }
+
+ public int peekToken() throws IOException {
+ int ret = getToken();
+ pushBackToken();
+ return ret;
+ }
+
+ public int getToken() throws IOException {
+ number = null;
+ string = null;
+ if (pushBackDepth > 0) {
+ pushBackDepth--;
+ op = pushBackInts[pushBackDepth];
+ if (pushBackObjects[pushBackDepth] != null) {
+ number = pushBackObjects[pushBackDepth] instanceof Number ? (Number)pushBackObjects[pushBackDepth] : null;
+ string = pushBackObjects[pushBackDepth] instanceof String ? (String)pushBackObjects[pushBackDepth] : null;
+ }
+ } else {
+ do {
+ op = _getToken();
+ if (op == EOL) { line++; col = 0; }
+ } while (op == EOL);
+ }
+ return op;
+ }
+
}
projects / org.ibex.core.git / blobdiff