--- /dev/null
+// Derived from org.mozilla.javascript.TokenStream [NPL]
+
+/**
+ * The contents of this file are subject to the Netscape Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/NPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ *
+ * The Initial Developer of the Original Code is Netscape
+ * Communications Corporation.
+ *
+ * Contributor(s): Roger Lawrence, Mike McCabe
+ */
+
+package org.ibex.js;
+import java.io.*;
+
+/** Lexes a stream of characters into a stream of Tokens */
+class Lexer implements Tokens {
+
+ /** for debugging */
+ public static void main(String[] s) throws Exception {
+ Lexer l = new Lexer(new InputStreamReader(System.in), "stdin", 0);
+ int tok = 0;
+ while((tok = l.getToken()) != -1) System.out.println(codeToString[tok]);
+ }
+
+ /** the token that was just parsed */
+ protected int op;
+
+ /** the most recently parsed token, <i>regardless of pushbacks</i> */
+ protected int mostRecentlyReadToken;
+
+ /** if the token just parsed was a NUMBER, this is the numeric value */
+ protected Number number = null;
+
+ /** if the token just parsed was a NAME or STRING, this is the string value */
+ protected String string = null;
+
+ /** the line number of the most recently <i>lexed</i> token */
+ private int line = 0;
+
+ /** the line number of the most recently <i>parsed</i> token */
+ protected int parserLine = 0;
+
+ /** the column number of the current token */
+ protected int col = 0;
+
+ /** the name of the source code file being lexed */
+ protected String sourceName;
+
+ private SmartReader in;
+ public Lexer(Reader r, String sourceName, int line) throws IOException {
+ this.sourceName = sourceName;
+ this.line = line;
+ this.parserLine = line;
+ in = new SmartReader(r);
+ }
+
+
+ // Predicates ///////////////////////////////////////////////////////////////////////
+
+ private static boolean isAlpha(int c) { return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')); }
+ private static boolean isDigit(int c) { return (c >= '0' && c <= '9'); }
+ private static int xDigitToInt(int c) {
+ if ('0' <= c && c <= '9') return c - '0';
+ else if ('a' <= c && c <= 'f') return c - ('a' - 10);
+ else if ('A' <= c && c <= 'F') return c - ('A' - 10);
+ else return -1;
+ }
+
+
+ // Token Subtype Handlers /////////////////////////////////////////////////////////
+
+ private int getKeyword(String name) throws IOException {
+ //#switch(name)
+ case "if": return IF;
+ case "lt": return LT;
+ case "gt": return GT;
+ case "in": return IN;
+ case "do": return DO;
+ case "and": return AND;
+ case "or": return OR;
+ case "for": return FOR;
+ case "int": return RESERVED;
+ case "new": return RESERVED;
+ case "try": return TRY;
+ case "var": return VAR;
+ case "byte": return RESERVED;
+ case "case": return CASE;
+ case "char": return RESERVED;
+ case "else": return ELSE;
+ case "enum": return RESERVED;
+ case "goto": return RESERVED;
+ case "long": return RESERVED;
+ case "null": return NULL;
+ case "true": return TRUE;
+ case "with": return RESERVED;
+ case "void": return RESERVED;
+ case "class": return RESERVED;
+ case "break": return BREAK;
+ case "while": return WHILE;
+ case "false": return FALSE;
+ case "const": return RESERVED;
+ case "final": return RESERVED;
+ case "super": return RESERVED;
+ case "throw": return THROW;
+ case "catch": return CATCH;
+ case "class": return RESERVED;
+ case "delete": return RESERVED;
+ case "return": return RETURN;
+ case "throws": return RESERVED;
+ case "double": return RESERVED;
+ case "assert": return ASSERT;
+ case "public": return RESERVED;
+ case "switch": return SWITCH;
+ case "typeof": return TYPEOF;
+ case "package": return RESERVED;
+ case "default": return DEFAULT;
+ case "finally": return FINALLY;
+ case "boolean": return RESERVED;
+ case "private": return RESERVED;
+ case "extends": return RESERVED;
+ case "abstract": return RESERVED;
+ case "continue": return CONTINUE;
+ case "debugger": return RESERVED;
+ case "function": return FUNCTION;
+ case "volatile": return RESERVED;
+ case "interface": return RESERVED;
+ case "protected": return RESERVED;
+ case "transient": return RESERVED;
+ case "implements": return RESERVED;
+ case "instanceof": return RESERVED;
+ case "synchronized": return RESERVED;
+ //#end
+ return -1;
+ }
+
+ private int getIdentifier(int c) throws IOException {
+ in.startString();
+ while (Character.isJavaIdentifierPart((char)(c = in.read())));
+ in.unread();
+ String str = in.getString();
+ int result = getKeyword(str);
+ if (result == RESERVED) throw new LexerException("The reserved word \"" + str + "\" is not permitted in Ibex scripts");
+ if (result != -1) return result;
+ this.string = str.intern();
+ return NAME;
+ }
+
+ private int getNumber(int c) throws IOException {
+ int base = 10;
+ in.startString();
+ double dval = Double.NaN;
+ long longval = 0;
+ boolean isInteger = true;
+
+ // figure out what base we're using
+ if (c == '0') {
+ if (Character.toLowerCase((char)(c = in.read())) == 'x') { base = 16; in.startString(); }
+ else if (isDigit(c)) base = 8;
+ }
+
+ while (0 <= xDigitToInt(c) && !(base < 16 && isAlpha(c))) c = in.read();
+ if (base == 10 && (c == '.' || c == 'e' || c == 'E')) {
+ isInteger = false;
+ if (c == '.') do { c = in.read(); } while (isDigit(c));
+ if (c == 'e' || c == 'E') {
+ c = in.read();
+ if (c == '+' || c == '-') c = in.read();
+ if (!isDigit(c)) throw new LexerException("float listeral did not have an exponent value");
+ do { c = in.read(); } while (isDigit(c));
+ }
+ }
+ in.unread();
+
+ String numString = in.getString();
+ if (base == 10 && !isInteger) {
+ try { dval = (Double.valueOf(numString)).doubleValue(); }
+ catch (NumberFormatException ex) { throw new LexerException("invalid numeric literal: \"" + numString + "\""); }
+ } else {
+ if (isInteger) {
+ longval = Long.parseLong(numString, base);
+ dval = (double)longval;
+ } else {
+ dval = Double.parseDouble(numString);
+ longval = (long) dval;
+ if (longval == dval) isInteger = true;
+ }
+ }
+
+ if (!isInteger) this.number = JS.N(dval);
+ else this.number = JS.N(longval);
+ return NUMBER;
+ }
+
+ private int getString(int c) throws IOException {
+ StringBuffer stringBuf = null;
+ int quoteChar = c;
+ int val = 0;
+ c = in.read();
+ in.startString(); // start after the first "
+ while(c != quoteChar) {
+ if (c == '\n' || c == -1) throw new LexerException("unterminated string literal");
+ if (c == '\\') {
+ if (stringBuf == null) {
+ in.unread(); // Don't include the backslash
+ stringBuf = new StringBuffer(in.getString());
+ in.read();
+ }
+ switch (c = in.read()) {
+ case 'b': c = '\b'; break;
+ case 'f': c = '\f'; break;
+ case 'n': c = '\n'; break;
+ case 'r': c = '\r'; break;
+ case 't': c = '\t'; break;
+ case 'v': c = '\u000B'; break;
+ case '\\': c = '\\'; break;
+ case 'u': {
+ int v = 0;
+ for(int i=0; i<4; i++) {
+ int ci = in.read();
+ if (!((ci >= '0' && ci <= '9') || (ci >= 'a' && ci <= 'f') || (ci >= 'A' && ci <= 'F')))
+ throw new LexerException("illegal character '" + ((char)c) + "' in \\u unicode escape sequence");
+ v = (v << 8) | Integer.parseInt(ci + "", 16);
+ }
+ c = (char)v;
+ break;
+ }
+ default:
+ // just use the character that was escaped
+ break;
+ }
+ }
+ if (stringBuf != null) stringBuf.append((char) c);
+ c = in.read();
+ }
+ if (stringBuf != null) this.string = stringBuf.toString().intern();
+ else {
+ in.unread(); // miss the trailing "
+ this.string = in.getString().intern();
+ in.read();
+ }
+ return STRING;
+ }
+
+ private int _getToken() throws IOException {
+ int c;
+ do { c = in.read(); } while (c == '\u0020' || c == '\u0009' || c == '\u000C' || c == '\u000B' || c == '\n' );
+ if (c == -1) return -1;
+ if (c == '\\' || Character.isJavaIdentifierStart((char)c)) return getIdentifier(c);
+ if (isDigit(c) || (c == '.' && isDigit(in.peek()))) return getNumber(c);
+ if (c == '"' || c == '\'') return getString(c);
+ switch (c) {
+ case ';': return SEMI;
+ case '[': return LB;
+ case ']': return RB;
+ case '{': return LC;
+ case '}': return RC;
+ case '(': return LP;
+ case ')': return RP;
+ case ',': return COMMA;
+ case '?': return HOOK;
+ case ':': return !in.match(':') ? COLON : in.match('=') ? GRAMMAR : le(":: is not a valid token");
+ case '.': return DOT;
+ case '|': return in.match('|') ? OR : (in.match('=') ? ASSIGN_BITOR : BITOR);
+ case '^': return in.match('=') ? ASSIGN_BITXOR : BITXOR;
+ case '&': return in.match('&') ? AND : in.match('=') ? ASSIGN_BITAND : BITAND;
+ case '=': return !in.match('=') ? ASSIGN : in.match('=') ? SHEQ : EQ;
+ case '!': return !in.match('=') ? BANG : in.match('=') ? SHNE : NE;
+ case '%': return in.match('=') ? ASSIGN_MOD : MOD;
+ case '~': return BITNOT;
+ case '+': return in.match('=') ? ASSIGN_ADD : in.match('+') ? (in.match('=') ? ADD_TRAP : INC) : ADD;
+ case '-': return in.match('=') ? ASSIGN_SUB: in.match('-') ? (in.match('=') ? DEL_TRAP : DEC) : SUB;
+ case '*': return in.match('=') ? ASSIGN_MUL : MUL;
+ case '<': return !in.match('<') ? (in.match('=') ? LE : LT) : in.match('=') ? ASSIGN_LSH : LSH;
+ case '>': return !in.match('>') ? (in.match('=') ? GE : GT) :
+ in.match('>') ? (in.match('=') ? ASSIGN_URSH : URSH) : (in.match('=') ? ASSIGN_RSH : RSH);
+ case '/':
+ if (in.match('=')) return ASSIGN_DIV;
+ if (in.match('/')) { while ((c = in.read()) != -1 && c != '\n'); in.unread(); return getToken(); }
+ if (!in.match('*')) return DIV;
+ while ((c = in.read()) != -1 && !(c == '*' && in.match('/'))) {
+ if (c == '\n' || c != '/' || !in.match('*')) continue;
+ if (in.match('/')) return getToken();
+ throw new LexerException("nested comments are not permitted");
+ }
+ if (c == -1) throw new LexerException("unterminated comment");
+ return getToken(); // `goto retry'
+ default: throw new LexerException("illegal character: \'" + ((char)c) + "\'");
+ }
+ }
+
+ private int le(String s) throws LexerException { if (true) throw new LexerException(s); return 0; }
+
+ // SmartReader ////////////////////////////////////////////////////////////////
+
+ /** a Reader that tracks line numbers and can push back tokens */
+ private class SmartReader {
+ PushbackReader reader = null;
+ int lastread = -1;
+
+ public SmartReader(Reader r) { reader = new PushbackReader(r); }
+ public void unread() throws IOException { unread((char)lastread); }
+ public void unread(char c) throws IOException {
+ reader.unread(c);
+ if(c == '\n') col = -1;
+ else col--;
+ if (accumulator != null) accumulator.setLength(accumulator.length() - 1);
+ }
+ public boolean match(char c) throws IOException { if (peek() == c) { reader.read(); return true; } else return false; }
+ public int peek() throws IOException {
+ int peeked = reader.read();
+ if (peeked != -1) reader.unread((char)peeked);
+ return peeked;
+ }
+ public int read() throws IOException {
+ lastread = reader.read();
+ if (accumulator != null) accumulator.append((char)lastread);
+ if (lastread != '\n' && lastread != '\r') col++;
+ if (lastread == '\n') {
+ // col is -1 if we just unread a newline, this is sort of ugly
+ if (col != -1) parserLine = ++line;
+ col = 0;
+ }
+ return lastread;
+ }
+
+ // FEATURE: could be much more efficient
+ StringBuffer accumulator = null;
+ public void startString() {
+ accumulator = new StringBuffer();
+ accumulator.append((char)lastread);
+ }
+ public String getString() throws IOException {
+ String ret = accumulator.toString().intern();
+ accumulator = null;
+ return ret;
+ }
+ }
+
+
+ // Token PushBack code ////////////////////////////////////////////////////////////
+
+ private int pushBackDepth = 0;
+ private int[] pushBackInts = new int[10];
+ private Object[] pushBackObjects = new Object[10];
+
+ /** push back a token */
+ public final void pushBackToken(int op, Object obj) {
+ if (pushBackDepth >= pushBackInts.length - 1) {
+ int[] newInts = new int[pushBackInts.length * 2];
+ System.arraycopy(pushBackInts, 0, newInts, 0, pushBackInts.length);
+ pushBackInts = newInts;
+ Object[] newObjects = new Object[pushBackObjects.length * 2];
+ System.arraycopy(pushBackObjects, 0, newObjects, 0, pushBackObjects.length);
+ pushBackObjects = newObjects;
+ }
+ pushBackInts[pushBackDepth] = op;
+ pushBackObjects[pushBackDepth] = obj;
+ pushBackDepth++;
+ }
+
+ /** push back the most recently read token */
+ public final void pushBackToken() { pushBackToken(op, number != null ? (Object)number : (Object)string); }
+
+ /** read a token but leave it in the stream */
+ public final int peekToken() throws IOException {
+ int ret = getToken();
+ pushBackToken();
+ return ret;
+ }
+
+ /** read a token */
+ public final int getToken() throws IOException {
+ number = null;
+ string = null;
+ if (pushBackDepth == 0) {
+ mostRecentlyReadToken = op;
+ return op = _getToken();
+ }
+ pushBackDepth--;
+ op = pushBackInts[pushBackDepth];
+ if (pushBackObjects[pushBackDepth] != null) {
+ number = pushBackObjects[pushBackDepth] instanceof Number ? (Number)pushBackObjects[pushBackDepth] : null;
+ string = pushBackObjects[pushBackDepth] instanceof String ? (String)pushBackObjects[pushBackDepth] : null;
+ }
+ return op;
+ }
+
+ class LexerException extends IOException {
+ public LexerException(String s) { super(sourceName + ":" + line + "," + col + ": " + s); }
+ }
+}
projects / org.ibex.core.git / blobdiff