package org.xwt.js;
import java.io.*;
+/** Lexes a stream of characters into a stream of Tokens */
class Lexer implements Tokens {
+ /** for debugging */
public static void main(String[] s) throws Exception {
Lexer l = new Lexer(new InputStreamReader(System.in));
int tok = 0;
- while((tok = l.getToken()) != -1)
- System.out.println(codeToString[tok]);
+ while((tok = l.getToken()) != -1) System.out.println(codeToString[tok]);
}
- public int op;
- public Number number = null;
- public String string = null;
+ /** the token that was just parsed */
+ protected int op;
- public int line = 0;
- public int col = 0;
+ /** if the token just parsed was a NUMBER, this is the numeric value */
+ protected Number number = null;
- private SmartReader in;
- public String sourceName = "unknown";
+ /** if the token just parsed was a NAME or STRING, this is the string value */
+ protected String string = null;
+
+ /** the line number of the current token */
+ protected int line = 0;
+
+ /** the column number of the current token */
+ protected int col = 0;
+ /** the name of the source code file being lexed */
+ protected String sourceName = "unknown";
+
+ private SmartReader in;
public Lexer(Reader r) throws IOException { in = new SmartReader(r); }
// Predicates ///////////////////////////////////////////////////////////////////////
- protected static boolean isJSIdentifier(String s) {
- int length = s.length();
- if (length == 0 || !Character.isJavaIdentifierStart(s.charAt(0))) return false;
- for (int i=1; i<length; i++) {
- char c = s.charAt(i);
- if (!Character.isJavaIdentifierPart(c) && c == '\\' && !((i + 5) < length) &&
- (s.charAt(i + 1) == 'u') && 0 <= xDigitToInt(s.charAt(i + 2)) && 0 <= xDigitToInt(s.charAt(i + 3)) &&
- 0 <= xDigitToInt(s.charAt(i + 4)) && 0 <= xDigitToInt(s.charAt(i + 5)))
- return false;
- }
- return true;
- }
-
private static boolean isAlpha(int c) { return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')); }
private static boolean isDigit(int c) { return (c >= '0' && c <= '9'); }
- private static boolean isLineBreak(int c) { return (c == '\n' || c == '\r' || c == 0x2028 || c == 0x2029); }
private static int xDigitToInt(int c) {
if ('0' <= c && c <= '9') return c - '0';
- if ('a' <= c && c <= 'f') return c - ('a' - 10);
- if ('A' <= c && c <= 'F') return c - ('A' - 10);
- return -1;
- }
- public static boolean isWhiteSpace(int c) {
- if (c == '\u0020' || c == '\u0009' || c == '\u000C' || c == '\u000B' || c == '\u00A0') return true;
- return Character.getType((char)c) == Character.SPACE_SEPARATOR;
+ else if ('a' <= c && c <= 'f') return c - ('a' - 10);
+ else if ('A' <= c && c <= 'F') return c - ('A' - 10);
+ else return -1;
}
in.unread();
String str = in.getString();
int result = getKeyword(str);
- if (result == RESERVED) throw new IOException("the reserved word \"" + str + "\" is not permitted in XWT scripts");
+ if (result == RESERVED) throw new LexerException("The reserved word \"" + str + "\" is not permitted in XWT scripts");
if (result != -1) return result;
- this.string = str;
+ this.string = str.intern();
return NAME;
}
if (c == 'e' || c == 'E') {
c = in.read();
if (c == '+' || c == '-') c = in.read();
- if (!isDigit(c)) throw new IOException("msg.missing.exponent");
+ if (!isDigit(c)) throw new LexerException("float listeral did not have an exponent value");
do { c = in.read(); } while (isDigit(c));
}
}
String numString = in.getString();
if (base == 10 && !isInteger) {
try { dval = (Double.valueOf(numString)).doubleValue(); }
- catch (NumberFormatException ex) { throw new IOException("msg.caught.nfe"); }
+ catch (NumberFormatException ex) { throw new LexerException("invalid numeric literal: \"" + numString + "\""); }
} else {
if (isInteger) {
longval = Long.parseLong(numString, base);
c = in.read();
in.startString(); // start after the first "
while(c != quoteChar) {
- if (c == '\n' || c == -1) throw new IOException("msg.unterminated.string.lit");
+ if (c == '\n' || c == -1) throw new LexerException("unterminated string literal");
if (c == '\\') {
if (stringBuf == null) {
in.unread(); // Don't include the backslash
for(int i=0; i<4; i++) {
int ci = in.read();
if (!((ci >= '0' && ci <= '9') || (ci >= 'a' && ci <= 'f') || (ci >= 'A' && ci <= 'F')))
- throw new IOException("illegal character '" + ((char)c) + "' in \\u unicode escape sequence");
+ throw new LexerException("illegal character '" + ((char)c) + "' in \\u unicode escape sequence");
v = (v << 8) | Integer.parseInt(ci + "", 16);
}
c = (char)v;
if (stringBuf != null) stringBuf.append((char) c);
c = in.read();
}
- if (stringBuf != null) this.string = stringBuf.toString();
+ if (stringBuf != null) this.string = stringBuf.toString().intern();
else {
in.unread(); // miss the trailing "
- this.string = in.getString();
+ this.string = in.getString().intern();
in.read();
}
return STRING;
public int _getToken() throws IOException {
int c;
- do { if ((c = in.read()) == '\n') break; } while (isWhiteSpace(c) || c == '\n');
+ do { c = in.read(); } while (c == '\u0020' || c == '\u0009' || c == '\u000C' || c == '\u000B' || c == '\n' );
if (c == -1) return -1;
if (c == '\\' || Character.isJavaIdentifierStart((char)c)) return getIdentifier(c);
if (isDigit(c) || (c == '.' && isDigit(in.peek()))) return getNumber(c);
if (c == '"' || c == '\'') return getString(c);
switch (c) {
- case '\n': return EOL;
case ';': return SEMI;
case '[': return LB;
case ']': return RB;
case '+': return in.match('=') ? ASSIGN_ADD : in.match('+') ? INC : ADD;
case '-': return in.match('=') ? ASSIGN_SUB: in.match('-') ? DEC : SUB;
case '*': return in.match('=') ? ASSIGN_MUL : MUL;
- case '<': return !in.match('<') ?
- (in.match('=') ? LE : LT) :
- in.match('=') ? ASSIGN_LSH : LSH;
- case '>': return !in.match('>') ?
- (in.match('=') ? GE : GT) :
- in.match('>') ?
- (in.match('=') ? ASSIGN_URSH : URSH) :
- (in.match('=') ? ASSIGN_RSH : RSH);
+ case '<': return !in.match('<') ? (in.match('=') ? LE : LT) : in.match('=') ? ASSIGN_LSH : LSH;
+ case '>': return !in.match('>') ? (in.match('=') ? GE : GT) :
+ in.match('>') ? (in.match('=') ? ASSIGN_URSH : URSH) : (in.match('=') ? ASSIGN_RSH : RSH);
case '/':
if (in.match('=')) return ASSIGN_DIV;
if (in.match('/')) { while ((c = in.read()) != -1 && c != '\n'); in.unread(); return getToken(); }
while ((c = in.read()) != -1 && !(c == '*' && in.match('/'))) {
if (c == '\n' || c != '/' || !in.match('*')) continue;
if (in.match('/')) return getToken();
- throw new IOException("msg.nested.comment");
+ throw new LexerException("nested comments are not permitted");
}
- if (c == -1) throw new IOException("msg.unterminated.comment");
+ if (c == -1) throw new LexerException("unterminated comment");
return getToken(); // `goto retry'
- default: throw new IOException("illegal character: " + ((char)c));
+ default: throw new LexerException("illegal character: \'" + ((char)c) + "\'");
}
}
+
+ // SmartReader ////////////////////////////////////////////////////////////////
+
+ /** a Reader that tracks line numbers and can push back tokens */
private class SmartReader {
PushbackReader reader = null;
int lastread = -1;
lastread = reader.read();
if (accumulator != null) accumulator.append((char)lastread);
if (lastread != '\n' && lastread != '\r') col++;
+ if (lastread == '\n') { line++; col = 0; }
return lastread;
}
}
- // PushBack Stuff ////////////////////////////////////////////////////////////
+ // Token PushBack code ////////////////////////////////////////////////////////////
- int pushBackDepth = 0;
- int[] pushBackInts = new int[10];
- Object[] pushBackObjects = new Object[10];
+ private int pushBackDepth = 0;
+ private int[] pushBackInts = new int[10];
+ private Object[] pushBackObjects = new Object[10];
+ /** push back a token */
public void pushBackToken(int op, Object obj) {
if (pushBackDepth >= pushBackInts.length - 1) {
int[] newInts = new int[pushBackInts.length * 2];
pushBackDepth++;
}
+ /** push back the most recently read token */
public void pushBackToken() { pushBackToken(op, number != null ? (Object)number : (Object)string); }
+ /** read a token but leave it in the stream */
public int peekToken() throws IOException {
int ret = getToken();
pushBackToken();
return ret;
}
+ /** read a token */
public int getToken() throws IOException {
number = null;
string = null;
- if (pushBackDepth > 0) {
- pushBackDepth--;
- op = pushBackInts[pushBackDepth];
- if (pushBackObjects[pushBackDepth] != null) {
- number = pushBackObjects[pushBackDepth] instanceof Number ? (Number)pushBackObjects[pushBackDepth] : null;
- string = pushBackObjects[pushBackDepth] instanceof String ? (String)pushBackObjects[pushBackDepth] : null;
- }
- } else {
- do {
- op = _getToken();
- if (op == EOL) { line++; col = 0; }
- } while (op == EOL);
+ if (pushBackDepth == 0) return _getToken();
+ pushBackDepth--;
+ op = pushBackInts[pushBackDepth];
+ if (pushBackObjects[pushBackDepth] != null) {
+ number = pushBackObjects[pushBackDepth] instanceof Number ? (Number)pushBackObjects[pushBackDepth] : null;
+ string = pushBackObjects[pushBackDepth] instanceof String ? (String)pushBackObjects[pushBackDepth] : null;
}
return op;
}
+ class LexerException extends IOException {
+ public LexerException(String s) { super(sourceName + ":" + line + "," + col + " " + s); }
+ }
}
projects / org.ibex.core.git / blobdiff