1 /*******************************************************************************
2 * Copyright (c) 2000, 2004 IBM Corporation and others.
3 * All rights reserved. This program and the accompanying materials
4 * are made available under the terms of the Common Public License v1.0
5 * which accompanies this distribution, and is available at
6 * http://www.eclipse.org/legal/cpl-v10.html
9 * IBM Corporation - initial API and implementation
10 *******************************************************************************/
11 package org.eclipse.jdt.internal.compiler.parser;
13 import java.util.Iterator;
15 import org.eclipse.jdt.core.compiler.CharOperation;
16 import org.eclipse.jdt.core.compiler.InvalidInputException;
17 import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
18 import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
21 * IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in
22 * org.eclipse.jdt.core.compiler public package where it is API.
23 * The mirror implementation is using the backward compatible ITerminalSymbols constant
24 * definitions (stable with 2.0), whereas the internal implementation uses TerminalTokens
25 * which constant values reflect the latest parser generation state.
27 public class Scanner implements TerminalTokens {
30 - getNextToken() which return the current type of the token
31 (this value is not memorized by the scanner)
32 - getCurrentTokenSource() which provides with the token "REAL" source
33 (aka all unicode have been transformed into a correct char)
34 - sourceStart gives the position into the stream
35 - currentPosition-1 gives the sourceEnd position into the stream
39 private boolean assertMode = false;
40 public boolean useAssertAsAnIndentifier = false;
41 //flag indicating if processed source contains occurrences of keyword assert
42 public boolean containsAssertKeyword = false;
44 public boolean recordLineSeparator = false;
45 public char currentCharacter;
46 public int startPosition;
47 public int currentPosition;
48 public int initialPosition, eofPosition;
49 // after this position eof are generated instead of real token from the source
51 public boolean tokenizeComments = false;
52 public boolean tokenizeWhiteSpace = false;
54 //source should be viewed as a window (aka a part)
55 //of a entire very large stream
59 public char[] withoutUnicodeBuffer;
60 public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token
61 public boolean unicodeAsBackSlash = false;
63 public boolean scanningFloatLiteral = false;
65 //support for /** comments
66 public int[] commentStops = new int[10];
67 public int[] commentStarts = new int[10];
68 public int commentPtr = -1; // no comment test with commentPtr value -1
69 protected int lastCommentLinePosition = -1;
72 public char[][] foundTaskTags = null;
73 public char[][] foundTaskMessages;
74 public char[][] foundTaskPriorities = null;
75 public int[][] foundTaskPositions;
76 public int foundTaskCount = 0;
77 public char[][] taskTags = null;
78 public char[][] taskPriorities = null;
79 public boolean isTaskCaseSensitive = true;
81 //diet parsing support - jump over some method body when requested
82 public boolean diet = false;
84 //support for the poor-line-debuggers ....
85 //remember the position of the cr/lf
86 public int[] lineEnds = new int[250];
87 public int linePtr = -1;
88 public boolean wasAcr = false;
90 public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
92 public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
93 public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
94 public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
95 public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
96 public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
97 public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
98 public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
100 public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
101 public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
102 public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
103 public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
105 //----------------optimized identifier managment------------------
106 static final char[] charArray_a = new char[] {'a'},
107 charArray_b = new char[] {'b'},
108 charArray_c = new char[] {'c'},
109 charArray_d = new char[] {'d'},
110 charArray_e = new char[] {'e'},
111 charArray_f = new char[] {'f'},
112 charArray_g = new char[] {'g'},
113 charArray_h = new char[] {'h'},
114 charArray_i = new char[] {'i'},
115 charArray_j = new char[] {'j'},
116 charArray_k = new char[] {'k'},
117 charArray_l = new char[] {'l'},
118 charArray_m = new char[] {'m'},
119 charArray_n = new char[] {'n'},
120 charArray_o = new char[] {'o'},
121 charArray_p = new char[] {'p'},
122 charArray_q = new char[] {'q'},
123 charArray_r = new char[] {'r'},
124 charArray_s = new char[] {'s'},
125 charArray_t = new char[] {'t'},
126 charArray_u = new char[] {'u'},
127 charArray_v = new char[] {'v'},
128 charArray_w = new char[] {'w'},
129 charArray_x = new char[] {'x'},
130 charArray_y = new char[] {'y'},
131 charArray_z = new char[] {'z'};
133 static final char[] initCharArray =
134 new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'};
135 static final int TableSize = 30, InternalTableSize = 6; //30*6 = 180 entries
136 public static final int OptimizedLength = 6;
137 public /*static*/ final char[][][][] charArray_length =
138 new char[OptimizedLength][TableSize][InternalTableSize][];
139 // support for detecting non-externalized string literals
140 NLSLine currentLine= null;
141 public static final String TAG_PREFIX= "//$NON-NLS-"; //$NON-NLS-1$
142 public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length();
143 public static final String TAG_POSTFIX= "$"; //$NON-NLS-1$
144 public static final int TAG_POSTFIX_LENGTH= TAG_POSTFIX.length();
145 public StringLiteral[] nonNLSStrings = null;
146 public boolean checkNonExternalizedStringLiterals = false;
147 public boolean wasNonExternalizedStringLiteral = false;
150 for (int i = 0; i < 6; i++) {
151 for (int j = 0; j < TableSize; j++) {
152 for (int k = 0; k < InternalTableSize; k++) {
153 this.charArray_length[i][j][k] = initCharArray;
158 static int newEntry2 = 0,
164 public static final int RoundBracket = 0;
165 public static final int SquareBracket = 1;
166 public static final int CurlyBracket = 2;
167 public static final int BracketKinds = 3;
170 this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/);
174 boolean tokenizeComments,
175 boolean tokenizeWhiteSpace,
176 boolean checkNonExternalizedStringLiterals,
179 char[][] taskPriorities,
180 boolean isTaskCaseSensitive) {
182 this.eofPosition = Integer.MAX_VALUE;
183 this.tokenizeComments = tokenizeComments;
184 this.tokenizeWhiteSpace = tokenizeWhiteSpace;
185 this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
186 this.assertMode = sourceLevel >= ClassFileConstants.JDK1_4;
187 this.taskTags = taskTags;
188 this.taskPriorities = taskPriorities;
189 this.isTaskCaseSensitive = isTaskCaseSensitive;
192 public final boolean atEnd() {
193 // This code is not relevant if source is
194 // Only a part of the real stream input
196 return this.source.length == this.currentPosition;
199 private void checkNonExternalizedString() {
200 if (this.currentLine == null)
202 parseTags(this.currentLine);
205 // chech presence of task: tags
206 // TODO (frederic) see if we need to take unicode characters into account...
207 public void checkTaskTag(int commentStart, int commentEnd) {
208 char[] src = this.source;
210 // only look for newer task: tags
211 if (this.foundTaskCount > 0
212 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
215 int foundTaskIndex = this.foundTaskCount;
216 char previous = src[commentStart+1]; // should be '*' or '/'
218 int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) {
220 char[] priority = null;
221 // check for tag occurrence only if not ambiguous with javadoc tag
222 if (previous != '@') {
223 nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) {
224 tag = this.taskTags[itag];
225 int tagLength = tag.length;
226 if (tagLength == 0) continue nextTag;
228 // ensure tag is not leaded with letter if tag starts with a letter
229 if (Character.isJavaIdentifierStart(tag[0])) {
230 if (Character.isJavaIdentifierPart(previous)) {
235 for (int t = 0; t < tagLength; t++) {
238 if (x >= this.eofPosition || x >= commentEnd) continue nextTag;
239 if ((sc = src[i + t]) != (tc = tag[t])) { // case sensitive check
240 if (this.isTaskCaseSensitive || (Character.toLowerCase(sc) != Character.toLowerCase(tc))) { // case insensitive check
245 // ensure tag is not followed with letter if tag finishes with a letter
246 if (i+tagLength < commentEnd && Character.isJavaIdentifierPart(src[i+tagLength-1])) {
247 if (Character.isJavaIdentifierPart(src[i + tagLength]))
250 if (this.foundTaskTags == null) {
251 this.foundTaskTags = new char[5][];
252 this.foundTaskMessages = new char[5][];
253 this.foundTaskPriorities = new char[5][];
254 this.foundTaskPositions = new int[5][];
255 } else if (this.foundTaskCount == this.foundTaskTags.length) {
256 System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
257 System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
258 System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
259 System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0, this.foundTaskCount);
262 priority = this.taskPriorities != null && itag < this.taskPriorities.length
263 ? this.taskPriorities[itag]
266 this.foundTaskTags[this.foundTaskCount] = tag;
267 this.foundTaskPriorities[this.foundTaskCount] = priority;
268 this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
269 this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
270 this.foundTaskCount++;
271 i += tagLength - 1; // will be incremented when looping
277 for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
278 // retrieve message start and end positions
279 int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
280 int max_value = i + 1 < this.foundTaskCount
281 ? this.foundTaskPositions[i + 1][0] - 1
283 // at most beginning of next task
284 if (max_value < msgStart) {
285 max_value = msgStart; // would only occur if tag is before EOF.
289 for (int j = msgStart; j < max_value; j++) {
290 if ((c = src[j]) == '\n' || c == '\r') {
296 for (int j = max_value; j > msgStart; j--) {
297 if ((c = src[j]) == '*') {
308 while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
310 while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
312 // update the end position of the task
313 this.foundTaskPositions[i][1] = end;
314 // get the message source
315 final int messageLength = end - msgStart + 1;
316 char[] message = new char[messageLength];
317 System.arraycopy(src, msgStart, message, 0, messageLength);
318 this.foundTaskMessages[i] = message;
321 public char[] getCurrentIdentifierSource() {
322 //return the token REAL source (aka unicodes are precomputed)
325 if (this.withoutUnicodePtr != 0) {
326 //0 is used as a fast test flag so the real first char is in position 1
328 this.withoutUnicodeBuffer,
330 result = new char[this.withoutUnicodePtr],
332 this.withoutUnicodePtr);
334 int length = this.currentPosition - this.startPosition;
335 if (length == this.source.length) return this.source;
336 switch (length) { // see OptimizedLength
338 return optimizedCurrentTokenSource1();
340 return optimizedCurrentTokenSource2();
342 return optimizedCurrentTokenSource3();
344 return optimizedCurrentTokenSource4();
346 return optimizedCurrentTokenSource5();
348 return optimizedCurrentTokenSource6();
351 System.arraycopy(this.source, this.startPosition, result = new char[length], 0, length);
355 public int getCurrentTokenEndPosition(){
356 return this.currentPosition - 1;
358 public final char[] getCurrentTokenSource() {
359 // Return the token REAL source (aka unicodes are precomputed)
362 if (this.withoutUnicodePtr != 0)
363 // 0 is used as a fast test flag so the real first char is in position 1
365 this.withoutUnicodeBuffer,
367 result = new char[this.withoutUnicodePtr],
369 this.withoutUnicodePtr);
375 result = new char[length = this.currentPosition - this.startPosition],
381 public final char[] getCurrentTokenSourceString() {
382 //return the token REAL source (aka unicodes are precomputed).
383 //REMOVE the two " that are at the beginning and the end.
386 if (this.withoutUnicodePtr != 0)
387 //0 is used as a fast test flag so the real first char is in position 1
388 System.arraycopy(this.withoutUnicodeBuffer, 2,
389 //2 is 1 (real start) + 1 (to jump over the ")
390 result = new char[this.withoutUnicodePtr - 2], 0, this.withoutUnicodePtr - 2);
395 this.startPosition + 1,
396 result = new char[length = this.currentPosition - this.startPosition - 2],
403 public final char[] getRawTokenSource() {
404 int length = this.currentPosition - this.startPosition;
405 char[] tokenSource = new char[length];
406 System.arraycopy(this.source, this.startPosition, tokenSource, 0, length);
410 public final char[] getRawTokenSourceEnd() {
411 int length = this.eofPosition - this.currentPosition - 1;
412 char[] sourceEnd = new char[length];
413 System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length);
417 public int getCurrentTokenStartPosition(){
418 return this.startPosition;
421 * Search the source position corresponding to the end of a given line number
423 * Line numbers are 1-based, and relative to the scanner initialPosition.
424 * Character positions are 0-based.
426 * In case the given line number is inconsistent, answers -1.
428 public final int getLineEnd(int lineNumber) {
430 if (this.lineEnds == null)
432 if (lineNumber > this.lineEnds.length+1)
436 if (lineNumber == this.lineEnds.length + 1)
437 return this.eofPosition;
438 return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
441 public final int[] getLineEnds() {
442 //return a bounded copy of this.lineEnds
445 System.arraycopy(this.lineEnds, 0, copy = new int[this.linePtr + 1], 0, this.linePtr + 1);
450 * Search the source position corresponding to the beginning of a given line number
452 * Line numbers are 1-based, and relative to the scanner initialPosition.
453 * Character positions are 0-based.
455 * e.g. getLineStart(1) --> 0 indicates that the first line starts at character 0.
457 * In case the given line number is inconsistent, answers -1.
459 * @param lineNumber int
462 public final int getLineStart(int lineNumber) {
464 if (this.lineEnds == null)
466 if (lineNumber > this.lineEnds.length + 1)
472 return this.initialPosition;
473 return this.lineEnds[lineNumber-2]+1; // next line start one character behind the lineEnd of the previous line
475 public final int getNextChar() {
477 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
478 && (this.source[this.currentPosition] == 'u')) {
479 //-------------unicode traitement ------------
482 this.currentPosition++;
483 while (this.source[this.currentPosition] == 'u') {
484 this.currentPosition++;
488 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
490 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
491 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
492 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
496 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
498 this.unicodeAsBackSlash = this.currentCharacter == '\\';
500 //need the unicode buffer
501 if (this.withoutUnicodePtr == 0) {
502 //buffer all the entries that have been left aside....
503 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
505 //fill the buffer with the char
506 unicodeStoreAt(++this.withoutUnicodePtr);
507 return this.currentCharacter;
509 } //-------------end unicode traitement--------------
511 this.unicodeAsBackSlash = false;
512 if (this.withoutUnicodePtr != 0) {
513 unicodeStoreAt(++this.withoutUnicodePtr);
515 return this.currentCharacter;
517 } catch (IndexOutOfBoundsException e) {
521 public final boolean getNextChar(char testedChar) {
523 //handle the case of unicode.
524 //when a unicode appears then we must use a buffer that holds char internal values
525 //At the end of this method currentCharacter holds the new visited char
526 //and currentPosition points right next after it
527 //Both previous lines are true if the currentCharacter is == to the testedChar
528 //On false, no side effect has occured.
530 //ALL getNextChar.... ARE OPTIMIZED COPIES
532 if (this.currentPosition >= this.source.length) { // handle the obvious case upfront
533 this.unicodeAsBackSlash = false;
537 int temp = this.currentPosition;
539 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
540 && (this.source[this.currentPosition] == 'u')) {
541 //-------------unicode traitement ------------
544 this.currentPosition++;
545 while (this.source[this.currentPosition] == 'u') {
546 this.currentPosition++;
550 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
552 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
553 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
554 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
555 this.currentPosition = temp;
559 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
560 if (this.currentCharacter != testedChar) {
561 this.currentPosition = temp;
564 this.unicodeAsBackSlash = this.currentCharacter == '\\';
566 //need the unicode buffer
567 if (this.withoutUnicodePtr == 0) {
568 //buffer all the entries that have been left aside....
569 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
571 //fill the buffer with the char
572 unicodeStoreAt(++this.withoutUnicodePtr);
575 } //-------------end unicode traitement--------------
577 if (this.currentCharacter != testedChar) {
578 this.currentPosition = temp;
581 this.unicodeAsBackSlash = false;
582 if (this.withoutUnicodePtr != 0)
583 unicodeStoreAt(++this.withoutUnicodePtr);
586 } catch (IndexOutOfBoundsException e) {
587 this.unicodeAsBackSlash = false;
588 this.currentPosition = temp;
592 public final int getNextChar(char testedChar1, char testedChar2) {
593 //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
594 //test can be done with (x==0) for the first and (x>0) for the second
595 //handle the case of unicode.
596 //when a unicode appears then we must use a buffer that holds char internal values
597 //At the end of this method currentCharacter holds the new visited char
598 //and currentPosition points right next after it
599 //Both previous lines are true if the currentCharacter is == to the testedChar1/2
600 //On false, no side effect has occured.
602 //ALL getNextChar.... ARE OPTIMIZED COPIES
603 if (this.currentPosition >= this.source.length) // handle the obvious case upfront
606 int temp = this.currentPosition;
609 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
610 && (this.source[this.currentPosition] == 'u')) {
611 //-------------unicode traitement ------------
614 this.currentPosition++;
615 while (this.source[this.currentPosition] == 'u') {
616 this.currentPosition++;
620 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
622 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
623 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
624 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
625 this.currentPosition = temp;
629 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
630 if (this.currentCharacter == testedChar1)
633 if (this.currentCharacter == testedChar2)
636 this.currentPosition = temp;
640 //need the unicode buffer
641 if (this.withoutUnicodePtr == 0) {
642 //buffer all the entries that have been left aside....
643 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
645 //fill the buffer with the char
646 unicodeStoreAt(++this.withoutUnicodePtr);
648 } //-------------end unicode traitement--------------
650 if (this.currentCharacter == testedChar1)
653 if (this.currentCharacter == testedChar2)
656 this.currentPosition = temp;
660 if (this.withoutUnicodePtr != 0)
661 unicodeStoreAt(++this.withoutUnicodePtr);
664 } catch (IndexOutOfBoundsException e) {
665 this.currentPosition = temp;
669 public final boolean getNextCharAsDigit() throws InvalidInputException {
671 //handle the case of unicode.
672 //when a unicode appears then we must use a buffer that holds char internal values
673 //At the end of this method currentCharacter holds the new visited char
674 //and currentPosition points right next after it
675 //Both previous lines are true if the currentCharacter is a digit
676 //On false, no side effect has occured.
678 //ALL getNextChar.... ARE OPTIMIZED COPIES
679 if (this.currentPosition >= this.source.length) // handle the obvious case upfront
682 int temp = this.currentPosition;
684 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
685 && (this.source[this.currentPosition] == 'u')) {
686 //-------------unicode traitement ------------
689 this.currentPosition++;
690 while (this.source[this.currentPosition] == 'u') {
691 this.currentPosition++;
695 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
697 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
698 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
699 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
700 this.currentPosition = temp;
704 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
705 if (!isDigit(this.currentCharacter)) {
706 this.currentPosition = temp;
710 //need the unicode buffer
711 if (this.withoutUnicodePtr == 0) {
712 //buffer all the entries that have been left aside....
713 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
715 //fill the buffer with the char
716 unicodeStoreAt(++this.withoutUnicodePtr);
718 } //-------------end unicode traitement--------------
720 if (!isDigit(this.currentCharacter)) {
721 this.currentPosition = temp;
724 if (this.withoutUnicodePtr != 0)
725 unicodeStoreAt(++this.withoutUnicodePtr);
728 } catch (IndexOutOfBoundsException e) {
729 this.currentPosition = temp;
733 public final boolean getNextCharAsDigit(int radix) {
735 //handle the case of unicode.
736 //when a unicode appears then we must use a buffer that holds char internal values
737 //At the end of this method currentCharacter holds the new visited char
738 //and currentPosition points right next after it
739 //Both previous lines are true if the currentCharacter is a digit base on radix
740 //On false, no side effect has occured.
742 //ALL getNextChar.... ARE OPTIMIZED COPIES
743 if (this.currentPosition >= this.source.length) // handle the obvious case upfront
746 int temp = this.currentPosition;
748 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
749 && (this.source[this.currentPosition] == 'u')) {
750 //-------------unicode traitement ------------
753 this.currentPosition++;
754 while (this.source[this.currentPosition] == 'u') {
755 this.currentPosition++;
759 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
761 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
762 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
763 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
764 this.currentPosition = temp;
768 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
769 if (Character.digit(this.currentCharacter, radix) == -1) {
770 this.currentPosition = temp;
774 //need the unicode buffer
775 if (this.withoutUnicodePtr == 0) {
776 //buffer all the entries that have been left aside....
777 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
779 //fill the buffer with the char
780 unicodeStoreAt(++this.withoutUnicodePtr);
782 } //-------------end unicode traitement--------------
784 if (Character.digit(this.currentCharacter, radix) == -1) {
785 this.currentPosition = temp;
788 if (this.withoutUnicodePtr != 0)
789 unicodeStoreAt(++this.withoutUnicodePtr);
792 } catch (IndexOutOfBoundsException e) {
793 this.currentPosition = temp;
797 public boolean getNextCharAsJavaIdentifierPart() {
799 //handle the case of unicode.
800 //when a unicode appears then we must use a buffer that holds char internal values
801 //At the end of this method currentCharacter holds the new visited char
802 //and currentPosition points right next after it
803 //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
804 //On false, no side effect has occured.
806 //ALL getNextChar.... ARE OPTIMIZED COPIES
807 if (this.currentPosition >= this.source.length) // handle the obvious case upfront
810 int temp = this.currentPosition;
812 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
813 && (this.source[this.currentPosition] == 'u')) {
814 //-------------unicode traitement ------------
817 this.currentPosition++;
818 while (this.source[this.currentPosition] == 'u') {
819 this.currentPosition++;
823 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
825 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
826 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
827 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
828 this.currentPosition = temp;
832 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
833 if (!Character.isJavaIdentifierPart(this.currentCharacter)) {
834 this.currentPosition = temp;
838 //need the unicode buffer
839 if (this.withoutUnicodePtr == 0) {
840 //buffer all the entries that have been left aside....
841 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
843 //fill the buffer with the char
844 unicodeStoreAt(++this.withoutUnicodePtr);
846 } //-------------end unicode traitement--------------
848 if (!Character.isJavaIdentifierPart(this.currentCharacter)) {
849 this.currentPosition = temp;
853 if (this.withoutUnicodePtr != 0)
854 unicodeStoreAt(++this.withoutUnicodePtr);
857 } catch (IndexOutOfBoundsException e) {
858 this.currentPosition = temp;
862 public int getNextToken() throws InvalidInputException {
865 jumpOverMethodBody();
867 return this.currentPosition > this.source.length ? TokenNameEOF : TokenNameRBRACE;
871 while (true) { //loop for jumping over comments
872 this.withoutUnicodePtr = 0;
873 //start with a new token (even comment written with unicode )
875 // ---------Consume white space and handles startPosition---------
876 whiteStart = this.currentPosition;
877 boolean isWhiteSpace, hasWhiteSpaces = false;
880 this.startPosition = this.currentPosition;
881 boolean checkIfUnicode = false;
883 checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
884 && (this.source[this.currentPosition] == 'u');
885 } catch(IndexOutOfBoundsException e) {
886 if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
887 // reposition scanner in case we are interested by spaces as tokens
888 this.currentPosition--;
889 this.startPosition = whiteStart;
890 return TokenNameWHITESPACE;
892 if (this.currentPosition > this.eofPosition)
895 if (checkIfUnicode) {
896 isWhiteSpace = jumpOverUnicodeWhiteSpace();
900 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
901 checkNonExternalizedString();
902 if (this.recordLineSeparator) {
905 this.currentLine = null;
909 (this.currentCharacter == ' ') || CharOperation.isWhitespace(this.currentCharacter);
912 hasWhiteSpaces = true;
914 } while (isWhiteSpace);
915 if (this.tokenizeWhiteSpace && hasWhiteSpaces) {
916 // reposition scanner in case we are interested by spaces as tokens
917 this.currentPosition-=offset;
918 this.startPosition = whiteStart;
919 return TokenNameWHITESPACE;
921 //little trick to get out in the middle of a source compuation
922 if (this.currentPosition > this.eofPosition)
925 // ---------Identify the next token-------------
927 switch (this.currentCharacter) {
929 return TokenNameLPAREN;
931 return TokenNameRPAREN;
933 return TokenNameLBRACE;
935 return TokenNameRBRACE;
937 return TokenNameLBRACKET;
939 return TokenNameRBRACKET;
941 return TokenNameSEMICOLON;
943 return TokenNameCOMMA;
945 if (getNextCharAsDigit())
946 return scanNumber(true);
951 if ((test = getNextChar('+', '=')) == 0)
952 return TokenNamePLUS_PLUS;
954 return TokenNamePLUS_EQUAL;
955 return TokenNamePLUS;
960 if ((test = getNextChar('-', '=')) == 0)
961 return TokenNameMINUS_MINUS;
963 return TokenNameMINUS_EQUAL;
964 return TokenNameMINUS;
967 return TokenNameTWIDDLE;
969 if (getNextChar('='))
970 return TokenNameNOT_EQUAL;
973 if (getNextChar('='))
974 return TokenNameMULTIPLY_EQUAL;
975 return TokenNameMULTIPLY;
977 if (getNextChar('='))
978 return TokenNameREMAINDER_EQUAL;
979 return TokenNameREMAINDER;
983 if ((test = getNextChar('=', '<')) == 0)
984 return TokenNameLESS_EQUAL;
986 if (getNextChar('='))
987 return TokenNameLEFT_SHIFT_EQUAL;
988 return TokenNameLEFT_SHIFT;
990 return TokenNameLESS;
995 if ((test = getNextChar('=', '>')) == 0)
996 return TokenNameGREATER_EQUAL;
998 if ((test = getNextChar('=', '>')) == 0)
999 return TokenNameRIGHT_SHIFT_EQUAL;
1001 if (getNextChar('='))
1002 return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
1003 return TokenNameUNSIGNED_RIGHT_SHIFT;
1005 return TokenNameRIGHT_SHIFT;
1007 return TokenNameGREATER;
1010 if (getNextChar('='))
1011 return TokenNameEQUAL_EQUAL;
1012 return TokenNameEQUAL;
1016 if ((test = getNextChar('&', '=')) == 0)
1017 return TokenNameAND_AND;
1019 return TokenNameAND_EQUAL;
1020 return TokenNameAND;
1025 if ((test = getNextChar('|', '=')) == 0)
1026 return TokenNameOR_OR;
1028 return TokenNameOR_EQUAL;
1032 if (getNextChar('='))
1033 return TokenNameXOR_EQUAL;
1034 return TokenNameXOR;
1036 return TokenNameQUESTION;
1038 return TokenNameCOLON;
1042 if ((test = getNextChar('\n', '\r')) == 0) {
1043 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1046 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1047 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1048 if (this.currentPosition + lookAhead == this.source.length)
1050 if (this.source[this.currentPosition + lookAhead] == '\n')
1052 if (this.source[this.currentPosition + lookAhead] == '\'') {
1053 this.currentPosition += lookAhead + 1;
1057 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1060 if (getNextChar('\'')) {
1061 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1062 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1063 if (this.currentPosition + lookAhead == this.source.length)
1065 if (this.source[this.currentPosition + lookAhead] == '\n')
1067 if (this.source[this.currentPosition + lookAhead] == '\'') {
1068 this.currentPosition += lookAhead + 1;
1072 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1074 if (getNextChar('\\'))
1075 scanEscapeCharacter();
1076 else { // consume next character
1077 this.unicodeAsBackSlash = false;
1078 boolean checkIfUnicode = false;
1080 checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1081 && (this.source[this.currentPosition] == 'u');
1082 } catch(IndexOutOfBoundsException e) {
1083 this.currentPosition--;
1084 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1086 if (checkIfUnicode) {
1087 getNextUnicodeChar();
1089 if (this.withoutUnicodePtr != 0) {
1090 unicodeStoreAt(++this.withoutUnicodePtr);
1094 if (getNextChar('\''))
1095 return TokenNameCharacterLiteral;
1096 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1097 for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
1098 if (this.currentPosition + lookAhead == this.source.length)
1100 if (this.source[this.currentPosition + lookAhead] == '\n')
1102 if (this.source[this.currentPosition + lookAhead] == '\'') {
1103 this.currentPosition += lookAhead + 1;
1107 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1110 // consume next character
1111 this.unicodeAsBackSlash = false;
1112 boolean isUnicode = false;
1113 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1114 && (this.source[this.currentPosition] == 'u')) {
1115 getNextUnicodeChar();
1118 if (this.withoutUnicodePtr != 0) {
1119 unicodeStoreAt(++this.withoutUnicodePtr);
1123 while (this.currentCharacter != '"') {
1124 /**** \r and \n are not valid in string literals ****/
1125 if ((this.currentCharacter == '\n') || (this.currentCharacter == '\r')) {
1126 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1128 int start = this.currentPosition;
1129 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1130 if (this.currentPosition >= this.eofPosition) {
1131 this.currentPosition = start;
1134 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
1136 getNextUnicodeChar();
1140 if (!isUnicode && this.currentCharacter == '\n') {
1141 this.currentPosition--; // set current position on new line character
1144 if (this.currentCharacter == '\"') {
1145 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1149 this.currentPosition--; // set current position on new line character
1151 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1153 if (this.currentCharacter == '\\') {
1154 int escapeSize = this.currentPosition;
1155 boolean backSlashAsUnicodeInString = this.unicodeAsBackSlash;
1156 //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
1157 scanEscapeCharacter();
1158 escapeSize = this.currentPosition - escapeSize;
1159 if (this.withoutUnicodePtr == 0) {
1160 //buffer all the entries that have been left aside....
1161 unicodeInitializeBuffer(this.currentPosition - escapeSize - 1 - this.startPosition);
1162 unicodeStoreAt(++this.withoutUnicodePtr);
1163 } else { //overwrite the / in the buffer
1164 unicodeStoreAt(this.withoutUnicodePtr);
1165 if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
1166 this.withoutUnicodePtr--;
1170 // consume next character
1171 this.unicodeAsBackSlash = false;
1172 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1173 && (this.source[this.currentPosition] == 'u')) {
1174 getNextUnicodeChar();
1176 if (this.withoutUnicodePtr != 0) {
1177 unicodeStoreAt(++this.withoutUnicodePtr);
1182 } catch (IndexOutOfBoundsException e) {
1183 this.currentPosition--;
1184 throw new InvalidInputException(UNTERMINATED_STRING);
1185 } catch (InvalidInputException e) {
1186 if (e.getMessage().equals(INVALID_ESCAPE)) {
1187 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1188 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1189 if (this.currentPosition + lookAhead == this.source.length)
1191 if (this.source[this.currentPosition + lookAhead] == '\n')
1193 if (this.source[this.currentPosition + lookAhead] == '\"') {
1194 this.currentPosition += lookAhead + 1;
1202 if (this.checkNonExternalizedStringLiterals){ // check for presence of NLS tags //$NON-NLS-?$ where ? is an int.
1203 if (this.currentLine == null) {
1204 this.currentLine = new NLSLine();
1206 this.currentLine.add(
1208 getCurrentTokenSourceString(),
1210 this.currentPosition - 1));
1212 return TokenNameStringLiteral;
1216 if ((test = getNextChar('/', '*')) == 0) { //line comment
1217 this.lastCommentLinePosition = this.currentPosition;
1218 try { //get the next char
1219 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1220 && (this.source[this.currentPosition] == 'u')) {
1221 //-------------unicode traitement ------------
1222 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1223 this.currentPosition++;
1224 while (this.source[this.currentPosition] == 'u') {
1225 this.currentPosition++;
1227 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1229 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1231 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1233 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1235 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1237 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1241 //handle the \\u case manually into comment
1242 if (this.currentCharacter == '\\') {
1243 if (this.source[this.currentPosition] == '\\')
1244 this.currentPosition++;
1245 } //jump over the \\
1246 boolean isUnicode = false;
1247 while (this.currentCharacter != '\r' && this.currentCharacter != '\n') {
1248 this.lastCommentLinePosition = this.currentPosition;
1251 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1252 && (this.source[this.currentPosition] == 'u')) {
1254 //-------------unicode traitement ------------
1255 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1256 this.currentPosition++;
1257 while (this.source[this.currentPosition] == 'u') {
1258 this.currentPosition++;
1260 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1262 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1264 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1266 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1268 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1270 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1273 //handle the \\u case manually into comment
1274 if (this.currentCharacter == '\\') {
1275 if (this.source[this.currentPosition] == '\\')
1276 this.currentPosition++;
1277 } //jump over the \\
1280 * We need to completely consume the line break
1282 if (this.currentCharacter == '\r'
1283 && this.source.length > this.currentPosition) {
1284 if (this.source[this.currentPosition] == '\n') {
1285 this.currentPosition++;
1286 this.currentCharacter = '\n';
1287 } else if ((this.source[this.currentPosition] == '\\')
1288 && (this.source[this.currentPosition + 1] == 'u')) {
1291 int index = this.currentPosition + 1;
1293 while (this.source[index] == 'u') {
1296 //-------------unicode traitement ------------
1297 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1298 if ((c1 = Character.getNumericValue(this.source[index++])) > 15
1300 || (c2 = Character.getNumericValue(this.source[index++])) > 15
1302 || (c3 = Character.getNumericValue(this.source[index++])) > 15
1304 || (c4 = Character.getNumericValue(this.source[index++])) > 15
1306 this.currentPosition = index;
1307 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1309 unicodeChar = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1311 if (unicodeChar == '\n') {
1312 this.currentPosition = index;
1313 this.currentCharacter = '\n';
1317 recordComment(TokenNameCOMMENT_LINE);
1318 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1319 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1320 checkNonExternalizedString();
1321 if (this.recordLineSeparator) {
1323 pushUnicodeLineSeparator();
1325 pushLineSeparator();
1328 this.currentLine = null;
1331 if (this.tokenizeComments) {
1332 return TokenNameCOMMENT_LINE;
1334 } catch (IndexOutOfBoundsException e) {
1335 this.currentPosition--;
1336 recordComment(TokenNameCOMMENT_LINE);
1337 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1338 if (this.tokenizeComments) {
1339 return TokenNameCOMMENT_LINE;
1341 this.currentPosition++;
1346 if (test > 0) { //traditional and javadoc comment
1347 try { //get the next char
1348 boolean isJavadoc = false, star = false;
1349 boolean isUnicode = false;
1350 // consume next character
1351 this.unicodeAsBackSlash = false;
1352 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1353 && (this.source[this.currentPosition] == 'u')) {
1354 getNextUnicodeChar();
1358 if (this.withoutUnicodePtr != 0) {
1359 unicodeStoreAt(++this.withoutUnicodePtr);
1363 if (this.currentCharacter == '*') {
1367 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1368 checkNonExternalizedString();
1369 if (this.recordLineSeparator) {
1371 pushLineSeparator();
1374 this.currentLine = null;
1378 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1379 && (this.source[this.currentPosition] == 'u')) {
1380 //-------------unicode traitement ------------
1381 getNextUnicodeChar();
1386 //handle the \\u case manually into comment
1387 if (this.currentCharacter == '\\') {
1388 if (this.source[this.currentPosition] == '\\')
1389 this.currentPosition++; //jump over the \\
1391 // empty comment is not a javadoc /**/
1392 if (this.currentCharacter == '/') {
1395 //loop until end of comment */
1396 while ((this.currentCharacter != '/') || (!star)) {
1397 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1398 checkNonExternalizedString();
1399 if (this.recordLineSeparator) {
1401 pushLineSeparator();
1404 this.currentLine = null;
1407 star = this.currentCharacter == '*';
1409 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1410 && (this.source[this.currentPosition] == 'u')) {
1411 //-------------unicode traitement ------------
1412 getNextUnicodeChar();
1417 //handle the \\u case manually into comment
1418 if (this.currentCharacter == '\\') {
1419 if (this.source[this.currentPosition] == '\\')
1420 this.currentPosition++;
1421 } //jump over the \\
1423 int token = isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK;
1424 recordComment(token);
1425 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1426 if (this.tokenizeComments) {
1429 return TokenNameCOMMENT_JAVADOC;
1430 return TokenNameCOMMENT_BLOCK;
1434 } catch (IndexOutOfBoundsException e) {
1435 this.currentPosition--;
1436 throw new InvalidInputException(UNTERMINATED_COMMENT);
1440 if (getNextChar('='))
1441 return TokenNameDIVIDE_EQUAL;
1442 return TokenNameDIVIDE;
1446 return TokenNameEOF;
1447 //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
1448 throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1451 if (Character.isJavaIdentifierStart(this.currentCharacter))
1452 return scanIdentifierOrKeyword();
1453 if (isDigit(this.currentCharacter))
1454 return scanNumber(false);
1455 return TokenNameERROR;
1458 } //-----------------end switch while try--------------------
1459 catch (IndexOutOfBoundsException e) {
1460 if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
1461 // reposition scanner in case we are interested by spaces as tokens
1462 this.currentPosition--;
1463 this.startPosition = whiteStart;
1464 return TokenNameWHITESPACE;
1467 return TokenNameEOF;
1469 public final void getNextUnicodeChar()
1470 throws InvalidInputException {
1472 //handle the case of unicode.
1473 //when a unicode appears then we must use a buffer that holds char internal values
1474 //At the end of this method currentCharacter holds the new visited char
1475 //and currentPosition points right next after it
1477 //ALL getNextChar.... ARE OPTIMIZED COPIES
1480 int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
1481 this.currentPosition++;
1482 while (this.source[this.currentPosition] == 'u') {
1483 this.currentPosition++;
1487 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1489 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1491 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1493 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1495 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1497 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1498 //need the unicode buffer
1499 if (this.withoutUnicodePtr == 0) {
1500 //buffer all the entries that have been left aside....
1501 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
1503 //fill the buffer with the char
1504 unicodeStoreAt(++this.withoutUnicodePtr);
1506 this.unicodeAsBackSlash = this.currentCharacter == '\\';
1507 } catch (ArrayIndexOutOfBoundsException e) {
1508 this.currentPosition--;
1509 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1513 public char[] getSource(){
1516 private boolean isDigit(char c) throws InvalidInputException {
1517 if (Character.isDigit(c)) {
1531 throw new InvalidInputException(Scanner.INVALID_INPUT);
1536 /* Tokenize a method body, assuming that curly brackets are properly balanced.
1538 public final void jumpOverMethodBody() {
1540 this.wasAcr = false;
1543 while (true) { //loop for jumping over comments
1544 // ---------Consume white space and handles startPosition---------
1545 boolean isWhiteSpace;
1547 this.startPosition = this.currentPosition;
1548 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1549 && (this.source[this.currentPosition] == 'u')) {
1550 isWhiteSpace = jumpOverUnicodeWhiteSpace();
1552 if (this.recordLineSeparator
1553 && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')))
1554 pushLineSeparator();
1555 isWhiteSpace = CharOperation.isWhitespace(this.currentCharacter);
1557 } while (isWhiteSpace);
1559 // -------consume token until } is found---------
1560 switch (this.currentCharacter) {
1572 test = getNextChar('\\');
1575 scanEscapeCharacter();
1576 } catch (InvalidInputException ex) {
1580 try { // consume next character
1581 this.unicodeAsBackSlash = false;
1582 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1583 && (this.source[this.currentPosition] == 'u')) {
1584 getNextUnicodeChar();
1586 if (this.withoutUnicodePtr != 0) {
1587 unicodeStoreAt(++this.withoutUnicodePtr);
1590 } catch (InvalidInputException ex) {
1599 try { // consume next character
1600 this.unicodeAsBackSlash = false;
1601 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1602 && (this.source[this.currentPosition] == 'u')) {
1603 getNextUnicodeChar();
1605 if (this.withoutUnicodePtr != 0) {
1606 unicodeStoreAt(++this.withoutUnicodePtr);
1609 } catch (InvalidInputException ex) {
1612 while (this.currentCharacter != '"') {
1613 if (this.currentCharacter == '\r'){
1614 if (this.source[this.currentPosition] == '\n') this.currentPosition++;
1615 break; // the string cannot go further that the line
1617 if (this.currentCharacter == '\n'){
1618 break; // the string cannot go further that the line
1620 if (this.currentCharacter == '\\') {
1622 scanEscapeCharacter();
1623 } catch (InvalidInputException ex) {
1627 try { // consume next character
1628 this.unicodeAsBackSlash = false;
1629 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1630 && (this.source[this.currentPosition] == 'u')) {
1631 getNextUnicodeChar();
1633 if (this.withoutUnicodePtr != 0) {
1634 unicodeStoreAt(++this.withoutUnicodePtr);
1637 } catch (InvalidInputException ex) {
1641 } catch (IndexOutOfBoundsException e) {
1648 if ((test = getNextChar('/', '*')) == 0) { //line comment
1650 this.lastCommentLinePosition = this.currentPosition;
1652 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1653 && (this.source[this.currentPosition] == 'u')) {
1654 //-------------unicode traitement ------------
1655 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1656 this.currentPosition++;
1657 while (this.source[this.currentPosition] == 'u') {
1658 this.currentPosition++;
1660 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1662 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1664 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1666 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1667 || c4 < 0) { //error don't care of the value
1668 this.currentCharacter = 'A';
1669 } //something different from \n and \r
1671 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1674 //handle the \\u case manually into comment
1675 if (this.currentCharacter == '\\') {
1676 if (this.source[this.currentPosition] == '\\')
1677 this.currentPosition++;
1678 } //jump over the \\
1679 boolean isUnicode = false;
1680 while (this.currentCharacter != '\r' && this.currentCharacter != '\n') {
1681 this.lastCommentLinePosition = this.currentPosition;
1684 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1685 && (this.source[this.currentPosition] == 'u')) {
1687 //-------------unicode traitement ------------
1688 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1689 this.currentPosition++;
1690 while (this.source[this.currentPosition] == 'u') {
1691 this.currentPosition++;
1693 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1695 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1697 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1699 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1700 || c4 < 0) { //error don't care of the value
1701 this.currentCharacter = 'A';
1702 } //something different from \n and \r
1704 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1707 //handle the \\u case manually into comment
1708 if (this.currentCharacter == '\\') {
1709 if (this.source[this.currentPosition] == '\\')
1710 this.currentPosition++;
1711 } //jump over the \\
1714 * We need to completely consume the line break
1716 if (this.currentCharacter == '\r'
1717 && this.source.length > this.currentPosition) {
1718 if (this.source[this.currentPosition] == '\n') {
1719 this.currentPosition++;
1720 this.currentCharacter = '\n';
1721 } else if ((this.source[this.currentPosition] == '\\')
1722 && (this.source[this.currentPosition + 1] == 'u')) {
1725 int index = this.currentPosition + 1;
1727 while (this.source[index] == 'u') {
1730 //-------------unicode traitement ------------
1731 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1732 if ((c1 = Character.getNumericValue(this.source[index++])) > 15
1734 || (c2 = Character.getNumericValue(this.source[index++])) > 15
1736 || (c3 = Character.getNumericValue(this.source[index++])) > 15
1738 || (c4 = Character.getNumericValue(this.source[index++])) > 15
1739 || c4 < 0) { //error don't care of the value
1742 unicodeChar = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1744 if (unicodeChar == '\n') {
1745 this.currentPosition = index;
1746 this.currentCharacter = '\n';
1750 recordComment(TokenNameCOMMENT_LINE);
1751 if (this.recordLineSeparator
1752 && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n'))) {
1754 pushUnicodeLineSeparator();
1756 pushLineSeparator();
1759 } catch (IndexOutOfBoundsException e) {
1760 //an eof will then be generated
1761 this.currentPosition--;
1762 recordComment(TokenNameCOMMENT_LINE);
1763 if (!this.tokenizeComments) {
1764 this.currentPosition++;
1769 if (test > 0) { //traditional and javadoc comment
1770 boolean isJavadoc = false;
1771 try { //get the next char
1772 boolean star = false;
1773 boolean isUnicode = false;
1774 // consume next character
1775 this.unicodeAsBackSlash = false;
1776 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1777 && (this.source[this.currentPosition] == 'u')) {
1778 getNextUnicodeChar();
1782 if (this.withoutUnicodePtr != 0) {
1783 unicodeStoreAt(++this.withoutUnicodePtr);
1787 if (this.currentCharacter == '*') {
1791 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1792 if (this.recordLineSeparator) {
1794 pushUnicodeLineSeparator();
1796 pushLineSeparator();
1799 this.currentLine = null;
1803 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1804 && (this.source[this.currentPosition] == 'u')) {
1805 //-------------unicode traitement ------------
1806 getNextUnicodeChar();
1811 //handle the \\u case manually into comment
1812 if (this.currentCharacter == '\\') {
1813 if (this.source[this.currentPosition] == '\\')
1814 this.currentPosition++; //jump over the \\
1816 // empty comment is not a javadoc /**/
1817 if (this.currentCharacter == '/') {
1820 //loop until end of comment */
1821 while ((this.currentCharacter != '/') || (!star)) {
1822 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1823 if (this.recordLineSeparator) {
1825 pushUnicodeLineSeparator();
1827 pushLineSeparator();
1830 this.currentLine = null;
1833 star = this.currentCharacter == '*';
1835 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1836 && (this.source[this.currentPosition] == 'u')) {
1837 //-------------unicode traitement ------------
1838 getNextUnicodeChar();
1843 //handle the \\u case manually into comment
1844 if (this.currentCharacter == '\\') {
1845 if (this.source[this.currentPosition] == '\\')
1846 this.currentPosition++;
1847 } //jump over the \\
1849 recordComment(isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK);
1850 } catch (IndexOutOfBoundsException e) {
1859 if (Character.isJavaIdentifierStart(this.currentCharacter)) {
1860 scanIdentifierOrKeyword();
1863 if (isDigit(this.currentCharacter)) {
1866 } catch (InvalidInputException ex) {
1873 //-----------------end switch while try--------------------
1874 } catch (IndexOutOfBoundsException e) {
1876 } catch (InvalidInputException e) {
1881 public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException {
1883 //handle the case of unicode. Jump over the next whiteSpace
1884 //making startPosition pointing on the next available char
1885 //On false, the currentCharacter is filled up with a potential
1889 this.wasAcr = false;
1891 int unicodeSize = 6;
1892 this.currentPosition++;
1893 while (this.source[this.currentPosition] == 'u') {
1894 this.currentPosition++;
1898 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1900 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
1901 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
1902 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
1903 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1906 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1907 if (CharOperation.isWhitespace(this.currentCharacter))
1910 //buffer the new char which is not a white space
1911 unicodeStoreAt(++this.withoutUnicodePtr);
1912 //this.withoutUnicodePtr == 1 is true here
1914 } catch (IndexOutOfBoundsException e){
1915 this.currentPosition--;
1916 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1920 final char[] optimizedCurrentTokenSource1() {
1921 //return always the same char[] build only once
1923 //optimization at no speed cost of 99.5 % of the singleCharIdentifier
1924 char charOne = this.source[this.startPosition];
1979 return new char[] {charOne};
1982 final char[] optimizedCurrentTokenSource2() {
1983 //try to return the same char[] build only once
1987 (((c0 = this.source[this.startPosition]) << 6) + (c1 = this.source[this.startPosition + 1]))
1989 char[][] table = this.charArray_length[0][hash];
1991 while (++i < InternalTableSize) {
1992 char[] charArray = table[i];
1993 if ((c0 == charArray[0]) && (c1 == charArray[1]))
1996 //---------other side---------
1998 int max = newEntry2;
1999 while (++i <= max) {
2000 char[] charArray = table[i];
2001 if ((c0 == charArray[0]) && (c1 == charArray[1]))
2004 //--------add the entry-------
2005 if (++max >= InternalTableSize) max = 0;
2007 table[max] = (r = new char[] {c0, c1});
2011 final char[] optimizedCurrentTokenSource3() {
2012 //try to return the same char[] build only once
2016 (((c0 = this.source[this.startPosition]) << 12)
2017 + ((c1 = this.source[this.startPosition + 1]) << 6)
2018 + (c2 = this.source[this.startPosition + 2]))
2020 char[][] table = this.charArray_length[1][hash];
2022 while (++i < InternalTableSize) {
2023 char[] charArray = table[i];
2024 if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2027 //---------other side---------
2029 int max = newEntry3;
2030 while (++i <= max) {
2031 char[] charArray = table[i];
2032 if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2035 //--------add the entry-------
2036 if (++max >= InternalTableSize) max = 0;
2038 table[max] = (r = new char[] {c0, c1, c2});
2042 final char[] optimizedCurrentTokenSource4() {
2043 //try to return the same char[] build only once
2045 char c0, c1, c2, c3;
2047 ((((long) (c0 = this.source[this.startPosition])) << 18)
2048 + ((c1 = this.source[this.startPosition + 1]) << 12)
2049 + ((c2 = this.source[this.startPosition + 2]) << 6)
2050 + (c3 = this.source[this.startPosition + 3]))
2052 char[][] table = this.charArray_length[2][(int) hash];
2054 while (++i < InternalTableSize) {
2055 char[] charArray = table[i];
2056 if ((c0 == charArray[0])
2057 && (c1 == charArray[1])
2058 && (c2 == charArray[2])
2059 && (c3 == charArray[3]))
2062 //---------other side---------
2064 int max = newEntry4;
2065 while (++i <= max) {
2066 char[] charArray = table[i];
2067 if ((c0 == charArray[0])
2068 && (c1 == charArray[1])
2069 && (c2 == charArray[2])
2070 && (c3 == charArray[3]))
2073 //--------add the entry-------
2074 if (++max >= InternalTableSize) max = 0;
2076 table[max] = (r = new char[] {c0, c1, c2, c3});
2081 final char[] optimizedCurrentTokenSource5() {
2082 //try to return the same char[] build only once
2084 char c0, c1, c2, c3, c4;
2086 ((((long) (c0 = this.source[this.startPosition])) << 24)
2087 + (((long) (c1 = this.source[this.startPosition + 1])) << 18)
2088 + ((c2 = this.source[this.startPosition + 2]) << 12)
2089 + ((c3 = this.source[this.startPosition + 3]) << 6)
2090 + (c4 = this.source[this.startPosition + 4]))
2092 char[][] table = this.charArray_length[3][(int) hash];
2094 while (++i < InternalTableSize) {
2095 char[] charArray = table[i];
2096 if ((c0 == charArray[0])
2097 && (c1 == charArray[1])
2098 && (c2 == charArray[2])
2099 && (c3 == charArray[3])
2100 && (c4 == charArray[4]))
2103 //---------other side---------
2105 int max = newEntry5;
2106 while (++i <= max) {
2107 char[] charArray = table[i];
2108 if ((c0 == charArray[0])
2109 && (c1 == charArray[1])
2110 && (c2 == charArray[2])
2111 && (c3 == charArray[3])
2112 && (c4 == charArray[4]))
2115 //--------add the entry-------
2116 if (++max >= InternalTableSize) max = 0;
2118 table[max] = (r = new char[] {c0, c1, c2, c3, c4});
2123 final char[] optimizedCurrentTokenSource6() {
2124 //try to return the same char[] build only once
2126 char c0, c1, c2, c3, c4, c5;
2128 ((((long) (c0 = this.source[this.startPosition])) << 32)
2129 + (((long) (c1 = this.source[this.startPosition + 1])) << 24)
2130 + (((long) (c2 = this.source[this.startPosition + 2])) << 18)
2131 + ((c3 = this.source[this.startPosition + 3]) << 12)
2132 + ((c4 = this.source[this.startPosition + 4]) << 6)
2133 + (c5 = this.source[this.startPosition + 5]))
2135 char[][] table = this.charArray_length[4][(int) hash];
2137 while (++i < InternalTableSize) {
2138 char[] charArray = table[i];
2139 if ((c0 == charArray[0])
2140 && (c1 == charArray[1])
2141 && (c2 == charArray[2])
2142 && (c3 == charArray[3])
2143 && (c4 == charArray[4])
2144 && (c5 == charArray[5]))
2147 //---------other side---------
2149 int max = newEntry6;
2150 while (++i <= max) {
2151 char[] charArray = table[i];
2152 if ((c0 == charArray[0])
2153 && (c1 == charArray[1])
2154 && (c2 == charArray[2])
2155 && (c3 == charArray[3])
2156 && (c4 == charArray[4])
2157 && (c5 == charArray[5]))
2160 //--------add the entry-------
2161 if (++max >= InternalTableSize) max = 0;
2163 table[max] = (r = new char[] {c0, c1, c2, c3, c4, c5});
2167 private void parseTags(NLSLine line) {
2168 String s = new String(getCurrentTokenSource());
2169 int pos = s.indexOf(TAG_PREFIX);
2170 int lineLength = line.size();
2172 int start = pos + TAG_PREFIX_LENGTH;
2173 int end = s.indexOf(TAG_POSTFIX, start);
2175 String index = s.substring(start, end);
2178 i = Integer.parseInt(index) - 1; // Tags are one based not zero based.
2179 } catch (NumberFormatException e) {
2180 i = -1; // we don't want to consider this as a valid NLS tag
2182 if (line.exists(i)) {
2186 pos = s.indexOf(TAG_PREFIX, start);
2189 this.nonNLSStrings = new StringLiteral[lineLength];
2190 int nonNLSCounter = 0;
2191 for (Iterator iterator = line.iterator(); iterator.hasNext(); ) {
2192 StringLiteral literal = (StringLiteral) iterator.next();
2193 if (literal != null) {
2194 this.nonNLSStrings[nonNLSCounter++] = literal;
2197 if (nonNLSCounter == 0) {
2198 this.nonNLSStrings = null;
2199 this.currentLine = null;
2202 this.wasNonExternalizedStringLiteral = true;
2203 if (nonNLSCounter != lineLength) {
2204 System.arraycopy(this.nonNLSStrings, 0, (this.nonNLSStrings = new StringLiteral[nonNLSCounter]), 0, nonNLSCounter);
2206 this.currentLine = null;
2209 public final void pushLineSeparator() {
2210 //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2211 final int INCREMENT = 250;
2213 if (this.checkNonExternalizedStringLiterals) {
2214 // reinitialize the current line for non externalize strings purpose
2215 this.currentLine = null;
2217 //currentCharacter is at position currentPosition-1
2220 if (this.currentCharacter == '\r') {
2221 int separatorPos = this.currentPosition - 1;
2222 //TODO (olivier) david - why the following line was "if ((this.linePtr > 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;" ?
2223 if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;
2224 //System.out.println("CR-" + separatorPos);
2225 int length = this.lineEnds.length;
2226 if (++this.linePtr >= length)
2227 System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[length + INCREMENT], 0, length);
2228 this.lineEnds[this.linePtr] = separatorPos;
2229 // look-ahead for merged cr+lf
2231 if (this.source[this.currentPosition] == '\n') {
2232 //System.out.println("look-ahead LF-" + this.currentPosition);
2233 this.lineEnds[this.linePtr] = this.currentPosition;
2234 this.currentPosition++;
2235 this.wasAcr = false;
2239 } catch(IndexOutOfBoundsException e) {
2244 if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf
2245 if (this.wasAcr && (this.lineEnds[this.linePtr] == (this.currentPosition - 2))) {
2246 //System.out.println("merge LF-" + (this.currentPosition - 1));
2247 this.lineEnds[this.linePtr] = this.currentPosition - 1;
2249 int separatorPos = this.currentPosition - 1;
2250 //TODO (olivier) david - why the following line was "if ((this.linePtr > 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;" ?
2251 if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;
2252 // System.out.println("LF-" + separatorPos);
2253 int length = this.lineEnds.length;
2254 if (++this.linePtr >= length)
2255 System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[length + INCREMENT], 0, length);
2256 this.lineEnds[this.linePtr] = separatorPos;
2258 this.wasAcr = false;
2262 public final void pushUnicodeLineSeparator() {
2263 if (this.checkNonExternalizedStringLiterals) {
2264 // reinitialize the current line for non externalize strings purpose
2265 this.currentLine = null;
2269 if (this.currentCharacter == '\r') {
2270 if (this.source[this.currentPosition] == '\n') {
2271 this.wasAcr = false;
2277 if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf
2278 this.wasAcr = false;
2282 public void recordComment(int token) {
2284 int stopPosition = this.currentPosition;
2286 case TokenNameCOMMENT_LINE:
2287 stopPosition = -this.lastCommentLinePosition;
2289 case TokenNameCOMMENT_BLOCK:
2290 stopPosition = -this.currentPosition;
2294 // a new comment is recorded
2295 int length = this.commentStops.length;
2296 if (++this.commentPtr >= length) {
2297 System.arraycopy(this.commentStops, 0, this.commentStops = new int[length + 30], 0, length);
2298 //grows the positions buffers too
2299 System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[length + 30], 0, length);
2301 this.commentStops[this.commentPtr] = stopPosition;
2302 this.commentStarts[this.commentPtr] = this.startPosition;
2306 * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position.
2307 * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>).
2309 * @param begin the given start position
2310 * @param end the given end position
2312 public void resetTo(int begin, int end) {
2313 //reset the scanner to a given position where it may rescan again
2316 this.initialPosition = this.startPosition = this.currentPosition = begin;
2317 this.eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
2318 this.commentPtr = -1; // reset comment stack
2319 this.foundTaskCount = 0;
2321 // // if resetTo is used with being > than end.
2322 // if (begin > this.eofPosition) {
2323 // begin = this.eofPosition;
2327 public final void scanEscapeCharacter() throws InvalidInputException {
2328 // the string with "\\u" is a legal string of two chars \ and u
2329 //thus we use a direct access to the source (for regular cases).
2331 if (this.unicodeAsBackSlash) {
2332 // consume next character
2333 this.unicodeAsBackSlash = false;
2334 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
2335 getNextUnicodeChar();
2337 if (this.withoutUnicodePtr != 0) {
2338 unicodeStoreAt(++this.withoutUnicodePtr);
2342 this.currentCharacter = this.source[this.currentPosition++];
2343 switch (this.currentCharacter) {
2345 this.currentCharacter = '\b';
2348 this.currentCharacter = '\t';
2351 this.currentCharacter = '\n';
2354 this.currentCharacter = '\f';
2357 this.currentCharacter = '\r';
2360 this.currentCharacter = '\"';
2363 this.currentCharacter = '\'';
2366 this.currentCharacter = '\\';
2369 // -----------octal escape--------------
2371 // OctalDigit OctalDigit
2372 // ZeroToThree OctalDigit OctalDigit
2374 int number = Character.getNumericValue(this.currentCharacter);
2375 if (number >= 0 && number <= 7) {
2376 boolean zeroToThreeNot = number > 3;
2377 if (isDigit(this.currentCharacter = this.source[this.currentPosition++])) {
2378 int digit = Character.getNumericValue(this.currentCharacter);
2379 if (digit >= 0 && digit <= 7) {
2380 number = (number * 8) + digit;
2381 if (isDigit(this.currentCharacter = this.source[this.currentPosition++])) {
2382 if (zeroToThreeNot) {// has read \NotZeroToThree OctalDigit Digit --> ignore last character
2383 this.currentPosition--;
2385 digit = Character.getNumericValue(this.currentCharacter);
2386 if (digit >= 0 && digit <= 7){ // has read \ZeroToThree OctalDigit OctalDigit
2387 number = (number * 8) + digit;
2388 } else {// has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
2389 this.currentPosition--;
2392 } else { // has read \OctalDigit NonDigit--> ignore last character
2393 this.currentPosition--;
2395 } else { // has read \OctalDigit NonOctalDigit--> ignore last character
2396 this.currentPosition--;
2398 } else { // has read \OctalDigit --> ignore last character
2399 this.currentPosition--;
2402 throw new InvalidInputException(INVALID_ESCAPE);
2403 this.currentCharacter = (char) number;
2405 throw new InvalidInputException(INVALID_ESCAPE);
2408 public int scanIdentifierOrKeyword() {
2411 //first dispatch on the first char.
2412 //then the length. If there are several
2413 //keywors with the same length AND the same first char, then do another
2414 //dispatch on the second char
2415 this.useAssertAsAnIndentifier = false;
2416 while (getNextCharAsJavaIdentifierPart()){/*empty*/}
2421 if (this.withoutUnicodePtr == 0)
2423 //quick test on length == 1 but not on length > 12 while most identifier
2424 //have a length which is <= 12...but there are lots of identifier with
2428 if ((length = this.currentPosition - this.startPosition) == 1)
2429 return TokenNameIdentifier;
2431 index = this.startPosition;
2433 if ((length = this.withoutUnicodePtr) == 1)
2434 return TokenNameIdentifier;
2435 data = this.withoutUnicodeBuffer;
2439 firstLetter = data[index];
2440 switch (firstLetter) {
2445 if ((data[++index] == 'b')
2446 && (data[++index] == 's')
2447 && (data[++index] == 't')
2448 && (data[++index] == 'r')
2449 && (data[++index] == 'a')
2450 && (data[++index] == 'c')
2451 && (data[++index] == 't')) {
2452 return TokenNameabstract;
2454 return TokenNameIdentifier;
2457 if ((data[++index] == 's')
2458 && (data[++index] == 's')
2459 && (data[++index] == 'e')
2460 && (data[++index] == 'r')
2461 && (data[++index] == 't')) {
2462 if (this.assertMode) {
2463 this.containsAssertKeyword = true;
2464 return TokenNameassert;
2466 this.useAssertAsAnIndentifier = true;
2467 return TokenNameIdentifier;
2470 return TokenNameIdentifier;
2473 return TokenNameIdentifier;
2475 case 'b' : //boolean break byte
2478 if ((data[++index] == 'y') && (data[++index] == 't') && (data[++index] == 'e'))
2479 return TokenNamebyte;
2481 return TokenNameIdentifier;
2483 if ((data[++index] == 'r')
2484 && (data[++index] == 'e')
2485 && (data[++index] == 'a')
2486 && (data[++index] == 'k'))
2487 return TokenNamebreak;
2489 return TokenNameIdentifier;
2491 if ((data[++index] == 'o')
2492 && (data[++index] == 'o')
2493 && (data[++index] == 'l')
2494 && (data[++index] == 'e')
2495 && (data[++index] == 'a')
2496 && (data[++index] == 'n'))
2497 return TokenNameboolean;
2499 return TokenNameIdentifier;
2501 return TokenNameIdentifier;
2504 case 'c' : //case char catch const class continue
2507 if (data[++index] == 'a')
2508 if ((data[++index] == 's') && (data[++index] == 'e'))
2509 return TokenNamecase;
2511 return TokenNameIdentifier;
2513 if ((data[index] == 'h') && (data[++index] == 'a') && (data[++index] == 'r'))
2514 return TokenNamechar;
2516 return TokenNameIdentifier;
2518 if (data[++index] == 'a')
2519 if ((data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
2520 return TokenNamecatch;
2522 return TokenNameIdentifier;
2524 if (data[index] == 'l')
2525 if ((data[++index] == 'a')
2526 && (data[++index] == 's')
2527 && (data[++index] == 's'))
2528 return TokenNameclass;
2530 return TokenNameIdentifier;
2531 else if ((data[index] == 'o')
2532 && (data[++index] == 'n')
2533 && (data[++index] == 's')
2534 && (data[++index] == 't'))
2535 return TokenNameERROR; //const is not used in java ???????
2537 return TokenNameIdentifier;
2539 if ((data[++index] == 'o')
2540 && (data[++index] == 'n')
2541 && (data[++index] == 't')
2542 && (data[++index] == 'i')
2543 && (data[++index] == 'n')
2544 && (data[++index] == 'u')
2545 && (data[++index] == 'e'))
2546 return TokenNamecontinue;
2548 return TokenNameIdentifier;
2550 return TokenNameIdentifier;
2553 case 'd' : //default do double
2556 if ((data[++index] == 'o'))
2559 return TokenNameIdentifier;
2561 if ((data[++index] == 'o')
2562 && (data[++index] == 'u')
2563 && (data[++index] == 'b')
2564 && (data[++index] == 'l')
2565 && (data[++index] == 'e'))
2566 return TokenNamedouble;
2568 return TokenNameIdentifier;
2570 if ((data[++index] == 'e')
2571 && (data[++index] == 'f')
2572 && (data[++index] == 'a')
2573 && (data[++index] == 'u')
2574 && (data[++index] == 'l')
2575 && (data[++index] == 't'))
2576 return TokenNamedefault;
2578 return TokenNameIdentifier;
2580 return TokenNameIdentifier;
2582 case 'e' : //else extends
2585 if ((data[++index] == 'l') && (data[++index] == 's') && (data[++index] == 'e'))
2586 return TokenNameelse;
2588 return TokenNameIdentifier;
2590 if ((data[++index] == 'x')
2591 && (data[++index] == 't')
2592 && (data[++index] == 'e')
2593 && (data[++index] == 'n')
2594 && (data[++index] == 'd')
2595 && (data[++index] == 's'))
2596 return TokenNameextends;
2598 return TokenNameIdentifier;
2600 return TokenNameIdentifier;
2603 case 'f' : //final finally float for false
2606 if ((data[++index] == 'o') && (data[++index] == 'r'))
2607 return TokenNamefor;
2609 return TokenNameIdentifier;
2611 if (data[++index] == 'i')
2612 if ((data[++index] == 'n')
2613 && (data[++index] == 'a')
2614 && (data[++index] == 'l')) {
2615 return TokenNamefinal;
2617 return TokenNameIdentifier;
2619 if (data[index] == 'l')
2620 if ((data[++index] == 'o')
2621 && (data[++index] == 'a')
2622 && (data[++index] == 't'))
2623 return TokenNamefloat;
2625 return TokenNameIdentifier;
2627 if ((data[index] == 'a')
2628 && (data[++index] == 'l')
2629 && (data[++index] == 's')
2630 && (data[++index] == 'e'))
2631 return TokenNamefalse;
2633 return TokenNameIdentifier;
2635 if ((data[++index] == 'i')
2636 && (data[++index] == 'n')
2637 && (data[++index] == 'a')
2638 && (data[++index] == 'l')
2639 && (data[++index] == 'l')
2640 && (data[++index] == 'y'))
2641 return TokenNamefinally;
2643 return TokenNameIdentifier;
2646 return TokenNameIdentifier;
2650 if ((data[++index] == 'o')
2651 && (data[++index] == 't')
2652 && (data[++index] == 'o')) {
2653 return TokenNameERROR;
2655 } //no goto in java are allowed, so why java removes this keyword ???
2656 return TokenNameIdentifier;
2658 case 'i' : //if implements import instanceof int interface
2661 if (data[++index] == 'f')
2664 return TokenNameIdentifier;
2666 if ((data[++index] == 'n') && (data[++index] == 't'))
2667 return TokenNameint;
2669 return TokenNameIdentifier;
2671 if ((data[++index] == 'm')
2672 && (data[++index] == 'p')
2673 && (data[++index] == 'o')
2674 && (data[++index] == 'r')
2675 && (data[++index] == 't'))
2676 return TokenNameimport;
2678 return TokenNameIdentifier;
2680 if ((data[++index] == 'n')
2681 && (data[++index] == 't')
2682 && (data[++index] == 'e')
2683 && (data[++index] == 'r')
2684 && (data[++index] == 'f')
2685 && (data[++index] == 'a')
2686 && (data[++index] == 'c')
2687 && (data[++index] == 'e'))
2688 return TokenNameinterface;
2690 return TokenNameIdentifier;
2692 if (data[++index] == 'm')
2693 if ((data[++index] == 'p')
2694 && (data[++index] == 'l')
2695 && (data[++index] == 'e')
2696 && (data[++index] == 'm')
2697 && (data[++index] == 'e')
2698 && (data[++index] == 'n')
2699 && (data[++index] == 't')
2700 && (data[++index] == 's'))
2701 return TokenNameimplements;
2703 return TokenNameIdentifier;
2705 if ((data[index] == 'n')
2706 && (data[++index] == 's')
2707 && (data[++index] == 't')
2708 && (data[++index] == 'a')
2709 && (data[++index] == 'n')
2710 && (data[++index] == 'c')
2711 && (data[++index] == 'e')
2712 && (data[++index] == 'o')
2713 && (data[++index] == 'f'))
2714 return TokenNameinstanceof;
2716 return TokenNameIdentifier;
2719 return TokenNameIdentifier;
2724 if ((data[++index] == 'o')
2725 && (data[++index] == 'n')
2726 && (data[++index] == 'g')) {
2727 return TokenNamelong;
2730 return TokenNameIdentifier;
2732 case 'n' : //native new null
2735 if ((data[++index] == 'e') && (data[++index] == 'w'))
2736 return TokenNamenew;
2738 return TokenNameIdentifier;
2740 if ((data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 'l'))
2741 return TokenNamenull;
2743 return TokenNameIdentifier;
2745 if ((data[++index] == 'a')
2746 && (data[++index] == 't')
2747 && (data[++index] == 'i')
2748 && (data[++index] == 'v')
2749 && (data[++index] == 'e')) {
2750 return TokenNamenative;
2752 return TokenNameIdentifier;
2754 return TokenNameIdentifier;
2757 case 'p' : //package private protected public
2760 if ((data[++index] == 'u')
2761 && (data[++index] == 'b')
2762 && (data[++index] == 'l')
2763 && (data[++index] == 'i')
2764 && (data[++index] == 'c')) {
2765 return TokenNamepublic;
2767 return TokenNameIdentifier;
2769 if (data[++index] == 'a')
2770 if ((data[++index] == 'c')
2771 && (data[++index] == 'k')
2772 && (data[++index] == 'a')
2773 && (data[++index] == 'g')
2774 && (data[++index] == 'e'))
2775 return TokenNamepackage;
2777 return TokenNameIdentifier;
2779 if ((data[index] == 'r')
2780 && (data[++index] == 'i')
2781 && (data[++index] == 'v')
2782 && (data[++index] == 'a')
2783 && (data[++index] == 't')
2784 && (data[++index] == 'e')) {
2785 return TokenNameprivate;
2787 return TokenNameIdentifier;
2789 if ((data[++index] == 'r')
2790 && (data[++index] == 'o')
2791 && (data[++index] == 't')
2792 && (data[++index] == 'e')
2793 && (data[++index] == 'c')
2794 && (data[++index] == 't')
2795 && (data[++index] == 'e')
2796 && (data[++index] == 'd')) {
2797 return TokenNameprotected;
2799 return TokenNameIdentifier;
2802 return TokenNameIdentifier;
2807 if ((data[++index] == 'e')
2808 && (data[++index] == 't')
2809 && (data[++index] == 'u')
2810 && (data[++index] == 'r')
2811 && (data[++index] == 'n')) {
2812 return TokenNamereturn;
2815 return TokenNameIdentifier;
2817 case 's' : //short static super switch synchronized strictfp
2820 if (data[++index] == 'h')
2821 if ((data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 't'))
2822 return TokenNameshort;
2824 return TokenNameIdentifier;
2826 if ((data[index] == 'u')
2827 && (data[++index] == 'p')
2828 && (data[++index] == 'e')
2829 && (data[++index] == 'r'))
2830 return TokenNamesuper;
2832 return TokenNameIdentifier;
2835 if (data[++index] == 't')
2836 if ((data[++index] == 'a')
2837 && (data[++index] == 't')
2838 && (data[++index] == 'i')
2839 && (data[++index] == 'c')) {
2840 return TokenNamestatic;
2842 return TokenNameIdentifier;
2844 if ((data[index] == 'w')
2845 && (data[++index] == 'i')
2846 && (data[++index] == 't')
2847 && (data[++index] == 'c')
2848 && (data[++index] == 'h'))
2849 return TokenNameswitch;
2851 return TokenNameIdentifier;
2853 if ((data[++index] == 't')
2854 && (data[++index] == 'r')
2855 && (data[++index] == 'i')
2856 && (data[++index] == 'c')
2857 && (data[++index] == 't')
2858 && (data[++index] == 'f')
2859 && (data[++index] == 'p'))
2860 return TokenNamestrictfp;
2862 return TokenNameIdentifier;
2864 if ((data[++index] == 'y')
2865 && (data[++index] == 'n')
2866 && (data[++index] == 'c')
2867 && (data[++index] == 'h')
2868 && (data[++index] == 'r')
2869 && (data[++index] == 'o')
2870 && (data[++index] == 'n')
2871 && (data[++index] == 'i')
2872 && (data[++index] == 'z')
2873 && (data[++index] == 'e')
2874 && (data[++index] == 'd')) {
2875 return TokenNamesynchronized;
2877 return TokenNameIdentifier;
2879 return TokenNameIdentifier;
2882 case 't' : //try throw throws transient this true
2885 if ((data[++index] == 'r') && (data[++index] == 'y'))
2886 return TokenNametry;
2888 return TokenNameIdentifier;
2890 if (data[++index] == 'h')
2891 if ((data[++index] == 'i') && (data[++index] == 's'))
2892 return TokenNamethis;
2894 return TokenNameIdentifier;
2896 if ((data[index] == 'r') && (data[++index] == 'u') && (data[++index] == 'e'))
2897 return TokenNametrue;
2899 return TokenNameIdentifier;
2901 if ((data[++index] == 'h')
2902 && (data[++index] == 'r')
2903 && (data[++index] == 'o')
2904 && (data[++index] == 'w'))
2905 return TokenNamethrow;
2907 return TokenNameIdentifier;
2909 if ((data[++index] == 'h')
2910 && (data[++index] == 'r')
2911 && (data[++index] == 'o')
2912 && (data[++index] == 'w')
2913 && (data[++index] == 's'))
2914 return TokenNamethrows;
2916 return TokenNameIdentifier;
2918 if ((data[++index] == 'r')
2919 && (data[++index] == 'a')
2920 && (data[++index] == 'n')
2921 && (data[++index] == 's')
2922 && (data[++index] == 'i')
2923 && (data[++index] == 'e')
2924 && (data[++index] == 'n')
2925 && (data[++index] == 't')) {
2926 return TokenNametransient;
2928 return TokenNameIdentifier;
2931 return TokenNameIdentifier;
2934 case 'v' : //void volatile
2937 if ((data[++index] == 'o') && (data[++index] == 'i') && (data[++index] == 'd'))
2938 return TokenNamevoid;
2940 return TokenNameIdentifier;
2942 if ((data[++index] == 'o')
2943 && (data[++index] == 'l')
2944 && (data[++index] == 'a')
2945 && (data[++index] == 't')
2946 && (data[++index] == 'i')
2947 && (data[++index] == 'l')
2948 && (data[++index] == 'e')) {
2949 return TokenNamevolatile;
2951 return TokenNameIdentifier;
2954 return TokenNameIdentifier;
2957 case 'w' : //while widefp
2960 if ((data[++index] == 'h')
2961 && (data[++index] == 'i')
2962 && (data[++index] == 'l')
2963 && (data[++index] == 'e'))
2964 return TokenNamewhile;
2966 return TokenNameIdentifier;
2967 //case 6:if ( (data[++index] =='i') && (data[++index]=='d') && (data[++index]=='e') && (data[++index]=='f')&& (data[++index]=='p'))
2968 //return TokenNamewidefp ;
2970 //return TokenNameIdentifier;
2972 return TokenNameIdentifier;
2976 return TokenNameIdentifier;
2979 public int scanNumber(boolean dotPrefix) throws InvalidInputException {
2981 //when entering this method the currentCharacter is the first
2982 //digit of the number. It may be preceeded by a '.' when
2985 boolean floating = dotPrefix;
2986 if ((!dotPrefix) && (this.currentCharacter == '0')) {
2987 if (getNextChar('x', 'X') >= 0) { //----------hexa-----------------
2988 //force the first char of the hexa number do exist...
2989 // consume next character
2990 this.unicodeAsBackSlash = false;
2991 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2992 && (this.source[this.currentPosition] == 'u')) {
2993 getNextUnicodeChar();
2995 if (this.withoutUnicodePtr != 0) {
2996 unicodeStoreAt(++this.withoutUnicodePtr);
2999 if (Character.digit(this.currentCharacter, 16) == -1)
3000 throw new InvalidInputException(INVALID_HEXA);
3002 while (getNextCharAsDigit(16)){/*empty*/}
3003 if (getNextChar('l', 'L') >= 0)
3004 return TokenNameLongLiteral;
3006 return TokenNameIntegerLiteral;
3009 //there is x or X in the number
3010 //potential octal ! ... some one may write 000099.0 ! thus 00100 < 00078.0 is true !!!!! crazy language
3011 if (getNextCharAsDigit()) { //-------------potential octal-----------------
3012 while (getNextCharAsDigit()){/*empty*/}
3014 if (getNextChar('l', 'L') >= 0) {
3015 return TokenNameLongLiteral;
3018 if (getNextChar('f', 'F') >= 0) {
3019 return TokenNameFloatingPointLiteral;
3022 if (getNextChar('d', 'D') >= 0) {
3023 return TokenNameDoubleLiteral;
3024 } else { //make the distinction between octal and float ....
3025 boolean isInteger = true;
3026 if (getNextChar('.')) {
3028 while (getNextCharAsDigit()){/*empty*/}
3030 if (getNextChar('e', 'E') >= 0) { // consume next character
3032 this.unicodeAsBackSlash = false;
3033 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3034 && (this.source[this.currentPosition] == 'u')) {
3035 getNextUnicodeChar();
3037 if (this.withoutUnicodePtr != 0) {
3038 unicodeStoreAt(++this.withoutUnicodePtr);
3042 if ((this.currentCharacter == '-')
3043 || (this.currentCharacter == '+')) { // consume next character
3044 this.unicodeAsBackSlash = false;
3045 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3046 && (this.source[this.currentPosition] == 'u')) {
3047 getNextUnicodeChar();
3049 if (this.withoutUnicodePtr != 0) {
3050 unicodeStoreAt(++this.withoutUnicodePtr);
3054 if (!isDigit(this.currentCharacter))
3055 throw new InvalidInputException(INVALID_FLOAT);
3056 while (getNextCharAsDigit()){/*empty*/}
3058 if (getNextChar('f', 'F') >= 0)
3059 return TokenNameFloatingPointLiteral;
3060 if (getNextChar('d', 'D') >= 0 || !isInteger)
3061 return TokenNameDoubleLiteral;
3062 return TokenNameIntegerLiteral;
3069 while (getNextCharAsDigit()){/*empty*/}
3071 if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3072 return TokenNameLongLiteral;
3074 if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty
3075 while (getNextCharAsDigit()){/*empty*/}
3079 //if floating is true both exponant and suffix may be optional
3081 if (getNextChar('e', 'E') >= 0) {
3083 // consume next character
3084 this.unicodeAsBackSlash = false;
3085 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3086 && (this.source[this.currentPosition] == 'u')) {
3087 getNextUnicodeChar();
3089 if (this.withoutUnicodePtr != 0) {
3090 unicodeStoreAt(++this.withoutUnicodePtr);
3094 if ((this.currentCharacter == '-')
3095 || (this.currentCharacter == '+')) { // consume next character
3096 this.unicodeAsBackSlash = false;
3097 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3098 && (this.source[this.currentPosition] == 'u')) {
3099 getNextUnicodeChar();
3101 if (this.withoutUnicodePtr != 0) {
3102 unicodeStoreAt(++this.withoutUnicodePtr);
3106 if (!isDigit(this.currentCharacter))
3107 throw new InvalidInputException(INVALID_FLOAT);
3108 while (getNextCharAsDigit()){/*empty*/}
3111 if (getNextChar('d', 'D') >= 0)
3112 return TokenNameDoubleLiteral;
3113 if (getNextChar('f', 'F') >= 0)
3114 return TokenNameFloatingPointLiteral;
3116 //the long flag has been tested before
3118 return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral;
3121 * Search the line number corresponding to a specific position
3122 * @param position int
3125 public final int getLineNumber(int position) {
3127 if (this.lineEnds == null)
3129 int length = this.linePtr+1;
3132 int g = 0, d = length - 1;
3136 if (position < this.lineEnds[m]) {
3138 } else if (position > this.lineEnds[m]) {
3144 if (position < this.lineEnds[m]) {
3149 public final void setSource(char[] sourceString){
3150 //the source-buffer is set to sourceString
3153 if (sourceString == null) {
3154 this.source = CharOperation.NO_CHAR;
3157 this.source = sourceString;
3158 sourceLength = sourceString.length;
3160 this.startPosition = -1;
3161 this.eofPosition = sourceLength;
3162 this.initialPosition = this.currentPosition = 0;
3163 this.containsAssertKeyword = false;
3166 public String toString() {
3167 if (this.startPosition == this.source.length)
3168 return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
3169 if (this.currentPosition > this.source.length)
3170 return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
3172 char front[] = new char[this.startPosition];
3173 System.arraycopy(this.source, 0, front, 0, this.startPosition);
3175 int middleLength = (this.currentPosition - 1) - this.startPosition + 1;
3177 if (middleLength > -1) {
3178 middle = new char[middleLength];
3186 middle = CharOperation.NO_CHAR;
3189 char end[] = new char[this.source.length - (this.currentPosition - 1)];
3192 (this.currentPosition - 1) + 1,
3195 this.source.length - (this.currentPosition - 1) - 1);
3197 return new String(front)
3198 + "\n===============================\nStarts here -->" //$NON-NLS-1$
3199 + new String(middle)
3200 + "<-- Ends here\n===============================\n" //$NON-NLS-1$
3203 public final String toStringAction(int act) {
3205 case TokenNameIdentifier :
3206 return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3207 case TokenNameabstract :
3208 return "abstract"; //$NON-NLS-1$
3209 case TokenNameboolean :
3210 return "boolean"; //$NON-NLS-1$
3211 case TokenNamebreak :
3212 return "break"; //$NON-NLS-1$
3213 case TokenNamebyte :
3214 return "byte"; //$NON-NLS-1$
3215 case TokenNamecase :
3216 return "case"; //$NON-NLS-1$
3217 case TokenNamecatch :
3218 return "catch"; //$NON-NLS-1$
3219 case TokenNamechar :
3220 return "char"; //$NON-NLS-1$
3221 case TokenNameclass :
3222 return "class"; //$NON-NLS-1$
3223 case TokenNamecontinue :
3224 return "continue"; //$NON-NLS-1$
3225 case TokenNamedefault :
3226 return "default"; //$NON-NLS-1$
3228 return "do"; //$NON-NLS-1$
3229 case TokenNamedouble :
3230 return "double"; //$NON-NLS-1$
3231 case TokenNameelse :
3232 return "else"; //$NON-NLS-1$
3233 case TokenNameextends :
3234 return "extends"; //$NON-NLS-1$
3235 case TokenNamefalse :
3236 return "false"; //$NON-NLS-1$
3237 case TokenNamefinal :
3238 return "final"; //$NON-NLS-1$
3239 case TokenNamefinally :
3240 return "finally"; //$NON-NLS-1$
3241 case TokenNamefloat :
3242 return "float"; //$NON-NLS-1$
3244 return "for"; //$NON-NLS-1$
3246 return "if"; //$NON-NLS-1$
3247 case TokenNameimplements :
3248 return "implements"; //$NON-NLS-1$
3249 case TokenNameimport :
3250 return "import"; //$NON-NLS-1$
3251 case TokenNameinstanceof :
3252 return "instanceof"; //$NON-NLS-1$
3254 return "int"; //$NON-NLS-1$
3255 case TokenNameinterface :
3256 return "interface"; //$NON-NLS-1$
3257 case TokenNamelong :
3258 return "long"; //$NON-NLS-1$
3259 case TokenNamenative :
3260 return "native"; //$NON-NLS-1$
3262 return "new"; //$NON-NLS-1$
3263 case TokenNamenull :
3264 return "null"; //$NON-NLS-1$
3265 case TokenNamepackage :
3266 return "package"; //$NON-NLS-1$
3267 case TokenNameprivate :
3268 return "private"; //$NON-NLS-1$
3269 case TokenNameprotected :
3270 return "protected"; //$NON-NLS-1$
3271 case TokenNamepublic :
3272 return "public"; //$NON-NLS-1$
3273 case TokenNamereturn :
3274 return "return"; //$NON-NLS-1$
3275 case TokenNameshort :
3276 return "short"; //$NON-NLS-1$
3277 case TokenNamestatic :
3278 return "static"; //$NON-NLS-1$
3279 case TokenNamesuper :
3280 return "super"; //$NON-NLS-1$
3281 case TokenNameswitch :
3282 return "switch"; //$NON-NLS-1$
3283 case TokenNamesynchronized :
3284 return "synchronized"; //$NON-NLS-1$
3285 case TokenNamethis :
3286 return "this"; //$NON-NLS-1$
3287 case TokenNamethrow :
3288 return "throw"; //$NON-NLS-1$
3289 case TokenNamethrows :
3290 return "throws"; //$NON-NLS-1$
3291 case TokenNametransient :
3292 return "transient"; //$NON-NLS-1$
3293 case TokenNametrue :
3294 return "true"; //$NON-NLS-1$
3296 return "try"; //$NON-NLS-1$
3297 case TokenNamevoid :
3298 return "void"; //$NON-NLS-1$
3299 case TokenNamevolatile :
3300 return "volatile"; //$NON-NLS-1$
3301 case TokenNamewhile :
3302 return "while"; //$NON-NLS-1$
3304 case TokenNameIntegerLiteral :
3305 return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3306 case TokenNameLongLiteral :
3307 return "Long(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3308 case TokenNameFloatingPointLiteral :
3309 return "Float(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3310 case TokenNameDoubleLiteral :
3311 return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3312 case TokenNameCharacterLiteral :
3313 return "Char(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3314 case TokenNameStringLiteral :
3315 return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3317 case TokenNamePLUS_PLUS :
3318 return "++"; //$NON-NLS-1$
3319 case TokenNameMINUS_MINUS :
3320 return "--"; //$NON-NLS-1$
3321 case TokenNameEQUAL_EQUAL :
3322 return "=="; //$NON-NLS-1$
3323 case TokenNameLESS_EQUAL :
3324 return "<="; //$NON-NLS-1$
3325 case TokenNameGREATER_EQUAL :
3326 return ">="; //$NON-NLS-1$
3327 case TokenNameNOT_EQUAL :
3328 return "!="; //$NON-NLS-1$
3329 case TokenNameLEFT_SHIFT :
3330 return "<<"; //$NON-NLS-1$
3331 case TokenNameRIGHT_SHIFT :
3332 return ">>"; //$NON-NLS-1$
3333 case TokenNameUNSIGNED_RIGHT_SHIFT :
3334 return ">>>"; //$NON-NLS-1$
3335 case TokenNamePLUS_EQUAL :
3336 return "+="; //$NON-NLS-1$
3337 case TokenNameMINUS_EQUAL :
3338 return "-="; //$NON-NLS-1$
3339 case TokenNameMULTIPLY_EQUAL :
3340 return "*="; //$NON-NLS-1$
3341 case TokenNameDIVIDE_EQUAL :
3342 return "/="; //$NON-NLS-1$
3343 case TokenNameAND_EQUAL :
3344 return "&="; //$NON-NLS-1$
3345 case TokenNameOR_EQUAL :
3346 return "|="; //$NON-NLS-1$
3347 case TokenNameXOR_EQUAL :
3348 return "^="; //$NON-NLS-1$
3349 case TokenNameREMAINDER_EQUAL :
3350 return "%="; //$NON-NLS-1$
3351 case TokenNameLEFT_SHIFT_EQUAL :
3352 return "<<="; //$NON-NLS-1$
3353 case TokenNameRIGHT_SHIFT_EQUAL :
3354 return ">>="; //$NON-NLS-1$
3355 case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL :
3356 return ">>>="; //$NON-NLS-1$
3357 case TokenNameOR_OR :
3358 return "||"; //$NON-NLS-1$
3359 case TokenNameAND_AND :
3360 return "&&"; //$NON-NLS-1$
3361 case TokenNamePLUS :
3362 return "+"; //$NON-NLS-1$
3363 case TokenNameMINUS :
3364 return "-"; //$NON-NLS-1$
3366 return "!"; //$NON-NLS-1$
3367 case TokenNameREMAINDER :
3368 return "%"; //$NON-NLS-1$
3370 return "^"; //$NON-NLS-1$
3372 return "&"; //$NON-NLS-1$
3373 case TokenNameMULTIPLY :
3374 return "*"; //$NON-NLS-1$
3376 return "|"; //$NON-NLS-1$
3377 case TokenNameTWIDDLE :
3378 return "~"; //$NON-NLS-1$
3379 case TokenNameDIVIDE :
3380 return "/"; //$NON-NLS-1$
3381 case TokenNameGREATER :
3382 return ">"; //$NON-NLS-1$
3383 case TokenNameLESS :
3384 return "<"; //$NON-NLS-1$
3385 case TokenNameLPAREN :
3386 return "("; //$NON-NLS-1$
3387 case TokenNameRPAREN :
3388 return ")"; //$NON-NLS-1$
3389 case TokenNameLBRACE :
3390 return "{"; //$NON-NLS-1$
3391 case TokenNameRBRACE :
3392 return "}"; //$NON-NLS-1$
3393 case TokenNameLBRACKET :
3394 return "["; //$NON-NLS-1$
3395 case TokenNameRBRACKET :
3396 return "]"; //$NON-NLS-1$
3397 case TokenNameSEMICOLON :
3398 return ";"; //$NON-NLS-1$
3399 case TokenNameQUESTION :
3400 return "?"; //$NON-NLS-1$
3401 case TokenNameCOLON :
3402 return ":"; //$NON-NLS-1$
3403 case TokenNameCOMMA :
3404 return ","; //$NON-NLS-1$
3406 return "."; //$NON-NLS-1$
3407 case TokenNameEQUAL :
3408 return "="; //$NON-NLS-1$
3410 return "EOF"; //$NON-NLS-1$
3412 return "not-a-token"; //$NON-NLS-1$
3415 public void unicodeInitializeBuffer(int length) {
3416 this.withoutUnicodePtr = length;
3417 if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[length+(1+10)];
3418 int bLength = this.withoutUnicodeBuffer.length;
3419 if (1+length >= bLength) {
3420 System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length + (1+10)], 0, bLength);
3422 System.arraycopy(this.source, this.startPosition, this.withoutUnicodeBuffer, 1, length);
3424 public void unicodeStoreAt(int pos) {
3425 if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[10];
3426 int length = this.withoutUnicodeBuffer.length;
3427 if (pos == length) {
3428 System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length * 2], 0, length);
3430 this.withoutUnicodeBuffer[pos] = this.currentCharacter;