src/org/eclipse/jdt/internal/compiler/parser/Scanner.java

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2004 IBM Corporation and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v10.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  *******************************************************************************/
  11 package org.eclipse.jdt.internal.compiler.parser;
  12
  13 import java.util.Iterator;
  14
  15 import org.eclipse.jdt.core.compiler.CharOperation;
  16 import org.eclipse.jdt.core.compiler.InvalidInputException;
  17 import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
  18 import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
  19
  20 /**
  21  * IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in
  22  * org.eclipse.jdt.core.compiler public package where it is API.
  23  * The mirror implementation is using the backward compatible ITerminalSymbols constant
  24  * definitions (stable with 2.0), whereas the internal implementation uses TerminalTokens
  25  * which constant values reflect the latest parser generation state.
  26  */
  27 public class Scanner implements TerminalTokens {
  28
  29         /* APIs ares
  30          - getNextToken() which return the current type of the token
  31            (this value is not memorized by the scanner)
  32          - getCurrentTokenSource() which provides with the token "REAL" source
  33            (aka all unicode have been transformed into a correct char)
  34          - sourceStart gives the position into the stream
  35          - currentPosition-1 gives the sourceEnd position into the stream
  36         */
  37
  38         // 1.4 feature
  39         private boolean assertMode = false;
  40         public boolean useAssertAsAnIndentifier = false;
  41         //flag indicating if processed source contains occurrences of keyword assert
  42         public boolean containsAssertKeyword = false;
  43
  44         public boolean recordLineSeparator = false;
  45         public char currentCharacter;
  46         public int startPosition;
  47         public int currentPosition;
  48         public int initialPosition, eofPosition;
  49         // after this position eof are generated instead of real token from the source
  50
  51         public boolean tokenizeComments = false;
  52         public boolean tokenizeWhiteSpace = false;
  53
  54         //source should be viewed as a window (aka a part)
  55         //of a entire very large stream
  56         public char source[];
  57
  58         //unicode support
  59         public char[] withoutUnicodeBuffer;
  60         public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token
  61         public boolean unicodeAsBackSlash = false;
  62
  63         public boolean scanningFloatLiteral = false;
  64
  65         //support for /** comments
  66         public int[] commentStops = new int[10];
  67         public int[] commentStarts = new int[10];
  68         public int commentPtr = -1; // no comment test with commentPtr value -1
  69         protected int lastCommentLinePosition = -1;
  70
  71         // task tag support
  72         public char[][] foundTaskTags = null;
  73         public char[][] foundTaskMessages;
  74         public char[][] foundTaskPriorities = null;
  75         public int[][] foundTaskPositions;
  76         public int foundTaskCount = 0;
  77         public char[][] taskTags = null;
  78         public char[][] taskPriorities = null;
  79         public boolean isTaskCaseSensitive = true;
  80
  81         //diet parsing support - jump over some method body when requested
  82         public boolean diet = false;
  83
  84         //support for the  poor-line-debuggers ....
  85         //remember the position of the cr/lf
  86         public int[] lineEnds = new int[250];
  87         public int linePtr = -1;
  88         public boolean wasAcr = false;
  89
  90         public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
  91
  92         public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
  93         public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
  94         public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant";  //$NON-NLS-1$
  95         public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
  96         public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
  97         public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
  98         public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
  99
 100         public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
 101         public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
 102         public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
 103         public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
 104
 105         //----------------optimized identifier managment------------------
 106         static final char[] charArray_a = new char[] {'a'},
 107                 charArray_b = new char[] {'b'},
 108                 charArray_c = new char[] {'c'},
 109                 charArray_d = new char[] {'d'},
 110                 charArray_e = new char[] {'e'},
 111                 charArray_f = new char[] {'f'},
 112                 charArray_g = new char[] {'g'},
 113                 charArray_h = new char[] {'h'},
 114                 charArray_i = new char[] {'i'},
 115                 charArray_j = new char[] {'j'},
 116                 charArray_k = new char[] {'k'},
 117                 charArray_l = new char[] {'l'},
 118                 charArray_m = new char[] {'m'},
 119                 charArray_n = new char[] {'n'},
 120                 charArray_o = new char[] {'o'},
 121                 charArray_p = new char[] {'p'},
 122                 charArray_q = new char[] {'q'},
 123                 charArray_r = new char[] {'r'},
 124                 charArray_s = new char[] {'s'},
 125                 charArray_t = new char[] {'t'},
 126                 charArray_u = new char[] {'u'},
 127                 charArray_v = new char[] {'v'},
 128                 charArray_w = new char[] {'w'},
 129                 charArray_x = new char[] {'x'},
 130                 charArray_y = new char[] {'y'},
 131                 charArray_z = new char[] {'z'};
 132
 133         static final char[] initCharArray =
 134                 new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'};
 135         static final int TableSize = 30, InternalTableSize = 6; //30*6 = 180 entries
 136         public static final int OptimizedLength = 6;
 137         public /*static*/ final char[][][][] charArray_length =
 138                 new char[OptimizedLength][TableSize][InternalTableSize][];
 139         // support for detecting non-externalized string literals
 140         NLSLine currentLine= null;
 141         public static final String TAG_PREFIX= "//$NON-NLS-"; //$NON-NLS-1$
 142         public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length();
 143         public static final String TAG_POSTFIX= "$"; //$NON-NLS-1$
 144         public static final int TAG_POSTFIX_LENGTH= TAG_POSTFIX.length();
 145         public StringLiteral[] nonNLSStrings = null;
 146         public boolean checkNonExternalizedStringLiterals = false;
 147         public boolean wasNonExternalizedStringLiteral = false;
 148
 149         /*static*/ {
 150                 for (int i = 0; i < 6; i++) {
 151                         for (int j = 0; j < TableSize; j++) {
 152                                 for (int k = 0; k < InternalTableSize; k++) {
 153                                         this.charArray_length[i][j][k] = initCharArray;
 154                                 }
 155                         }
 156                 }
 157         }
 158         static int newEntry2 = 0,
 159                 newEntry3 = 0,
 160                 newEntry4 = 0,
 161                 newEntry5 = 0,
 162                 newEntry6 = 0;
 163
 164         public static final int RoundBracket = 0;
 165         public static final int SquareBracket = 1;
 166         public static final int CurlyBracket = 2;
 167         public static final int BracketKinds = 3;
 168
 169 public Scanner() {
 170         this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/);
 171 }
 172
 173 public Scanner(
 174         boolean tokenizeComments,
 175         boolean tokenizeWhiteSpace,
 176         boolean checkNonExternalizedStringLiterals,
 177         long sourceLevel,
 178         char[][] taskTags,
 179         char[][] taskPriorities,
 180         boolean isTaskCaseSensitive) {
 181
 182         this.eofPosition = Integer.MAX_VALUE;
 183         this.tokenizeComments = tokenizeComments;
 184         this.tokenizeWhiteSpace = tokenizeWhiteSpace;
 185         this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
 186         this.assertMode = sourceLevel >= ClassFileConstants.JDK1_4;
 187         this.taskTags = taskTags;
 188         this.taskPriorities = taskPriorities;
 189         this.isTaskCaseSensitive = isTaskCaseSensitive;
 190 }
 191
 192 public  final boolean atEnd() {
 193         // This code is not relevant if source is
 194         // Only a part of the real stream input
 195
 196         return this.source.length == this.currentPosition;
 197 }
 198
 199 private void checkNonExternalizedString() {
 200         if (this.currentLine == null)
 201                 return;
 202         parseTags(this.currentLine);
 203 }
 204
 205 // chech presence of task: tags
 206 // TODO (frederic) see if we need to take unicode characters into account...
 207 public void checkTaskTag(int commentStart, int commentEnd) {
 208         char[] src = this.source;
 209
 210         // only look for newer task: tags
 211         if (this.foundTaskCount > 0
 212                 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
 213                 return;
 214         }
 215         int foundTaskIndex = this.foundTaskCount;
 216         char previous = src[commentStart+1]; // should be '*' or '/'
 217         nextChar : for (
 218                 int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) {
 219                 char[] tag = null;
 220                 char[] priority = null;
 221                 // check for tag occurrence only if not ambiguous with javadoc tag
 222                 if (previous != '@') {
 223                         nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) {
 224                                 tag = this.taskTags[itag];
 225                                 int tagLength = tag.length;
 226                                 if (tagLength == 0) continue nextTag;
 227
 228                                 // ensure tag is not leaded with letter if tag starts with a letter
 229                                 if (Character.isJavaIdentifierStart(tag[0])) {
 230                                         if (Character.isJavaIdentifierPart(previous)) {
 231                                                 continue nextTag;
 232                                         }
 233                                 }
 234
 235                                 for (int t = 0; t < tagLength; t++) {
 236                                         char sc, tc;
 237                                         int x = i+t;
 238                                         if (x >= this.eofPosition || x >= commentEnd) continue nextTag;
 239                                         if ((sc = src[i + t]) != (tc = tag[t])) {                                                                                                                                                                       // case sensitive check
 240                                                 if (this.isTaskCaseSensitive || (Character.toLowerCase(sc) != Character.toLowerCase(tc))) {     // case insensitive check
 241                                                         continue nextTag;
 242                                                 }
 243                                         }
 244                                 }
 245                                 // ensure tag is not followed with letter if tag finishes with a letter
 246                                 if (i+tagLength < commentEnd && Character.isJavaIdentifierPart(src[i+tagLength-1])) {
 247                                         if (Character.isJavaIdentifierPart(src[i + tagLength]))
 248                                                 continue nextTag;
 249                                 }
 250                                 if (this.foundTaskTags == null) {
 251                                         this.foundTaskTags = new char[5][];
 252                                         this.foundTaskMessages = new char[5][];
 253                                         this.foundTaskPriorities = new char[5][];
 254                                         this.foundTaskPositions = new int[5][];
 255                                 } else if (this.foundTaskCount == this.foundTaskTags.length) {
 256                                         System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
 257                                         System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
 258                                         System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
 259                                         System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0, this.foundTaskCount);
 260                                 }
 261
 262                                 priority = this.taskPriorities != null && itag < this.taskPriorities.length
 263                                                         ? this.taskPriorities[itag]
 264                                                         : null;
 265
 266                                 this.foundTaskTags[this.foundTaskCount] = tag;
 267                                 this.foundTaskPriorities[this.foundTaskCount] = priority;
 268                                 this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
 269                                 this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
 270                                 this.foundTaskCount++;
 271                                 i += tagLength - 1; // will be incremented when looping
 272                                 break nextTag;
 273                         }
 274                 }
 275                 previous = src[i];
 276         }
 277         for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
 278                 // retrieve message start and end positions
 279                 int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
 280                 int max_value = i + 1 < this.foundTaskCount
 281                                 ? this.foundTaskPositions[i + 1][0] - 1
 282                                 : commentEnd - 1;
 283                 // at most beginning of next task
 284                 if (max_value < msgStart) {
 285                         max_value = msgStart; // would only occur if tag is before EOF.
 286                 }
 287                 int end = -1;
 288                 char c;
 289                 for (int j = msgStart; j < max_value; j++) {
 290                         if ((c = src[j]) == '\n' || c == '\r') {
 291                                 end = j - 1;
 292                                 break;
 293                         }
 294                 }
 295                 if (end == -1) {
 296                         for (int j = max_value; j > msgStart; j--) {
 297                                 if ((c = src[j]) == '*') {
 298                                         end = j - 1;
 299                                         break;
 300                                 }
 301                         }
 302                         if (end == -1)
 303                                 end = max_value;
 304                 }
 305                 if (msgStart == end)
 306                         continue; // empty
 307                 // trim the message
 308                 while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
 309                         end--;
 310                 while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
 311                         msgStart++;
 312                 // update the end position of the task
 313                 this.foundTaskPositions[i][1] = end;
 314                 // get the message source
 315                 final int messageLength = end - msgStart + 1;
 316                 char[] message = new char[messageLength];
 317                 System.arraycopy(src, msgStart, message, 0, messageLength);
 318                 this.foundTaskMessages[i] = message;
 319         }
 320 }
 321 public char[] getCurrentIdentifierSource() {
 322         //return the token REAL source (aka unicodes are precomputed)
 323
 324         char[] result;
 325         if (this.withoutUnicodePtr != 0) {
 326                 //0 is used as a fast test flag so the real first char is in position 1
 327                 System.arraycopy(
 328                         this.withoutUnicodeBuffer,
 329                         1,
 330                         result = new char[this.withoutUnicodePtr],
 331                         0,
 332                         this.withoutUnicodePtr);
 333         } else {
 334                 int length = this.currentPosition - this.startPosition;
 335                 if (length == this.source.length) return this.source;
 336                 switch (length) { // see OptimizedLength
 337                         case 1 :
 338                                 return optimizedCurrentTokenSource1();
 339                         case 2 :
 340                                 return optimizedCurrentTokenSource2();
 341                         case 3 :
 342                                 return optimizedCurrentTokenSource3();
 343                         case 4 :
 344                                 return optimizedCurrentTokenSource4();
 345                         case 5 :
 346                                 return optimizedCurrentTokenSource5();
 347                         case 6 :
 348                                 return optimizedCurrentTokenSource6();
 349                 }
 350                 //no optimization
 351                 System.arraycopy(this.source, this.startPosition, result = new char[length], 0, length);
 352         }
 353         return result;
 354 }
 355 public int getCurrentTokenEndPosition(){
 356         return this.currentPosition - 1;
 357 }
 358 public final char[] getCurrentTokenSource() {
 359         // Return the token REAL source (aka unicodes are precomputed)
 360
 361         char[] result;
 362         if (this.withoutUnicodePtr != 0)
 363                 // 0 is used as a fast test flag so the real first char is in position 1
 364                 System.arraycopy(
 365                         this.withoutUnicodeBuffer,
 366                         1,
 367                         result = new char[this.withoutUnicodePtr],
 368                         0,
 369                         this.withoutUnicodePtr);
 370         else {
 371                 int length;
 372                 System.arraycopy(
 373                         this.source,
 374                         this.startPosition,
 375                         result = new char[length = this.currentPosition - this.startPosition],
 376                         0,
 377                         length);
 378         }
 379         return result;
 380 }
 381 public final char[] getCurrentTokenSourceString() {
 382         //return the token REAL source (aka unicodes are precomputed).
 383         //REMOVE the two " that are at the beginning and the end.
 384
 385         char[] result;
 386         if (this.withoutUnicodePtr != 0)
 387                 //0 is used as a fast test flag so the real first char is in position 1
 388                 System.arraycopy(this.withoutUnicodeBuffer, 2,
 389                 //2 is 1 (real start) + 1 (to jump over the ")
 390                 result = new char[this.withoutUnicodePtr - 2], 0, this.withoutUnicodePtr - 2);
 391         else {
 392                 int length;
 393                 System.arraycopy(
 394                         this.source,
 395                         this.startPosition + 1,
 396                         result = new char[length = this.currentPosition - this.startPosition - 2],
 397                         0,
 398                         length);
 399         }
 400         return result;
 401 }
 402
 403 public final char[] getRawTokenSource() {
 404         int length = this.currentPosition - this.startPosition;
 405         char[] tokenSource = new char[length];
 406         System.arraycopy(this.source, this.startPosition, tokenSource, 0, length);
 407         return tokenSource;
 408 }
 409
 410 public final char[] getRawTokenSourceEnd() {
 411         int length = this.eofPosition - this.currentPosition - 1;
 412         char[] sourceEnd = new char[length];
 413         System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length);
 414         return sourceEnd;
 415 }
 416
 417 public int getCurrentTokenStartPosition(){
 418         return this.startPosition;
 419 }
 420 /*
 421  * Search the source position corresponding to the end of a given line number
 422  *
 423  * Line numbers are 1-based, and relative to the scanner initialPosition.
 424  * Character positions are 0-based.
 425  *
 426  * In case the given line number is inconsistent, answers -1.
 427  */
 428 public final int getLineEnd(int lineNumber) {
 429
 430         if (this.lineEnds == null)
 431                 return -1;
 432         if (lineNumber > this.lineEnds.length+1)
 433                 return -1;
 434         if (lineNumber <= 0)
 435                 return -1;
 436         if (lineNumber == this.lineEnds.length + 1)
 437                 return this.eofPosition;
 438         return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
 439 }
 440
 441 public final int[] getLineEnds() {
 442         //return a bounded copy of this.lineEnds
 443
 444         int[] copy;
 445         System.arraycopy(this.lineEnds, 0, copy = new int[this.linePtr + 1], 0, this.linePtr + 1);
 446         return copy;
 447 }
 448
 449 /**
 450  * Search the source position corresponding to the beginning of a given line number
 451  *
 452  * Line numbers are 1-based, and relative to the scanner initialPosition.
 453  * Character positions are 0-based.
 454  *
 455  * e.g. getLineStart(1) --> 0   indicates that the first line starts at character 0.
 456  *
 457  * In case the given line number is inconsistent, answers -1.
 458  *
 459  * @param lineNumber int
 460  * @return int
 461  */
 462 public final int getLineStart(int lineNumber) {
 463
 464         if (this.lineEnds == null)
 465                 return -1;
 466         if (lineNumber > this.lineEnds.length + 1)
 467                 return -1;
 468         if (lineNumber <= 0)
 469                 return -1;
 470
 471         if (lineNumber == 1)
 472                 return this.initialPosition;
 473         return this.lineEnds[lineNumber-2]+1; // next line start one character behind the lineEnd of the previous line
 474 }
 475 public final int getNextChar() {
 476         try {
 477                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 478                         && (this.source[this.currentPosition] == 'u')) {
 479                         //-------------unicode traitement ------------
 480                         int c1, c2, c3, c4;
 481                         int unicodeSize = 6;
 482                         this.currentPosition++;
 483                         while (this.source[this.currentPosition] == 'u') {
 484                                 this.currentPosition++;
 485                                 unicodeSize++;
 486                         }
 487
 488                         if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
 489                                 || c1 < 0)
 490                                 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
 491                                 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
 492                                 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
 493                                 return -1;
 494                         }
 495
 496                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 497
 498                         this.unicodeAsBackSlash = this.currentCharacter == '\\';
 499
 500                         //need the unicode buffer
 501                         if (this.withoutUnicodePtr == 0) {
 502                                 //buffer all the entries that have been left aside....
 503                             unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
 504                         }
 505                         //fill the buffer with the char
 506                         unicodeStoreAt(++this.withoutUnicodePtr);
 507                         return this.currentCharacter;
 508
 509                 } //-------------end unicode traitement--------------
 510                 else {
 511                         this.unicodeAsBackSlash = false;
 512                         if (this.withoutUnicodePtr != 0) {
 513                             unicodeStoreAt(++this.withoutUnicodePtr);
 514                         }
 515                         return this.currentCharacter;
 516                 }
 517         } catch (IndexOutOfBoundsException e) {
 518                 return -1;
 519         }
 520 }
 521 public final boolean getNextChar(char testedChar) {
 522         //BOOLEAN
 523         //handle the case of unicode.
 524         //when a unicode appears then we must use a buffer that holds char internal values
 525         //At the end of this method currentCharacter holds the new visited char
 526         //and currentPosition points right next after it
 527         //Both previous lines are true if the currentCharacter is == to the testedChar
 528         //On false, no side effect has occured.
 529
 530         //ALL getNextChar.... ARE OPTIMIZED COPIES
 531
 532         if (this.currentPosition >= this.source.length) { // handle the obvious case upfront
 533                 this.unicodeAsBackSlash = false;
 534                 return false;
 535         }
 536
 537         int temp = this.currentPosition;
 538         try {
 539                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 540                         && (this.source[this.currentPosition] == 'u')) {
 541                         //-------------unicode traitement ------------
 542                         int c1, c2, c3, c4;
 543                         int unicodeSize = 6;
 544                         this.currentPosition++;
 545                         while (this.source[this.currentPosition] == 'u') {
 546                                 this.currentPosition++;
 547                                 unicodeSize++;
 548                         }
 549
 550                         if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
 551                                 || c1 < 0)
 552                                 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
 553                                 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
 554                                 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
 555                                 this.currentPosition = temp;
 556                                 return false;
 557                         }
 558
 559                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 560                         if (this.currentCharacter != testedChar) {
 561                                 this.currentPosition = temp;
 562                                 return false;
 563                         }
 564                         this.unicodeAsBackSlash = this.currentCharacter == '\\';
 565
 566                         //need the unicode buffer
 567                         if (this.withoutUnicodePtr == 0) {
 568                                 //buffer all the entries that have been left aside....
 569                             unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
 570                         }
 571                         //fill the buffer with the char
 572                         unicodeStoreAt(++this.withoutUnicodePtr);
 573                         return true;
 574
 575                 } //-------------end unicode traitement--------------
 576                 else {
 577                         if (this.currentCharacter != testedChar) {
 578                                 this.currentPosition = temp;
 579                                 return false;
 580                         }
 581                         this.unicodeAsBackSlash = false;
 582                         if (this.withoutUnicodePtr != 0)
 583                                 unicodeStoreAt(++this.withoutUnicodePtr);
 584                         return true;
 585                 }
 586         } catch (IndexOutOfBoundsException e) {
 587                 this.unicodeAsBackSlash = false;
 588                 this.currentPosition = temp;
 589                 return false;
 590         }
 591 }
 592 public final int getNextChar(char testedChar1, char testedChar2) {
 593         //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
 594         //test can be done with (x==0) for the first and (x>0) for the second
 595         //handle the case of unicode.
 596         //when a unicode appears then we must use a buffer that holds char internal values
 597         //At the end of this method currentCharacter holds the new visited char
 598         //and currentPosition points right next after it
 599         //Both previous lines are true if the currentCharacter is == to the testedChar1/2
 600         //On false, no side effect has occured.
 601
 602         //ALL getNextChar.... ARE OPTIMIZED COPIES
 603         if (this.currentPosition >= this.source.length) // handle the obvious case upfront
 604                 return -1;
 605
 606         int temp = this.currentPosition;
 607         try {
 608                 int result;
 609                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 610                         && (this.source[this.currentPosition] == 'u')) {
 611                         //-------------unicode traitement ------------
 612                         int c1, c2, c3, c4;
 613                         int unicodeSize = 6;
 614                         this.currentPosition++;
 615                         while (this.source[this.currentPosition] == 'u') {
 616                                 this.currentPosition++;
 617                                 unicodeSize++;
 618                         }
 619
 620                         if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
 621                                 || c1 < 0)
 622                                 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
 623                                 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
 624                                 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
 625                                 this.currentPosition = temp;
 626                                 return 2;
 627                         }
 628
 629                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 630                         if (this.currentCharacter == testedChar1)
 631                                 result = 0;
 632                         else
 633                                 if (this.currentCharacter == testedChar2)
 634                                         result = 1;
 635                                 else {
 636                                         this.currentPosition = temp;
 637                                         return -1;
 638                                 }
 639
 640                         //need the unicode buffer
 641                         if (this.withoutUnicodePtr == 0) {
 642                                 //buffer all the entries that have been left aside....
 643                                 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
 644                         }
 645                         //fill the buffer with the char
 646                         unicodeStoreAt(++this.withoutUnicodePtr);
 647                         return result;
 648                 } //-------------end unicode traitement--------------
 649                 else {
 650                         if (this.currentCharacter == testedChar1)
 651                                 result = 0;
 652                         else
 653                                 if (this.currentCharacter == testedChar2)
 654                                         result = 1;
 655                                 else {
 656                                         this.currentPosition = temp;
 657                                         return -1;
 658                                 }
 659
 660                         if (this.withoutUnicodePtr != 0)
 661                                 unicodeStoreAt(++this.withoutUnicodePtr);
 662                         return result;
 663                 }
 664         } catch (IndexOutOfBoundsException e) {
 665                 this.currentPosition = temp;
 666                 return -1;
 667         }
 668 }
 669 public final boolean getNextCharAsDigit() throws InvalidInputException {
 670         //BOOLEAN
 671         //handle the case of unicode.
 672         //when a unicode appears then we must use a buffer that holds char internal values
 673         //At the end of this method currentCharacter holds the new visited char
 674         //and currentPosition points right next after it
 675         //Both previous lines are true if the currentCharacter is a digit
 676         //On false, no side effect has occured.
 677
 678         //ALL getNextChar.... ARE OPTIMIZED COPIES
 679         if (this.currentPosition >= this.source.length) // handle the obvious case upfront
 680                 return false;
 681
 682         int temp = this.currentPosition;
 683         try {
 684                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 685                         && (this.source[this.currentPosition] == 'u')) {
 686                         //-------------unicode traitement ------------
 687                         int c1, c2, c3, c4;
 688                         int unicodeSize = 6;
 689                         this.currentPosition++;
 690                         while (this.source[this.currentPosition] == 'u') {
 691                                 this.currentPosition++;
 692                                 unicodeSize++;
 693                         }
 694
 695                         if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
 696                                 || c1 < 0)
 697                                 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
 698                                 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
 699                                 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
 700                                 this.currentPosition = temp;
 701                                 return false;
 702                         }
 703
 704                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 705                         if (!isDigit(this.currentCharacter)) {
 706                                 this.currentPosition = temp;
 707                                 return false;
 708                         }
 709
 710                         //need the unicode buffer
 711                         if (this.withoutUnicodePtr == 0) {
 712                                 //buffer all the entries that have been left aside....
 713                                 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
 714                         }
 715                         //fill the buffer with the char
 716                         unicodeStoreAt(++this.withoutUnicodePtr);
 717                         return true;
 718                 } //-------------end unicode traitement--------------
 719                 else {
 720                         if (!isDigit(this.currentCharacter)) {
 721                                 this.currentPosition = temp;
 722                                 return false;
 723                         }
 724                         if (this.withoutUnicodePtr != 0)
 725                                 unicodeStoreAt(++this.withoutUnicodePtr);
 726                         return true;
 727                 }
 728         } catch (IndexOutOfBoundsException e) {
 729                 this.currentPosition = temp;
 730                 return false;
 731         }
 732 }
 733 public final boolean getNextCharAsDigit(int radix) {
 734         //BOOLEAN
 735         //handle the case of unicode.
 736         //when a unicode appears then we must use a buffer that holds char internal values
 737         //At the end of this method currentCharacter holds the new visited char
 738         //and currentPosition points right next after it
 739         //Both previous lines are true if the currentCharacter is a digit base on radix
 740         //On false, no side effect has occured.
 741
 742         //ALL getNextChar.... ARE OPTIMIZED COPIES
 743         if (this.currentPosition >= this.source.length) // handle the obvious case upfront
 744                 return false;
 745
 746         int temp = this.currentPosition;
 747         try {
 748                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 749                         && (this.source[this.currentPosition] == 'u')) {
 750                         //-------------unicode traitement ------------
 751                         int c1, c2, c3, c4;
 752                         int unicodeSize = 6;
 753                         this.currentPosition++;
 754                         while (this.source[this.currentPosition] == 'u') {
 755                                 this.currentPosition++;
 756                                 unicodeSize++;
 757                         }
 758
 759                         if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
 760                                 || c1 < 0)
 761                                 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
 762                                 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
 763                                 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
 764                                 this.currentPosition = temp;
 765                                 return false;
 766                         }
 767
 768                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 769                         if (Character.digit(this.currentCharacter, radix) == -1) {
 770                                 this.currentPosition = temp;
 771                                 return false;
 772                         }
 773
 774                         //need the unicode buffer
 775                         if (this.withoutUnicodePtr == 0) {
 776                                 //buffer all the entries that have been left aside....
 777                                 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
 778                         }
 779                         //fill the buffer with the char
 780                         unicodeStoreAt(++this.withoutUnicodePtr);
 781                         return true;
 782                 } //-------------end unicode traitement--------------
 783                 else {
 784                         if (Character.digit(this.currentCharacter, radix) == -1) {
 785                                 this.currentPosition = temp;
 786                                 return false;
 787                         }
 788                         if (this.withoutUnicodePtr != 0)
 789                                 unicodeStoreAt(++this.withoutUnicodePtr);
 790                         return true;
 791                 }
 792         } catch (IndexOutOfBoundsException e) {
 793                 this.currentPosition = temp;
 794                 return false;
 795         }
 796 }
 797 public boolean getNextCharAsJavaIdentifierPart() {
 798         //BOOLEAN
 799         //handle the case of unicode.
 800         //when a unicode appears then we must use a buffer that holds char internal values
 801         //At the end of this method currentCharacter holds the new visited char
 802         //and currentPosition points right next after it
 803         //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
 804         //On false, no side effect has occured.
 805
 806         //ALL getNextChar.... ARE OPTIMIZED COPIES
 807         if (this.currentPosition >= this.source.length) // handle the obvious case upfront
 808                 return false;
 809
 810         int temp = this.currentPosition;
 811         try {
 812                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 813                         && (this.source[this.currentPosition] == 'u')) {
 814                         //-------------unicode traitement ------------
 815                         int c1, c2, c3, c4;
 816                         int unicodeSize = 6;
 817                         this.currentPosition++;
 818                         while (this.source[this.currentPosition] == 'u') {
 819                                 this.currentPosition++;
 820                                 unicodeSize++;
 821                         }
 822
 823                         if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
 824                                 || c1 < 0)
 825                                 || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
 826                                 || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
 827                                 || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
 828                                 this.currentPosition = temp;
 829                                 return false;
 830                         }
 831
 832                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 833                         if (!Character.isJavaIdentifierPart(this.currentCharacter)) {
 834                                 this.currentPosition = temp;
 835                                 return false;
 836                         }
 837
 838                         //need the unicode buffer
 839                         if (this.withoutUnicodePtr == 0) {
 840                                 //buffer all the entries that have been left aside....
 841                                 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
 842                         }
 843                         //fill the buffer with the char
 844                     unicodeStoreAt(++this.withoutUnicodePtr);
 845                         return true;
 846                 } //-------------end unicode traitement--------------
 847                 else {
 848                         if (!Character.isJavaIdentifierPart(this.currentCharacter)) {
 849                                 this.currentPosition = temp;
 850                                 return false;
 851                         }
 852
 853                         if (this.withoutUnicodePtr != 0)
 854                             unicodeStoreAt(++this.withoutUnicodePtr);
 855                         return true;
 856                 }
 857         } catch (IndexOutOfBoundsException e) {
 858                 this.currentPosition = temp;
 859                 return false;
 860         }
 861 }
 862 public int getNextToken() throws InvalidInputException {
 863         this.wasAcr = false;
 864         if (this.diet) {
 865                 jumpOverMethodBody();
 866                 this.diet = false;
 867                 return this.currentPosition > this.source.length ? TokenNameEOF : TokenNameRBRACE;
 868         }
 869         int whiteStart = 0;
 870         try {
 871                 while (true) { //loop for jumping over comments
 872                         this.withoutUnicodePtr = 0;
 873                         //start with a new token (even comment written with unicode )
 874
 875                         // ---------Consume white space and handles startPosition---------
 876                         whiteStart = this.currentPosition;
 877                         boolean isWhiteSpace, hasWhiteSpaces = false;
 878                         int offset = 0;
 879                         do {
 880                                 this.startPosition = this.currentPosition;
 881                                 boolean checkIfUnicode = false;
 882                                 try {
 883                                         checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 884                                                 && (this.source[this.currentPosition] == 'u');
 885                                 } catch(IndexOutOfBoundsException e) {
 886                                         if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
 887                                                 // reposition scanner in case we are interested by spaces as tokens
 888                                                 this.currentPosition--;
 889                                                 this.startPosition = whiteStart;
 890                                                 return TokenNameWHITESPACE;
 891                                         }
 892                                         if (this.currentPosition > this.eofPosition)
 893                                                 return TokenNameEOF;
 894                                 }
 895                                 if (checkIfUnicode) {
 896                                         isWhiteSpace = jumpOverUnicodeWhiteSpace();
 897                                         offset = 6;
 898                                 } else {
 899                                         offset = 1;
 900                                         if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
 901                                                 checkNonExternalizedString();
 902                                                 if (this.recordLineSeparator) {
 903                                                         pushLineSeparator();
 904                                                 } else {
 905                                                         this.currentLine = null;
 906                                                 }
 907                                         }
 908                                         isWhiteSpace =
 909                                                 (this.currentCharacter == ' ') || CharOperation.isWhitespace(this.currentCharacter);
 910                                 }
 911                                 if (isWhiteSpace) {
 912                                         hasWhiteSpaces = true;
 913                                 }
 914                         } while (isWhiteSpace);
 915                         if (this.tokenizeWhiteSpace && hasWhiteSpaces) {
 916                                 // reposition scanner in case we are interested by spaces as tokens
 917                                 this.currentPosition-=offset;
 918                                 this.startPosition = whiteStart;
 919                                 return TokenNameWHITESPACE;
 920                         }
 921                         //little trick to get out in the middle of a source compuation
 922                         if (this.currentPosition > this.eofPosition)
 923                                 return TokenNameEOF;
 924
 925                         // ---------Identify the next token-------------
 926
 927                         switch (this.currentCharacter) {
 928                                 case '(' :
 929                                         return TokenNameLPAREN;
 930                                 case ')' :
 931                                         return TokenNameRPAREN;
 932                                 case '{' :
 933                                         return TokenNameLBRACE;
 934                                 case '}' :
 935                                         return TokenNameRBRACE;
 936                                 case '[' :
 937                                         return TokenNameLBRACKET;
 938                                 case ']' :
 939                                         return TokenNameRBRACKET;
 940                                 case ';' :
 941                                         return TokenNameSEMICOLON;
 942                                 case ',' :
 943                                         return TokenNameCOMMA;
 944                                 case '.' :
 945                                         if (getNextCharAsDigit())
 946                                                 return scanNumber(true);
 947                                         return TokenNameDOT;
 948                                 case '+' :
 949                                         {
 950                                                 int test;
 951                                                 if ((test = getNextChar('+', '=')) == 0)
 952                                                         return TokenNamePLUS_PLUS;
 953                                                 if (test > 0)
 954                                                         return TokenNamePLUS_EQUAL;
 955                                                 return TokenNamePLUS;
 956                                         }
 957                                 case '-' :
 958                                         {
 959                                                 int test;
 960                                                 if ((test = getNextChar('-', '=')) == 0)
 961                                                         return TokenNameMINUS_MINUS;
 962                                                 if (test > 0)
 963                                                         return TokenNameMINUS_EQUAL;
 964                                                 return TokenNameMINUS;
 965                                         }
 966                                 case '~' :
 967                                         return TokenNameTWIDDLE;
 968                                 case '!' :
 969                                         if (getNextChar('='))
 970                                                 return TokenNameNOT_EQUAL;
 971                                         return TokenNameNOT;
 972                                 case '*' :
 973                                         if (getNextChar('='))
 974                                                 return TokenNameMULTIPLY_EQUAL;
 975                                         return TokenNameMULTIPLY;
 976                                 case '%' :
 977                                         if (getNextChar('='))
 978                                                 return TokenNameREMAINDER_EQUAL;
 979                                         return TokenNameREMAINDER;
 980                                 case '<' :
 981                                         {
 982                                                 int test;
 983                                                 if ((test = getNextChar('=', '<')) == 0)
 984                                                         return TokenNameLESS_EQUAL;
 985                                                 if (test > 0) {
 986                                                         if (getNextChar('='))
 987                                                                 return TokenNameLEFT_SHIFT_EQUAL;
 988                                                         return TokenNameLEFT_SHIFT;
 989                                                 }
 990                                                 return TokenNameLESS;
 991                                         }
 992                                 case '>' :
 993                                         {
 994                                                 int test;
 995                                                 if ((test = getNextChar('=', '>')) == 0)
 996                                                         return TokenNameGREATER_EQUAL;
 997                                                 if (test > 0) {
 998                                                         if ((test = getNextChar('=', '>')) == 0)
 999                                                                 return TokenNameRIGHT_SHIFT_EQUAL;
1000                                                         if (test > 0) {
1001                                                                 if (getNextChar('='))
1002                                                                         return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
1003                                                                 return TokenNameUNSIGNED_RIGHT_SHIFT;
1004                                                         }
1005                                                         return TokenNameRIGHT_SHIFT;
1006                                                 }
1007                                                 return TokenNameGREATER;
1008                                         }
1009                                 case '=' :
1010                                         if (getNextChar('='))
1011                                                 return TokenNameEQUAL_EQUAL;
1012                                         return TokenNameEQUAL;
1013                                 case '&' :
1014                                         {
1015                                                 int test;
1016                                                 if ((test = getNextChar('&', '=')) == 0)
1017                                                         return TokenNameAND_AND;
1018                                                 if (test > 0)
1019                                                         return TokenNameAND_EQUAL;
1020                                                 return TokenNameAND;
1021                                         }
1022                                 case '|' :
1023                                         {
1024                                                 int test;
1025                                                 if ((test = getNextChar('|', '=')) == 0)
1026                                                         return TokenNameOR_OR;
1027                                                 if (test > 0)
1028                                                         return TokenNameOR_EQUAL;
1029                                                 return TokenNameOR;
1030                                         }
1031                                 case '^' :
1032                                         if (getNextChar('='))
1033                                                 return TokenNameXOR_EQUAL;
1034                                         return TokenNameXOR;
1035                                 case '?' :
1036                                         return TokenNameQUESTION;
1037                                 case ':' :
1038                                         return TokenNameCOLON;
1039                                 case '\'' :
1040                                         {
1041                                                 int test;
1042                                                 if ((test = getNextChar('\n', '\r')) == 0) {
1043                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1044                                                 }
1045                                                 if (test > 0) {
1046                                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1047                                                         for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1048                                                                 if (this.currentPosition + lookAhead == this.source.length)
1049                                                                         break;
1050                                                                 if (this.source[this.currentPosition + lookAhead] == '\n')
1051                                                                         break;
1052                                                                 if (this.source[this.currentPosition + lookAhead] == '\'') {
1053                                                                         this.currentPosition += lookAhead + 1;
1054                                                                         break;
1055                                                                 }
1056                                                         }
1057                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1058                                                 }
1059                                         }
1060                                         if (getNextChar('\'')) {
1061                                                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1062                                                 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1063                                                         if (this.currentPosition + lookAhead == this.source.length)
1064                                                                 break;
1065                                                         if (this.source[this.currentPosition + lookAhead] == '\n')
1066                                                                 break;
1067                                                         if (this.source[this.currentPosition + lookAhead] == '\'') {
1068                                                                 this.currentPosition += lookAhead + 1;
1069                                                                 break;
1070                                                         }
1071                                                 }
1072                                                 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1073                                         }
1074                                         if (getNextChar('\\'))
1075                                                 scanEscapeCharacter();
1076                                         else { // consume next character
1077                                                 this.unicodeAsBackSlash = false;
1078                                                 boolean checkIfUnicode = false;
1079                                                 try {
1080                                                         checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1081                                                         && (this.source[this.currentPosition] == 'u');
1082                                                 } catch(IndexOutOfBoundsException e) {
1083                                                         this.currentPosition--;
1084                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1085                                                 }
1086                                                 if (checkIfUnicode) {
1087                                                         getNextUnicodeChar();
1088                                                 } else {
1089                                                         if (this.withoutUnicodePtr != 0) {
1090                                                                 unicodeStoreAt(++this.withoutUnicodePtr);
1091                                                         }
1092                                                 }
1093                                         }
1094                                         if (getNextChar('\''))
1095                                                 return TokenNameCharacterLiteral;
1096                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1097                                         for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
1098                                                 if (this.currentPosition + lookAhead == this.source.length)
1099                                                         break;
1100                                                 if (this.source[this.currentPosition + lookAhead] == '\n')
1101                                                         break;
1102                                                 if (this.source[this.currentPosition + lookAhead] == '\'') {
1103                                                         this.currentPosition += lookAhead + 1;
1104                                                         break;
1105                                                 }
1106                                         }
1107                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1108                                 case '"' :
1109                                         try {
1110                                                 // consume next character
1111                                                 this.unicodeAsBackSlash = false;
1112                                                 boolean isUnicode = false;
1113                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1114                                                         && (this.source[this.currentPosition] == 'u')) {
1115                                                         getNextUnicodeChar();
1116                                                         isUnicode = true;
1117                                                 } else {
1118                                                         if (this.withoutUnicodePtr != 0) {
1119                                                                 unicodeStoreAt(++this.withoutUnicodePtr);
1120                                                         }
1121                                                 }
1122
1123                                                 while (this.currentCharacter != '"') {
1124                                                         /**** \r and \n are not valid in string literals ****/
1125                                                         if ((this.currentCharacter == '\n') || (this.currentCharacter == '\r')) {
1126                                                                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1127                                                                 if (isUnicode) {
1128                                                                         int start = this.currentPosition;
1129                                                                         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1130                                                                                 if (this.currentPosition >= this.eofPosition) {
1131                                                                                         this.currentPosition = start;
1132                                                                                         break;
1133                                                                                 }
1134                                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
1135                                                                                         isUnicode = true;
1136                                                                                         getNextUnicodeChar();
1137                                                                                 } else {
1138                                                                                         isUnicode = false;
1139                                                                                 }
1140                                                                                 if (!isUnicode && this.currentCharacter == '\n') {
1141                                                                                         this.currentPosition--; // set current position on new line character
1142                                                                                         break;
1143                                                                                 }
1144                                                                                 if (this.currentCharacter == '\"') {
1145                                                                                         throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1146                                                                                 }
1147                                                                         }
1148                                                                 } else {
1149                                                                         this.currentPosition--; // set current position on new line character
1150                                                                 }
1151                                                                 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1152                                                         }
1153                                                         if (this.currentCharacter == '\\') {
1154                                                                 int escapeSize = this.currentPosition;
1155                                                                 boolean backSlashAsUnicodeInString = this.unicodeAsBackSlash;
1156                                                                 //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
1157                                                                 scanEscapeCharacter();
1158                                                                 escapeSize = this.currentPosition - escapeSize;
1159                                                                 if (this.withoutUnicodePtr == 0) {
1160                                                                         //buffer all the entries that have been left aside....
1161                                                                     unicodeInitializeBuffer(this.currentPosition - escapeSize - 1 - this.startPosition);
1162                                                                     unicodeStoreAt(++this.withoutUnicodePtr);
1163                                                                 } else { //overwrite the / in the buffer
1164                                                                     unicodeStoreAt(this.withoutUnicodePtr);
1165                                                                         if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
1166                                                                                 this.withoutUnicodePtr--;
1167                                                                         }
1168                                                                 }
1169                                                         }
1170                                                         // consume next character
1171                                                         this.unicodeAsBackSlash = false;
1172                                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1173                                                                 && (this.source[this.currentPosition] == 'u')) {
1174                                                                 getNextUnicodeChar();
1175                                                         } else {
1176                                                                 if (this.withoutUnicodePtr != 0) {
1177                                                                         unicodeStoreAt(++this.withoutUnicodePtr);
1178                                                                 }
1179                                                         }
1180
1181                                                 }
1182                                         } catch (IndexOutOfBoundsException e) {
1183                                                 this.currentPosition--;
1184                                                 throw new InvalidInputException(UNTERMINATED_STRING);
1185                                         } catch (InvalidInputException e) {
1186                                                 if (e.getMessage().equals(INVALID_ESCAPE)) {
1187                                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1188                                                         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1189                                                                 if (this.currentPosition + lookAhead == this.source.length)
1190                                                                         break;
1191                                                                 if (this.source[this.currentPosition + lookAhead] == '\n')
1192                                                                         break;
1193                                                                 if (this.source[this.currentPosition + lookAhead] == '\"') {
1194                                                                         this.currentPosition += lookAhead + 1;
1195                                                                         break;
1196                                                                 }
1197                                                         }
1198
1199                                                 }
1200                                                 throw e; // rethrow
1201                                         }
1202                                         if (this.checkNonExternalizedStringLiterals){ // check for presence of  NLS tags //$NON-NLS-?$ where ? is an int.
1203                                                 if (this.currentLine == null) {
1204                                                         this.currentLine = new NLSLine();
1205                                                 }
1206                                                 this.currentLine.add(
1207                                                         new StringLiteral(
1208                                                                 getCurrentTokenSourceString(),
1209                                                                 this.startPosition,
1210                                                                 this.currentPosition - 1));
1211                                         }
1212                                         return TokenNameStringLiteral;
1213                                 case '/' :
1214                                         {
1215                                                 int test;
1216                                                 if ((test = getNextChar('/', '*')) == 0) { //line comment
1217                                                         this.lastCommentLinePosition = this.currentPosition;
1218                                                         try { //get the next char
1219                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1220                                                                         && (this.source[this.currentPosition] == 'u')) {
1221                                                                         //-------------unicode traitement ------------
1222                                                                         int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1223                                                                         this.currentPosition++;
1224                                                                         while (this.source[this.currentPosition] == 'u') {
1225                                                                                 this.currentPosition++;
1226                                                                         }
1227                                                                         if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1228                                                                                 || c1 < 0
1229                                                                                 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1230                                                                                 || c2 < 0
1231                                                                                 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1232                                                                                 || c3 < 0
1233                                                                                 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1234                                                                                 || c4 < 0) {
1235                                                                                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1236                                                                         } else {
1237                                                                                 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1238                                                                         }
1239                                                                 }
1240
1241                                                                 //handle the \\u case manually into comment
1242                                                                 if (this.currentCharacter == '\\') {
1243                                                                         if (this.source[this.currentPosition] == '\\')
1244                                                                                 this.currentPosition++;
1245                                                                 } //jump over the \\
1246                                                                 boolean isUnicode = false;
1247                                                                 while (this.currentCharacter != '\r' && this.currentCharacter != '\n') {
1248                                                                         this.lastCommentLinePosition = this.currentPosition;
1249                                                                         //get the next char
1250                                                                         isUnicode = false;
1251                                                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1252                                                                                 && (this.source[this.currentPosition] == 'u')) {
1253                                                                                 isUnicode = true;
1254                                                                                 //-------------unicode traitement ------------
1255                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1256                                                                                 this.currentPosition++;
1257                                                                                 while (this.source[this.currentPosition] == 'u') {
1258                                                                                         this.currentPosition++;
1259                                                                                 }
1260                                                                                 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1261                                                                                         || c1 < 0
1262                                                                                         || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1263                                                                                         || c2 < 0
1264                                                                                         || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1265                                                                                         || c3 < 0
1266                                                                                         || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1267                                                                                         || c4 < 0) {
1268                                                                                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1269                                                                                 } else {
1270                                                                                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1271                                                                                 }
1272                                                                         }
1273                                                                         //handle the \\u case manually into comment
1274                                                                         if (this.currentCharacter == '\\') {
1275                                                                                 if (this.source[this.currentPosition] == '\\')
1276                                                                                         this.currentPosition++;
1277                                                                         } //jump over the \\
1278                                                                 }
1279                                                                 /*
1280                                                                  * We need to completely consume the line break
1281                                                                  */
1282                                                                 if (this.currentCharacter == '\r'
1283                                                                    && this.source.length > this.currentPosition) {
1284                                                                         if (this.source[this.currentPosition] == '\n') {
1285                                                                                 this.currentPosition++;
1286                                                                                 this.currentCharacter = '\n';
1287                                                                         } else if ((this.source[this.currentPosition] == '\\')
1288                                                                                 && (this.source[this.currentPosition + 1] == 'u')) {
1289                                                                                 isUnicode = true;
1290                                                                                 char unicodeChar;
1291                                                                                 int index = this.currentPosition + 1;
1292                                                                                 index++;
1293                                                                                 while (this.source[index] == 'u') {
1294                                                                                         index++;
1295                                                                                 }
1296                                                                                 //-------------unicode traitement ------------
1297                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1298                                                                                 if ((c1 = Character.getNumericValue(this.source[index++])) > 15
1299                                                                                         || c1 < 0
1300                                                                                         || (c2 = Character.getNumericValue(this.source[index++])) > 15
1301                                                                                         || c2 < 0
1302                                                                                         || (c3 = Character.getNumericValue(this.source[index++])) > 15
1303                                                                                         || c3 < 0
1304                                                                                         || (c4 = Character.getNumericValue(this.source[index++])) > 15
1305                                                                                         || c4 < 0) {
1306                                                                                         this.currentPosition = index;
1307                                                                                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1308                                                                                 } else {
1309                                                                                         unicodeChar = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1310                                                                                 }
1311                                                                                 if (unicodeChar == '\n') {
1312                                                                                         this.currentPosition = index;
1313                                                                                         this.currentCharacter = '\n';
1314                                                                                 }
1315                                                                         }
1316                                                                 }
1317                                                                 recordComment(TokenNameCOMMENT_LINE);
1318                                                                 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1319                                                                 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1320                                                                         checkNonExternalizedString();
1321                                                                         if (this.recordLineSeparator) {
1322                                                                                 if (isUnicode) {
1323                                                                                         pushUnicodeLineSeparator();
1324                                                                                 } else {
1325                                                                                         pushLineSeparator();
1326                                                                                 }
1327                                                                         } else {
1328                                                                                 this.currentLine = null;
1329                                                                         }
1330                                                                 }
1331                                                                 if (this.tokenizeComments) {
1332                                                                         return TokenNameCOMMENT_LINE;
1333                                                                 }
1334                                                         } catch (IndexOutOfBoundsException e) {
1335                                                                 this.currentPosition--;
1336                                                                 recordComment(TokenNameCOMMENT_LINE);
1337                                                                 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1338                                                                 if (this.tokenizeComments) {
1339                                                                         return TokenNameCOMMENT_LINE;
1340                                                                 } else {
1341                                                                         this.currentPosition++;
1342                                                                 }
1343                                                         }
1344                                                         break;
1345                                                 }
1346                                                 if (test > 0) { //traditional and javadoc comment
1347                                                         try { //get the next char
1348                                                                 boolean isJavadoc = false, star = false;
1349                                                                 boolean isUnicode = false;
1350                                                                 // consume next character
1351                                                                 this.unicodeAsBackSlash = false;
1352                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1353                                                                         && (this.source[this.currentPosition] == 'u')) {
1354                                                                         getNextUnicodeChar();
1355                                                                         isUnicode = true;
1356                                                                 } else {
1357                                                                         isUnicode = false;
1358                                                                         if (this.withoutUnicodePtr != 0) {
1359                                                                                 unicodeStoreAt(++this.withoutUnicodePtr);
1360                                                                         }
1361                                                                 }
1362
1363                                                                 if (this.currentCharacter == '*') {
1364                                                                         isJavadoc = true;
1365                                                                         star = true;
1366                                                                 }
1367                                                                 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1368                                                                         checkNonExternalizedString();
1369                                                                         if (this.recordLineSeparator) {
1370                                                                                 if (!isUnicode) {
1371                                                                                         pushLineSeparator();
1372                                                                                 }
1373                                                                         } else {
1374                                                                                 this.currentLine = null;
1375                                                                         }
1376                                                                 }
1377                                                                 isUnicode = false;
1378                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1379                                                                         && (this.source[this.currentPosition] == 'u')) {
1380                                                                         //-------------unicode traitement ------------
1381                                                                         getNextUnicodeChar();
1382                                                                         isUnicode = true;
1383                                                                 } else {
1384                                                                         isUnicode = false;
1385                                                                 }
1386                                                                 //handle the \\u case manually into comment
1387                                                                 if (this.currentCharacter == '\\') {
1388                                                                         if (this.source[this.currentPosition] == '\\')
1389                                                                                 this.currentPosition++; //jump over the \\
1390                                                                 }
1391                                                                 // empty comment is not a javadoc /**/
1392                                                                 if (this.currentCharacter == '/') {
1393                                                                         isJavadoc = false;
1394                                                                 }
1395                                                                 //loop until end of comment */
1396                                                                 while ((this.currentCharacter != '/') || (!star)) {
1397                                                                         if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1398                                                                                 checkNonExternalizedString();
1399                                                                                 if (this.recordLineSeparator) {
1400                                                                                         if (!isUnicode) {
1401                                                                                                 pushLineSeparator();
1402                                                                                         }
1403                                                                                 } else {
1404                                                                                         this.currentLine = null;
1405                                                                                 }
1406                                                                         }
1407                                                                         star = this.currentCharacter == '*';
1408                                                                         //get next char
1409                                                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1410                                                                                 && (this.source[this.currentPosition] == 'u')) {
1411                                                                                 //-------------unicode traitement ------------
1412                                                                                 getNextUnicodeChar();
1413                                                                                 isUnicode = true;
1414                                                                         } else {
1415                                                                                 isUnicode = false;
1416                                                                         }
1417                                                                         //handle the \\u case manually into comment
1418                                                                         if (this.currentCharacter == '\\') {
1419                                                                                 if (this.source[this.currentPosition] == '\\')
1420                                                                                         this.currentPosition++;
1421                                                                         } //jump over the \\
1422                                                                 }
1423                                                                 int token = isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK;
1424                                                                 recordComment(token);
1425                                                                 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1426                                                                 if (this.tokenizeComments) {
1427                                                                         /*
1428                                                                         if (isJavadoc)
1429                                                                                 return TokenNameCOMMENT_JAVADOC;
1430                                                                         return TokenNameCOMMENT_BLOCK;
1431                                                                         */
1432                                                                         return token;
1433                                                                 }
1434                                                         } catch (IndexOutOfBoundsException e) {
1435                                                                 this.currentPosition--;
1436                                                                 throw new InvalidInputException(UNTERMINATED_COMMENT);
1437                                                         }
1438                                                         break;
1439                                                 }
1440                                                 if (getNextChar('='))
1441                                                         return TokenNameDIVIDE_EQUAL;
1442                                                 return TokenNameDIVIDE;
1443                                         }
1444                                 case '\u001a' :
1445                                         if (atEnd())
1446                                                 return TokenNameEOF;
1447                                         //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
1448                                         throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1449
1450                                 default :
1451                                         if (Character.isJavaIdentifierStart(this.currentCharacter))
1452                                                 return scanIdentifierOrKeyword();
1453                                         if (isDigit(this.currentCharacter))
1454                                                 return scanNumber(false);
1455                                         return TokenNameERROR;
1456                         }
1457                 }
1458         } //-----------------end switch while try--------------------
1459         catch (IndexOutOfBoundsException e) {
1460                 if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
1461                         // reposition scanner in case we are interested by spaces as tokens
1462                         this.currentPosition--;
1463                         this.startPosition = whiteStart;
1464                         return TokenNameWHITESPACE;
1465                 }
1466         }
1467         return TokenNameEOF;
1468 }
1469 public final void getNextUnicodeChar()
1470         throws InvalidInputException {
1471         //VOID
1472         //handle the case of unicode.
1473         //when a unicode appears then we must use a buffer that holds char internal values
1474         //At the end of this method currentCharacter holds the new visited char
1475         //and currentPosition points right next after it
1476
1477         //ALL getNextChar.... ARE OPTIMIZED COPIES
1478
1479         try {
1480                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
1481                 this.currentPosition++;
1482                 while (this.source[this.currentPosition] == 'u') {
1483                         this.currentPosition++;
1484                         unicodeSize++;
1485                 }
1486
1487                 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1488                         || c1 < 0
1489                         || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1490                         || c2 < 0
1491                         || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1492                         || c3 < 0
1493                         || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1494                         || c4 < 0){
1495                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1496                 } else {
1497                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1498                         //need the unicode buffer
1499                         if (this.withoutUnicodePtr == 0) {
1500                                 //buffer all the entries that have been left aside....
1501                                 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
1502                         }
1503                         //fill the buffer with the char
1504                         unicodeStoreAt(++this.withoutUnicodePtr);
1505                 }
1506                 this.unicodeAsBackSlash = this.currentCharacter == '\\';
1507         } catch (ArrayIndexOutOfBoundsException e) {
1508                 this.currentPosition--;
1509                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1510         }
1511 }
1512
1513 public char[] getSource(){
1514         return this.source;
1515 }
1516 private boolean isDigit(char c) throws InvalidInputException {
1517         if (Character.isDigit(c)) {
1518                 switch(c) {
1519                         case '0' :
1520                         case '1' :
1521                         case '2' :
1522                         case '3' :
1523                         case '4' :
1524                         case '5' :
1525                         case '6' :
1526                         case '7' :
1527                         case '8' :
1528                         case '9' :
1529                                 return true;
1530                 }
1531                 throw new InvalidInputException(Scanner.INVALID_INPUT);
1532         } else {
1533                 return false;
1534         }
1535 }
1536 /* Tokenize a method body, assuming that curly brackets are properly balanced.
1537  */
1538 public final void jumpOverMethodBody() {
1539
1540         this.wasAcr = false;
1541         int found = 1;
1542         try {
1543                 while (true) { //loop for jumping over comments
1544                         // ---------Consume white space and handles startPosition---------
1545                         boolean isWhiteSpace;
1546                         do {
1547                                 this.startPosition = this.currentPosition;
1548                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1549                                         && (this.source[this.currentPosition] == 'u')) {
1550                                         isWhiteSpace = jumpOverUnicodeWhiteSpace();
1551                                 } else {
1552                                         if (this.recordLineSeparator
1553                                                 && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')))
1554                                                 pushLineSeparator();
1555                                         isWhiteSpace = CharOperation.isWhitespace(this.currentCharacter);
1556                                 }
1557                         } while (isWhiteSpace);
1558
1559                         // -------consume token until } is found---------
1560                         switch (this.currentCharacter) {
1561                                 case '{' :
1562                                         found++;
1563                                         break;
1564                                 case '}' :
1565                                         found--;
1566                                         if (found == 0)
1567                                                 return;
1568                                         break;
1569                                 case '\'' :
1570                                         {
1571                                                 boolean test;
1572                                                 test = getNextChar('\\');
1573                                                 if (test) {
1574                                                         try {
1575                                                                 scanEscapeCharacter();
1576                                                         } catch (InvalidInputException ex) {
1577                                                                 // ignore
1578                                                         }
1579                                                 } else {
1580                                                         try { // consume next character
1581                                                                 this.unicodeAsBackSlash = false;
1582                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1583                                                                         && (this.source[this.currentPosition] == 'u')) {
1584                                                                         getNextUnicodeChar();
1585                                                                 } else {
1586                                                                         if (this.withoutUnicodePtr != 0) {
1587                                                                                 unicodeStoreAt(++this.withoutUnicodePtr);
1588                                                                         }
1589                                                                 }
1590                                                         } catch (InvalidInputException ex) {
1591                                                                 // ignore
1592                                                         }
1593                                                 }
1594                                                 getNextChar('\'');
1595                                                 break;
1596                                         }
1597                                 case '"' :
1598                                         try {
1599                                                 try { // consume next character
1600                                                         this.unicodeAsBackSlash = false;
1601                                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1602                                                                 && (this.source[this.currentPosition] == 'u')) {
1603                                                                 getNextUnicodeChar();
1604                                                         } else {
1605                                                                 if (this.withoutUnicodePtr != 0) {
1606                                                                     unicodeStoreAt(++this.withoutUnicodePtr);
1607                                                                 }
1608                                                         }
1609                                                 } catch (InvalidInputException ex) {
1610                                                                 // ignore
1611                                                 }
1612                                                 while (this.currentCharacter != '"') {
1613                                                         if (this.currentCharacter == '\r'){
1614                                                                 if (this.source[this.currentPosition] == '\n') this.currentPosition++;
1615                                                                 break; // the string cannot go further that the line
1616                                                         }
1617                                                         if (this.currentCharacter == '\n'){
1618                                                                 break; // the string cannot go further that the line
1619                                                         }
1620                                                         if (this.currentCharacter == '\\') {
1621                                                                 try {
1622                                                                         scanEscapeCharacter();
1623                                                                 } catch (InvalidInputException ex) {
1624                                                                         // ignore
1625                                                                 }
1626                                                         }
1627                                                         try { // consume next character
1628                                                                 this.unicodeAsBackSlash = false;
1629                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1630                                                                         && (this.source[this.currentPosition] == 'u')) {
1631                                                                         getNextUnicodeChar();
1632                                                                 } else {
1633                                                                         if (this.withoutUnicodePtr != 0) {
1634                                                                                 unicodeStoreAt(++this.withoutUnicodePtr);
1635                                                                         }
1636                                                                 }
1637                                                         } catch (InvalidInputException ex) {
1638                                                                 // ignore
1639                                                         }
1640                                                 }
1641                                         } catch (IndexOutOfBoundsException e) {
1642                                                 return;
1643                                         }
1644                                         break;
1645                                 case '/' :
1646                                         {
1647                                                 int test;
1648                                                 if ((test = getNextChar('/', '*')) == 0) { //line comment
1649                                                         try {
1650                                                                 this.lastCommentLinePosition = this.currentPosition;
1651                                                                 //get the next char
1652                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1653                                                                         && (this.source[this.currentPosition] == 'u')) {
1654                                                                         //-------------unicode traitement ------------
1655                                                                         int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1656                                                                         this.currentPosition++;
1657                                                                         while (this.source[this.currentPosition] == 'u') {
1658                                                                                 this.currentPosition++;
1659                                                                         }
1660                                                                         if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1661                                                                                 || c1 < 0
1662                                                                                 || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1663                                                                                 || c2 < 0
1664                                                                                 || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1665                                                                                 || c3 < 0
1666                                                                                 || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1667                                                                                 || c4 < 0) { //error don't care of the value
1668                                                                                 this.currentCharacter = 'A';
1669                                                                         } //something different from \n and \r
1670                                                                         else {
1671                                                                                 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1672                                                                         }
1673                                                                 }
1674                                                                 //handle the \\u case manually into comment
1675                                                                 if (this.currentCharacter == '\\') {
1676                                                                         if (this.source[this.currentPosition] == '\\')
1677                                                                                 this.currentPosition++;
1678                                                                 } //jump over the \\
1679                                                                 boolean isUnicode = false;
1680                                                                 while (this.currentCharacter != '\r' && this.currentCharacter != '\n') {
1681                                                                         this.lastCommentLinePosition = this.currentPosition;
1682                                                                         //get the next char
1683                                                                         isUnicode = false;
1684                                                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1685                                                                                 && (this.source[this.currentPosition] == 'u')) {
1686                                                                                 isUnicode = true;
1687                                                                                 //-------------unicode traitement ------------
1688                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1689                                                                                 this.currentPosition++;
1690                                                                                 while (this.source[this.currentPosition] == 'u') {
1691                                                                                         this.currentPosition++;
1692                                                                                 }
1693                                                                                 if ((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1694                                                                                         || c1 < 0
1695                                                                                         || (c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1696                                                                                         || c2 < 0
1697                                                                                         || (c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1698                                                                                         || c3 < 0
1699                                                                                         || (c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1700                                                                                         || c4 < 0) { //error don't care of the value
1701                                                                                         this.currentCharacter = 'A';
1702                                                                                 } //something different from \n and \r
1703                                                                                 else {
1704                                                                                         this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1705                                                                                 }
1706                                                                         }
1707                                                                         //handle the \\u case manually into comment
1708                                                                         if (this.currentCharacter == '\\') {
1709                                                                                 if (this.source[this.currentPosition] == '\\')
1710                                                                                         this.currentPosition++;
1711                                                                         } //jump over the \\
1712                                                                 }
1713                                                                 /*
1714                                                                  * We need to completely consume the line break
1715                                                                  */
1716                                                                 if (this.currentCharacter == '\r'
1717                                                                    && this.source.length > this.currentPosition) {
1718                                                                         if (this.source[this.currentPosition] == '\n') {
1719                                                                                 this.currentPosition++;
1720                                                                                 this.currentCharacter = '\n';
1721                                                                         } else if ((this.source[this.currentPosition] == '\\')
1722                                                                                 && (this.source[this.currentPosition + 1] == 'u')) {
1723                                                                                 isUnicode = true;
1724                                                                                 char unicodeChar;
1725                                                                                 int index = this.currentPosition + 1;
1726                                                                                 index++;
1727                                                                                 while (this.source[index] == 'u') {
1728                                                                                         index++;
1729                                                                                 }
1730                                                                                 //-------------unicode traitement ------------
1731                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1732                                                                                 if ((c1 = Character.getNumericValue(this.source[index++])) > 15
1733                                                                                         || c1 < 0
1734                                                                                         || (c2 = Character.getNumericValue(this.source[index++])) > 15
1735                                                                                         || c2 < 0
1736                                                                                         || (c3 = Character.getNumericValue(this.source[index++])) > 15
1737                                                                                         || c3 < 0
1738                                                                                         || (c4 = Character.getNumericValue(this.source[index++])) > 15
1739                                                                                         || c4 < 0) { //error don't care of the value
1740                                                                                         unicodeChar = 'A';
1741                                                                                 } else {
1742                                                                                         unicodeChar = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1743                                                                                 }
1744                                                                                 if (unicodeChar == '\n') {
1745                                                                                         this.currentPosition = index;
1746                                                                                         this.currentCharacter = '\n';
1747                                                                                 }
1748                                                                         }
1749                                                                 }
1750                                                                 recordComment(TokenNameCOMMENT_LINE);
1751                                                                 if (this.recordLineSeparator
1752                                                                         && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n'))) {
1753                                                                                 if (isUnicode) {
1754                                                                                         pushUnicodeLineSeparator();
1755                                                                                 } else {
1756                                                                                         pushLineSeparator();
1757                                                                                 }
1758                                                                         }
1759                                                         } catch (IndexOutOfBoundsException e) {
1760                                                                  //an eof will then be generated
1761                                                                 this.currentPosition--;
1762                                                                 recordComment(TokenNameCOMMENT_LINE);
1763                                                                 if (!this.tokenizeComments) {
1764                                                                         this.currentPosition++;
1765                                                                 }
1766                                                         }
1767                                                         break;
1768                                                 }
1769                                                 if (test > 0) { //traditional and javadoc comment
1770                                                         boolean isJavadoc = false;
1771                                                         try { //get the next char
1772                                                                 boolean star = false;
1773                                                                 boolean isUnicode = false;
1774                                                                 // consume next character
1775                                                                 this.unicodeAsBackSlash = false;
1776                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1777                                                                         && (this.source[this.currentPosition] == 'u')) {
1778                                                                         getNextUnicodeChar();
1779                                                                         isUnicode = true;
1780                                                                 } else {
1781                                                                         isUnicode = false;
1782                                                                         if (this.withoutUnicodePtr != 0) {
1783                                                                     unicodeStoreAt(++this.withoutUnicodePtr);
1784                                                                         }
1785                                                                 }
1786
1787                                                                 if (this.currentCharacter == '*') {
1788                                                                         isJavadoc = true;
1789                                                                         star = true;
1790                                                                 }
1791                                                                 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1792                                                                         if (this.recordLineSeparator) {
1793                                                                                 if (isUnicode) {
1794                                                                                         pushUnicodeLineSeparator();
1795                                                                                 } else {
1796                                                                                         pushLineSeparator();
1797                                                                                 }
1798                                                                         } else {
1799                                                                                 this.currentLine = null;
1800                                                                         }
1801                                                                 }
1802                                                                 isUnicode = false;
1803                                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1804                                                                         && (this.source[this.currentPosition] == 'u')) {
1805                                                                         //-------------unicode traitement ------------
1806                                                                         getNextUnicodeChar();
1807                                                                         isUnicode = true;
1808                                                                 } else {
1809                                                                         isUnicode = false;
1810                                                                 }
1811                                                                 //handle the \\u case manually into comment
1812                                                                 if (this.currentCharacter == '\\') {
1813                                                                         if (this.source[this.currentPosition] == '\\')
1814                                                                                 this.currentPosition++; //jump over the \\
1815                                                                 }
1816                                                                 // empty comment is not a javadoc /**/
1817                                                                 if (this.currentCharacter == '/') {
1818                                                                         isJavadoc = false;
1819                                                                 }
1820                                                                 //loop until end of comment */
1821                                                                 while ((this.currentCharacter != '/') || (!star)) {
1822                                                                         if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1823                                                                                 if (this.recordLineSeparator) {
1824                                                                                         if (isUnicode) {
1825                                                                                                 pushUnicodeLineSeparator();
1826                                                                                         } else {
1827                                                                                                 pushLineSeparator();
1828                                                                                         }
1829                                                                                 } else {
1830                                                                                         this.currentLine = null;
1831                                                                                 }
1832                                                                         }
1833                                                                         star = this.currentCharacter == '*';
1834                                                                         //get next char
1835                                                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1836                                                                                 && (this.source[this.currentPosition] == 'u')) {
1837                                                                                 //-------------unicode traitement ------------
1838                                                                                 getNextUnicodeChar();
1839                                                                                 isUnicode = true;
1840                                                                         } else {
1841                                                                                 isUnicode = false;
1842                                                                         }
1843                                                                         //handle the \\u case manually into comment
1844                                                                         if (this.currentCharacter == '\\') {
1845                                                                                 if (this.source[this.currentPosition] == '\\')
1846                                                                                         this.currentPosition++;
1847                                                                         } //jump over the \\
1848                                                                 }
1849                                                                 recordComment(isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK);
1850                                                         } catch (IndexOutOfBoundsException e) {
1851                                                                 return;
1852                                                         }
1853                                                         break;
1854                                                 }
1855                                                 break;
1856                                         }
1857
1858                                 default :
1859                                         if (Character.isJavaIdentifierStart(this.currentCharacter)) {
1860                                                 scanIdentifierOrKeyword();
1861                                                 break;
1862                                         }
1863                                         if (isDigit(this.currentCharacter)) {
1864                                                 try {
1865                                                         scanNumber(false);
1866                                                 } catch (InvalidInputException ex) {
1867                                                         // ignore
1868                                                 }
1869                                                 break;
1870                                         }
1871                         }
1872                 }
1873                 //-----------------end switch while try--------------------
1874         } catch (IndexOutOfBoundsException e) {
1875                 // ignore
1876         } catch (InvalidInputException e) {
1877                 // ignore
1878         }
1879         return;
1880 }
1881 public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException {
1882         //BOOLEAN
1883         //handle the case of unicode. Jump over the next whiteSpace
1884         //making startPosition pointing on the next available char
1885         //On false, the currentCharacter is filled up with a potential
1886         //correct char
1887
1888         try {
1889                 this.wasAcr = false;
1890                 int c1, c2, c3, c4;
1891                 int unicodeSize = 6;
1892                 this.currentPosition++;
1893                 while (this.source[this.currentPosition] == 'u') {
1894                         this.currentPosition++;
1895                         unicodeSize++;
1896                 }
1897
1898                 if (((c1 = Character.getNumericValue(this.source[this.currentPosition++])) > 15
1899                         || c1 < 0)
1900                         || ((c2 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c2 < 0)
1901                         || ((c3 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c3 < 0)
1902                         || ((c4 = Character.getNumericValue(this.source[this.currentPosition++])) > 15 || c4 < 0)) {
1903                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1904                 }
1905
1906                 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1907                 if (CharOperation.isWhitespace(this.currentCharacter))
1908                         return true;
1909
1910                 //buffer the new char which is not a white space
1911                 unicodeStoreAt(++this.withoutUnicodePtr);
1912                 //this.withoutUnicodePtr == 1 is true here
1913                 return false;
1914         } catch (IndexOutOfBoundsException e){
1915                 this.currentPosition--;
1916                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1917         }
1918 }
1919
1920 final char[] optimizedCurrentTokenSource1() {
1921         //return always the same char[] build only once
1922
1923         //optimization at no speed cost of 99.5 % of the singleCharIdentifier
1924         char charOne = this.source[this.startPosition];
1925         switch (charOne) {
1926                 case 'a' :
1927                         return charArray_a;
1928                 case 'b' :
1929                         return charArray_b;
1930                 case 'c' :
1931                         return charArray_c;
1932                 case 'd' :
1933                         return charArray_d;
1934                 case 'e' :
1935                         return charArray_e;
1936                 case 'f' :
1937                         return charArray_f;
1938                 case 'g' :
1939                         return charArray_g;
1940                 case 'h' :
1941                         return charArray_h;
1942                 case 'i' :
1943                         return charArray_i;
1944                 case 'j' :
1945                         return charArray_j;
1946                 case 'k' :
1947                         return charArray_k;
1948                 case 'l' :
1949                         return charArray_l;
1950                 case 'm' :
1951                         return charArray_m;
1952                 case 'n' :
1953                         return charArray_n;
1954                 case 'o' :
1955                         return charArray_o;
1956                 case 'p' :
1957                         return charArray_p;
1958                 case 'q' :
1959                         return charArray_q;
1960                 case 'r' :
1961                         return charArray_r;
1962                 case 's' :
1963                         return charArray_s;
1964                 case 't' :
1965                         return charArray_t;
1966                 case 'u' :
1967                         return charArray_u;
1968                 case 'v' :
1969                         return charArray_v;
1970                 case 'w' :
1971                         return charArray_w;
1972                 case 'x' :
1973                         return charArray_x;
1974                 case 'y' :
1975                         return charArray_y;
1976                 case 'z' :
1977                         return charArray_z;
1978                 default :
1979                         return new char[] {charOne};
1980         }
1981 }
1982 final char[] optimizedCurrentTokenSource2() {
1983         //try to return the same char[] build only once
1984
1985         char c0, c1;
1986         int hash =
1987                 (((c0 = this.source[this.startPosition]) << 6) + (c1 = this.source[this.startPosition + 1]))
1988                         % TableSize;
1989         char[][] table = this.charArray_length[0][hash];
1990         int i = newEntry2;
1991         while (++i < InternalTableSize) {
1992                 char[] charArray = table[i];
1993                 if ((c0 == charArray[0]) && (c1 == charArray[1]))
1994                         return charArray;
1995         }
1996         //---------other side---------
1997         i = -1;
1998         int max = newEntry2;
1999         while (++i <= max) {
2000                 char[] charArray = table[i];
2001                 if ((c0 == charArray[0]) && (c1 == charArray[1]))
2002                         return charArray;
2003         }
2004         //--------add the entry-------
2005         if (++max >= InternalTableSize) max = 0;
2006         char[] r;
2007         table[max] = (r = new char[] {c0, c1});
2008         newEntry2 = max;
2009         return r;
2010 }
2011 final char[] optimizedCurrentTokenSource3() {
2012         //try to return the same char[] build only once
2013
2014         char c0, c1, c2;
2015         int hash =
2016                 (((c0 = this.source[this.startPosition]) << 12)
2017                         + ((c1 = this.source[this.startPosition + 1]) << 6)
2018                         + (c2 = this.source[this.startPosition + 2]))
2019                         % TableSize;
2020         char[][] table = this.charArray_length[1][hash];
2021         int i = newEntry3;
2022         while (++i < InternalTableSize) {
2023                 char[] charArray = table[i];
2024                 if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2025                         return charArray;
2026         }
2027         //---------other side---------
2028         i = -1;
2029         int max = newEntry3;
2030         while (++i <= max) {
2031                 char[] charArray = table[i];
2032                 if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2033                         return charArray;
2034         }
2035         //--------add the entry-------
2036         if (++max >= InternalTableSize) max = 0;
2037         char[] r;
2038         table[max] = (r = new char[] {c0, c1, c2});
2039         newEntry3 = max;
2040         return r;
2041 }
2042 final char[] optimizedCurrentTokenSource4() {
2043         //try to return the same char[] build only once
2044
2045         char c0, c1, c2, c3;
2046         long hash =
2047                 ((((long) (c0 = this.source[this.startPosition])) << 18)
2048                         + ((c1 = this.source[this.startPosition + 1]) << 12)
2049                         + ((c2 = this.source[this.startPosition + 2]) << 6)
2050                         + (c3 = this.source[this.startPosition + 3]))
2051                         % TableSize;
2052         char[][] table = this.charArray_length[2][(int) hash];
2053         int i = newEntry4;
2054         while (++i < InternalTableSize) {
2055                 char[] charArray = table[i];
2056                 if ((c0 == charArray[0])
2057                         && (c1 == charArray[1])
2058                         && (c2 == charArray[2])
2059                         && (c3 == charArray[3]))
2060                         return charArray;
2061         }
2062         //---------other side---------
2063         i = -1;
2064         int max = newEntry4;
2065         while (++i <= max) {
2066                 char[] charArray = table[i];
2067                 if ((c0 == charArray[0])
2068                         && (c1 == charArray[1])
2069                         && (c2 == charArray[2])
2070                         && (c3 == charArray[3]))
2071                         return charArray;
2072         }
2073         //--------add the entry-------
2074         if (++max >= InternalTableSize) max = 0;
2075         char[] r;
2076         table[max] = (r = new char[] {c0, c1, c2, c3});
2077         newEntry4 = max;
2078         return r;
2079
2080 }
2081 final char[] optimizedCurrentTokenSource5() {
2082         //try to return the same char[] build only once
2083
2084         char c0, c1, c2, c3, c4;
2085         long hash =
2086                 ((((long) (c0 = this.source[this.startPosition])) << 24)
2087                         + (((long) (c1 = this.source[this.startPosition + 1])) << 18)
2088                         + ((c2 = this.source[this.startPosition + 2]) << 12)
2089                         + ((c3 = this.source[this.startPosition + 3]) << 6)
2090                         + (c4 = this.source[this.startPosition + 4]))
2091                         % TableSize;
2092         char[][] table = this.charArray_length[3][(int) hash];
2093         int i = newEntry5;
2094         while (++i < InternalTableSize) {
2095                 char[] charArray = table[i];
2096                 if ((c0 == charArray[0])
2097                         && (c1 == charArray[1])
2098                         && (c2 == charArray[2])
2099                         && (c3 == charArray[3])
2100                         && (c4 == charArray[4]))
2101                         return charArray;
2102         }
2103         //---------other side---------
2104         i = -1;
2105         int max = newEntry5;
2106         while (++i <= max) {
2107                 char[] charArray = table[i];
2108                 if ((c0 == charArray[0])
2109                         && (c1 == charArray[1])
2110                         && (c2 == charArray[2])
2111                         && (c3 == charArray[3])
2112                         && (c4 == charArray[4]))
2113                         return charArray;
2114         }
2115         //--------add the entry-------
2116         if (++max >= InternalTableSize) max = 0;
2117         char[] r;
2118         table[max] = (r = new char[] {c0, c1, c2, c3, c4});
2119         newEntry5 = max;
2120         return r;
2121
2122 }
2123 final char[] optimizedCurrentTokenSource6() {
2124         //try to return the same char[] build only once
2125
2126         char c0, c1, c2, c3, c4, c5;
2127         long hash =
2128                 ((((long) (c0 = this.source[this.startPosition])) << 32)
2129                         + (((long) (c1 = this.source[this.startPosition + 1])) << 24)
2130                         + (((long) (c2 = this.source[this.startPosition + 2])) << 18)
2131                         + ((c3 = this.source[this.startPosition + 3]) << 12)
2132                         + ((c4 = this.source[this.startPosition + 4]) << 6)
2133                         + (c5 = this.source[this.startPosition + 5]))
2134                         % TableSize;
2135         char[][] table = this.charArray_length[4][(int) hash];
2136         int i = newEntry6;
2137         while (++i < InternalTableSize) {
2138                 char[] charArray = table[i];
2139                 if ((c0 == charArray[0])
2140                         && (c1 == charArray[1])
2141                         && (c2 == charArray[2])
2142                         && (c3 == charArray[3])
2143                         && (c4 == charArray[4])
2144                         && (c5 == charArray[5]))
2145                         return charArray;
2146         }
2147         //---------other side---------
2148         i = -1;
2149         int max = newEntry6;
2150         while (++i <= max) {
2151                 char[] charArray = table[i];
2152                 if ((c0 == charArray[0])
2153                         && (c1 == charArray[1])
2154                         && (c2 == charArray[2])
2155                         && (c3 == charArray[3])
2156                         && (c4 == charArray[4])
2157                         && (c5 == charArray[5]))
2158                         return charArray;
2159         }
2160         //--------add the entry-------
2161         if (++max >= InternalTableSize) max = 0;
2162         char[] r;
2163         table[max] = (r = new char[] {c0, c1, c2, c3, c4, c5});
2164         newEntry6 = max;
2165         return r;
2166 }
2167 private void parseTags(NLSLine line) {
2168         String s = new String(getCurrentTokenSource());
2169         int pos = s.indexOf(TAG_PREFIX);
2170         int lineLength = line.size();
2171         while (pos != -1) {
2172                 int start = pos + TAG_PREFIX_LENGTH;
2173                 int end = s.indexOf(TAG_POSTFIX, start);
2174                 if (end != -1) {
2175                         String index = s.substring(start, end);
2176                         int i = 0;
2177                         try {
2178                                 i = Integer.parseInt(index) - 1; // Tags are one based not zero based.
2179                         } catch (NumberFormatException e) {
2180                                 i = -1; // we don't want to consider this as a valid NLS tag
2181                         }
2182                         if (line.exists(i)) {
2183                                 line.set(i, null);
2184                         }
2185                 }
2186                 pos = s.indexOf(TAG_PREFIX, start);
2187         }
2188
2189         this.nonNLSStrings = new StringLiteral[lineLength];
2190         int nonNLSCounter = 0;
2191         for (Iterator iterator = line.iterator(); iterator.hasNext(); ) {
2192                 StringLiteral literal = (StringLiteral) iterator.next();
2193                 if (literal != null) {
2194                         this.nonNLSStrings[nonNLSCounter++] = literal;
2195                 }
2196         }
2197         if (nonNLSCounter == 0) {
2198                 this.nonNLSStrings = null;
2199                 this.currentLine = null;
2200                 return;
2201         }
2202         this.wasNonExternalizedStringLiteral = true;
2203         if (nonNLSCounter != lineLength) {
2204                 System.arraycopy(this.nonNLSStrings, 0, (this.nonNLSStrings = new StringLiteral[nonNLSCounter]), 0, nonNLSCounter);
2205         }
2206         this.currentLine = null;
2207 }
2208
2209 public final void pushLineSeparator() {
2210         //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2211         final int INCREMENT = 250;
2212
2213         if (this.checkNonExternalizedStringLiterals) {
2214         // reinitialize the current line for non externalize strings purpose
2215                 this.currentLine = null;
2216         }
2217         //currentCharacter is at position currentPosition-1
2218
2219         // cr 000D
2220         if (this.currentCharacter == '\r') {
2221                 int separatorPos = this.currentPosition - 1;
2222                 //TODO (olivier) david - why the following line was "if ((this.linePtr > 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;" ?
2223                 if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;
2224                 //System.out.println("CR-" + separatorPos);
2225                 int length = this.lineEnds.length;
2226                 if (++this.linePtr >=  length)
2227                         System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[length + INCREMENT], 0, length);
2228                 this.lineEnds[this.linePtr] = separatorPos;
2229                 // look-ahead for merged cr+lf
2230                 try {
2231                         if (this.source[this.currentPosition] == '\n') {
2232                                 //System.out.println("look-ahead LF-" + this.currentPosition);
2233                                 this.lineEnds[this.linePtr] = this.currentPosition;
2234                                 this.currentPosition++;
2235                                 this.wasAcr = false;
2236                         } else {
2237                                 this.wasAcr = true;
2238                         }
2239                 } catch(IndexOutOfBoundsException e) {
2240                         this.wasAcr = true;
2241                 }
2242         } else {
2243                 // lf 000A
2244                 if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf
2245                         if (this.wasAcr && (this.lineEnds[this.linePtr] == (this.currentPosition - 2))) {
2246                                 //System.out.println("merge LF-" + (this.currentPosition - 1));
2247                                 this.lineEnds[this.linePtr] = this.currentPosition - 1;
2248                         } else {
2249                                 int separatorPos = this.currentPosition - 1;
2250                                 //TODO (olivier) david - why the following line was "if ((this.linePtr > 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;" ?
2251                                 if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;
2252                                 // System.out.println("LF-" + separatorPos);
2253                                 int length = this.lineEnds.length;
2254                                 if (++this.linePtr >=  length)
2255                                         System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[length + INCREMENT], 0, length);
2256                                 this.lineEnds[this.linePtr] = separatorPos;
2257                         }
2258                         this.wasAcr = false;
2259                 }
2260         }
2261 }
2262 public final void pushUnicodeLineSeparator() {
2263         if (this.checkNonExternalizedStringLiterals) {
2264         // reinitialize the current line for non externalize strings purpose
2265                 this.currentLine = null;
2266         }
2267
2268         // cr 000D
2269         if (this.currentCharacter == '\r') {
2270                 if (this.source[this.currentPosition] == '\n') {
2271                         this.wasAcr = false;
2272                 } else {
2273                         this.wasAcr = true;
2274                 }
2275         } else {
2276                 // lf 000A
2277                 if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf
2278                         this.wasAcr = false;
2279                 }
2280         }
2281 }
2282 public void recordComment(int token) {
2283         // compute position
2284         int stopPosition = this.currentPosition;
2285         switch (token) {
2286                 case TokenNameCOMMENT_LINE:
2287                         stopPosition = -this.lastCommentLinePosition;
2288                         break;
2289                 case TokenNameCOMMENT_BLOCK:
2290                         stopPosition = -this.currentPosition;
2291                         break;
2292         }
2293
2294         // a new comment is recorded
2295         int length = this.commentStops.length;
2296         if (++this.commentPtr >=  length) {
2297                 System.arraycopy(this.commentStops, 0, this.commentStops = new int[length + 30], 0, length);
2298                 //grows the positions buffers too
2299                 System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[length + 30], 0, length);
2300         }
2301         this.commentStops[this.commentPtr] = stopPosition;
2302         this.commentStarts[this.commentPtr] = this.startPosition;
2303 }
2304
2305 /**
2306  * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position.
2307  * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>).
2308  *
2309  * @param begin the given start position
2310  * @param end the given end position
2311  */
2312 public void resetTo(int begin, int end) {
2313         //reset the scanner to a given position where it may rescan again
2314
2315         this.diet = false;
2316         this.initialPosition = this.startPosition = this.currentPosition = begin;
2317         this.eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
2318         this.commentPtr = -1; // reset comment stack
2319         this.foundTaskCount = 0;
2320
2321 //      // if resetTo is used with being > than end.
2322 //      if (begin > this.eofPosition) {
2323 //              begin = this.eofPosition;
2324 //      }
2325 }
2326
2327 public final void scanEscapeCharacter() throws InvalidInputException {
2328         // the string with "\\u" is a legal string of two chars \ and u
2329         //thus we use a direct access to the source (for regular cases).
2330
2331         if (this.unicodeAsBackSlash) {
2332                 // consume next character
2333                 this.unicodeAsBackSlash = false;
2334                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
2335                         getNextUnicodeChar();
2336                 } else {
2337                         if (this.withoutUnicodePtr != 0) {
2338                                 unicodeStoreAt(++this.withoutUnicodePtr);
2339                         }
2340                 }
2341         } else
2342                 this.currentCharacter = this.source[this.currentPosition++];
2343         switch (this.currentCharacter) {
2344                 case 'b' :
2345                         this.currentCharacter = '\b';
2346                         break;
2347                 case 't' :
2348                         this.currentCharacter = '\t';
2349                         break;
2350                 case 'n' :
2351                         this.currentCharacter = '\n';
2352                         break;
2353                 case 'f' :
2354                         this.currentCharacter = '\f';
2355                         break;
2356                 case 'r' :
2357                         this.currentCharacter = '\r';
2358                         break;
2359                 case '\"' :
2360                         this.currentCharacter = '\"';
2361                         break;
2362                 case '\'' :
2363                         this.currentCharacter = '\'';
2364                         break;
2365                 case '\\' :
2366                         this.currentCharacter = '\\';
2367                         break;
2368                 default :
2369                         // -----------octal escape--------------
2370                         // OctalDigit
2371                         // OctalDigit OctalDigit
2372                         // ZeroToThree OctalDigit OctalDigit
2373
2374                         int number = Character.getNumericValue(this.currentCharacter);
2375                         if (number >= 0 && number <= 7) {
2376                                 boolean zeroToThreeNot = number > 3;
2377                                 if (isDigit(this.currentCharacter = this.source[this.currentPosition++])) {
2378                                         int digit = Character.getNumericValue(this.currentCharacter);
2379                                         if (digit >= 0 && digit <= 7) {
2380                                                 number = (number * 8) + digit;
2381                                                 if (isDigit(this.currentCharacter = this.source[this.currentPosition++])) {
2382                                                         if (zeroToThreeNot) {// has read \NotZeroToThree OctalDigit Digit --> ignore last character
2383                                                                 this.currentPosition--;
2384                                                         } else {
2385                                                                 digit = Character.getNumericValue(this.currentCharacter);
2386                                                                 if (digit >= 0 && digit <= 7){ // has read \ZeroToThree OctalDigit OctalDigit
2387                                                                         number = (number * 8) + digit;
2388                                                                 } else {// has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
2389                                                                         this.currentPosition--;
2390                                                                 }
2391                                                         }
2392                                                 } else { // has read \OctalDigit NonDigit--> ignore last character
2393                                                         this.currentPosition--;
2394                                                 }
2395                                         } else { // has read \OctalDigit NonOctalDigit--> ignore last character
2396                                                 this.currentPosition--;
2397                                         }
2398                                 } else { // has read \OctalDigit --> ignore last character
2399                                         this.currentPosition--;
2400                                 }
2401                                 if (number > 255)
2402                                         throw new InvalidInputException(INVALID_ESCAPE);
2403                                 this.currentCharacter = (char) number;
2404                         } else
2405                                 throw new InvalidInputException(INVALID_ESCAPE);
2406         }
2407 }
2408 public int scanIdentifierOrKeyword() {
2409         //test keywords
2410
2411         //first dispatch on the first char.
2412         //then the length. If there are several
2413         //keywors with the same length AND the same first char, then do another
2414         //dispatch on the second char
2415         this.useAssertAsAnIndentifier = false;
2416         while (getNextCharAsJavaIdentifierPart()){/*empty*/}
2417
2418         int index, length;
2419         char[] data;
2420         char firstLetter;
2421         if (this.withoutUnicodePtr == 0)
2422
2423                 //quick test on length == 1 but not on length > 12 while most identifier
2424                 //have a length which is <= 12...but there are lots of identifier with
2425                 //only one char....
2426
2427                 {
2428                 if ((length = this.currentPosition - this.startPosition) == 1)
2429                         return TokenNameIdentifier;
2430                 data = this.source;
2431                 index = this.startPosition;
2432         } else {
2433                 if ((length = this.withoutUnicodePtr) == 1)
2434                         return TokenNameIdentifier;
2435                 data = this.withoutUnicodeBuffer;
2436                 index = 1;
2437         }
2438
2439         firstLetter = data[index];
2440         switch (firstLetter) {
2441
2442                 case 'a' :
2443                         switch(length) {
2444                                 case 8: //abstract
2445                                         if ((data[++index] == 'b')
2446                                                 && (data[++index] == 's')
2447                                                 && (data[++index] == 't')
2448                                                 && (data[++index] == 'r')
2449                                                 && (data[++index] == 'a')
2450                                                 && (data[++index] == 'c')
2451                                                 && (data[++index] == 't')) {
2452                                                         return TokenNameabstract;
2453                                                 } else {
2454                                                         return TokenNameIdentifier;
2455                                                 }
2456                                 case 6: // assert
2457                                         if ((data[++index] == 's')
2458                                                 && (data[++index] == 's')
2459                                                 && (data[++index] == 'e')
2460                                                 && (data[++index] == 'r')
2461                                                 && (data[++index] == 't')) {
2462                                                         if (this.assertMode) {
2463                                                                 this.containsAssertKeyword = true;
2464                                                                 return TokenNameassert;
2465                                                         } else {
2466                                                                 this.useAssertAsAnIndentifier = true;
2467                                                                 return TokenNameIdentifier;
2468                                                         }
2469                                                 } else {
2470                                                         return TokenNameIdentifier;
2471                                                 }
2472                                 default:
2473                                         return TokenNameIdentifier;
2474                         }
2475                 case 'b' : //boolean break byte
2476                         switch (length) {
2477                                 case 4 :
2478                                         if ((data[++index] == 'y') && (data[++index] == 't') && (data[++index] == 'e'))
2479                                                 return TokenNamebyte;
2480                                         else
2481                                                 return TokenNameIdentifier;
2482                                 case 5 :
2483                                         if ((data[++index] == 'r')
2484                                                 && (data[++index] == 'e')
2485                                                 && (data[++index] == 'a')
2486                                                 && (data[++index] == 'k'))
2487                                                 return TokenNamebreak;
2488                                         else
2489                                                 return TokenNameIdentifier;
2490                                 case 7 :
2491                                         if ((data[++index] == 'o')
2492                                                 && (data[++index] == 'o')
2493                                                 && (data[++index] == 'l')
2494                                                 && (data[++index] == 'e')
2495                                                 && (data[++index] == 'a')
2496                                                 && (data[++index] == 'n'))
2497                                                 return TokenNameboolean;
2498                                         else
2499                                                 return TokenNameIdentifier;
2500                                 default :
2501                                         return TokenNameIdentifier;
2502                         }
2503
2504                 case 'c' : //case char catch const class continue
2505                         switch (length) {
2506                                 case 4 :
2507                                         if (data[++index] == 'a')
2508                                                 if ((data[++index] == 's') && (data[++index] == 'e'))
2509                                                         return TokenNamecase;
2510                                                 else
2511                                                         return TokenNameIdentifier;
2512                                         else
2513                                                 if ((data[index] == 'h') && (data[++index] == 'a') && (data[++index] == 'r'))
2514                                                         return TokenNamechar;
2515                                                 else
2516                                                         return TokenNameIdentifier;
2517                                 case 5 :
2518                                         if (data[++index] == 'a')
2519                                                 if ((data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
2520                                                         return TokenNamecatch;
2521                                                 else
2522                                                         return TokenNameIdentifier;
2523                                         else
2524                                                 if (data[index] == 'l')
2525                                                         if ((data[++index] == 'a')
2526                                                                 && (data[++index] == 's')
2527                                                                 && (data[++index] == 's'))
2528                                                                 return TokenNameclass;
2529                                                         else
2530                                                                 return TokenNameIdentifier;
2531                                                 else if ((data[index] == 'o')
2532                                                         && (data[++index] == 'n')
2533                                                         && (data[++index] == 's')
2534                                                         && (data[++index] == 't'))
2535                                                         return TokenNameERROR; //const is not used in java ???????
2536                                                 else
2537                                                         return TokenNameIdentifier;
2538                                 case 8 :
2539                                         if ((data[++index] == 'o')
2540                                                 && (data[++index] == 'n')
2541                                                 && (data[++index] == 't')
2542                                                 && (data[++index] == 'i')
2543                                                 && (data[++index] == 'n')
2544                                                 && (data[++index] == 'u')
2545                                                 && (data[++index] == 'e'))
2546                                                 return TokenNamecontinue;
2547                                         else
2548                                                 return TokenNameIdentifier;
2549                                 default :
2550                                         return TokenNameIdentifier;
2551                         }
2552
2553                 case 'd' : //default do double
2554                         switch (length) {
2555                                 case 2 :
2556                                         if ((data[++index] == 'o'))
2557                                                 return TokenNamedo;
2558                                         else
2559                                                 return TokenNameIdentifier;
2560                                 case 6 :
2561                                         if ((data[++index] == 'o')
2562                                                 && (data[++index] == 'u')
2563                                                 && (data[++index] == 'b')
2564                                                 && (data[++index] == 'l')
2565                                                 && (data[++index] == 'e'))
2566                                                 return TokenNamedouble;
2567                                         else
2568                                                 return TokenNameIdentifier;
2569                                 case 7 :
2570                                         if ((data[++index] == 'e')
2571                                                 && (data[++index] == 'f')
2572                                                 && (data[++index] == 'a')
2573                                                 && (data[++index] == 'u')
2574                                                 && (data[++index] == 'l')
2575                                                 && (data[++index] == 't'))
2576                                                 return TokenNamedefault;
2577                                         else
2578                                                 return TokenNameIdentifier;
2579                                 default :
2580                                         return TokenNameIdentifier;
2581                         }
2582                 case 'e' : //else extends
2583                         switch (length) {
2584                                 case 4 :
2585                                         if ((data[++index] == 'l') && (data[++index] == 's') && (data[++index] == 'e'))
2586                                                 return TokenNameelse;
2587                                         else
2588                                                 return TokenNameIdentifier;
2589                                 case 7 :
2590                                         if ((data[++index] == 'x')
2591                                                 && (data[++index] == 't')
2592                                                 && (data[++index] == 'e')
2593                                                 && (data[++index] == 'n')
2594                                                 && (data[++index] == 'd')
2595                                                 && (data[++index] == 's'))
2596                                                 return TokenNameextends;
2597                                         else
2598                                                 return TokenNameIdentifier;
2599                                 default :
2600                                         return TokenNameIdentifier;
2601                         }
2602
2603                 case 'f' : //final finally float for false
2604                         switch (length) {
2605                                 case 3 :
2606                                         if ((data[++index] == 'o') && (data[++index] == 'r'))
2607                                                 return TokenNamefor;
2608                                         else
2609                                                 return TokenNameIdentifier;
2610                                 case 5 :
2611                                         if (data[++index] == 'i')
2612                                                 if ((data[++index] == 'n')
2613                                                         && (data[++index] == 'a')
2614                                                         && (data[++index] == 'l')) {
2615                                                         return TokenNamefinal;
2616                                                 } else
2617                                                         return TokenNameIdentifier;
2618                                         else
2619                                                 if (data[index] == 'l')
2620                                                         if ((data[++index] == 'o')
2621                                                                 && (data[++index] == 'a')
2622                                                                 && (data[++index] == 't'))
2623                                                                 return TokenNamefloat;
2624                                                         else
2625                                                                 return TokenNameIdentifier;
2626                                                 else
2627                                                         if ((data[index] == 'a')
2628                                                                 && (data[++index] == 'l')
2629                                                                 && (data[++index] == 's')
2630                                                                 && (data[++index] == 'e'))
2631                                                                 return TokenNamefalse;
2632                                                         else
2633                                                                 return TokenNameIdentifier;
2634                                 case 7 :
2635                                         if ((data[++index] == 'i')
2636                                                 && (data[++index] == 'n')
2637                                                 && (data[++index] == 'a')
2638                                                 && (data[++index] == 'l')
2639                                                 && (data[++index] == 'l')
2640                                                 && (data[++index] == 'y'))
2641                                                 return TokenNamefinally;
2642                                         else
2643                                                 return TokenNameIdentifier;
2644
2645                                 default :
2646                                         return TokenNameIdentifier;
2647                         }
2648                 case 'g' : //goto
2649                         if (length == 4) {
2650                                 if ((data[++index] == 'o')
2651                                         && (data[++index] == 't')
2652                                         && (data[++index] == 'o')) {
2653                                         return TokenNameERROR;
2654                                 }
2655                         } //no goto in java are allowed, so why java removes this keyword ???
2656                         return TokenNameIdentifier;
2657
2658                 case 'i' : //if implements import instanceof int interface
2659                         switch (length) {
2660                                 case 2 :
2661                                         if (data[++index] == 'f')
2662                                                 return TokenNameif;
2663                                         else
2664                                                 return TokenNameIdentifier;
2665                                 case 3 :
2666                                         if ((data[++index] == 'n') && (data[++index] == 't'))
2667                                                 return TokenNameint;
2668                                         else
2669                                                 return TokenNameIdentifier;
2670                                 case 6 :
2671                                         if ((data[++index] == 'm')
2672                                                 && (data[++index] == 'p')
2673                                                 && (data[++index] == 'o')
2674                                                 && (data[++index] == 'r')
2675                                                 && (data[++index] == 't'))
2676                                                 return TokenNameimport;
2677                                         else
2678                                                 return TokenNameIdentifier;
2679                                 case 9 :
2680                                         if ((data[++index] == 'n')
2681                                                 && (data[++index] == 't')
2682                                                 && (data[++index] == 'e')
2683                                                 && (data[++index] == 'r')
2684                                                 && (data[++index] == 'f')
2685                                                 && (data[++index] == 'a')
2686                                                 && (data[++index] == 'c')
2687                                                 && (data[++index] == 'e'))
2688                                                 return TokenNameinterface;
2689                                         else
2690                                                 return TokenNameIdentifier;
2691                                 case 10 :
2692                                         if (data[++index] == 'm')
2693                                                 if ((data[++index] == 'p')
2694                                                         && (data[++index] == 'l')
2695                                                         && (data[++index] == 'e')
2696                                                         && (data[++index] == 'm')
2697                                                         && (data[++index] == 'e')
2698                                                         && (data[++index] == 'n')
2699                                                         && (data[++index] == 't')
2700                                                         && (data[++index] == 's'))
2701                                                         return TokenNameimplements;
2702                                                 else
2703                                                         return TokenNameIdentifier;
2704                                         else
2705                                                 if ((data[index] == 'n')
2706                                                         && (data[++index] == 's')
2707                                                         && (data[++index] == 't')
2708                                                         && (data[++index] == 'a')
2709                                                         && (data[++index] == 'n')
2710                                                         && (data[++index] == 'c')
2711                                                         && (data[++index] == 'e')
2712                                                         && (data[++index] == 'o')
2713                                                         && (data[++index] == 'f'))
2714                                                         return TokenNameinstanceof;
2715                                                 else
2716                                                         return TokenNameIdentifier;
2717
2718                                 default :
2719                                         return TokenNameIdentifier;
2720                         }
2721
2722                 case 'l' : //long
2723                         if (length == 4) {
2724                                 if ((data[++index] == 'o')
2725                                         && (data[++index] == 'n')
2726                                         && (data[++index] == 'g')) {
2727                                         return TokenNamelong;
2728                                 }
2729                         }
2730                         return TokenNameIdentifier;
2731
2732                 case 'n' : //native new null
2733                         switch (length) {
2734                                 case 3 :
2735                                         if ((data[++index] == 'e') && (data[++index] == 'w'))
2736                                                 return TokenNamenew;
2737                                         else
2738                                                 return TokenNameIdentifier;
2739                                 case 4 :
2740                                         if ((data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 'l'))
2741                                                 return TokenNamenull;
2742                                         else
2743                                                 return TokenNameIdentifier;
2744                                 case 6 :
2745                                         if ((data[++index] == 'a')
2746                                                 && (data[++index] == 't')
2747                                                 && (data[++index] == 'i')
2748                                                 && (data[++index] == 'v')
2749                                                 && (data[++index] == 'e')) {
2750                                                 return TokenNamenative;
2751                                         } else
2752                                                 return TokenNameIdentifier;
2753                                 default :
2754                                         return TokenNameIdentifier;
2755                         }
2756
2757                 case 'p' : //package private protected public
2758                         switch (length) {
2759                                 case 6 :
2760                                         if ((data[++index] == 'u')
2761                                                 && (data[++index] == 'b')
2762                                                 && (data[++index] == 'l')
2763                                                 && (data[++index] == 'i')
2764                                                 && (data[++index] == 'c')) {
2765                                                 return TokenNamepublic;
2766                                         } else
2767                                                 return TokenNameIdentifier;
2768                                 case 7 :
2769                                         if (data[++index] == 'a')
2770                                                 if ((data[++index] == 'c')
2771                                                         && (data[++index] == 'k')
2772                                                         && (data[++index] == 'a')
2773                                                         && (data[++index] == 'g')
2774                                                         && (data[++index] == 'e'))
2775                                                         return TokenNamepackage;
2776                                                 else
2777                                                         return TokenNameIdentifier;
2778                                         else
2779                                                 if ((data[index] == 'r')
2780                                                         && (data[++index] == 'i')
2781                                                         && (data[++index] == 'v')
2782                                                         && (data[++index] == 'a')
2783                                                         && (data[++index] == 't')
2784                                                         && (data[++index] == 'e')) {
2785                                                         return TokenNameprivate;
2786                                                 } else
2787                                                         return TokenNameIdentifier;
2788                                 case 9 :
2789                                         if ((data[++index] == 'r')
2790                                                 && (data[++index] == 'o')
2791                                                 && (data[++index] == 't')
2792                                                 && (data[++index] == 'e')
2793                                                 && (data[++index] == 'c')
2794                                                 && (data[++index] == 't')
2795                                                 && (data[++index] == 'e')
2796                                                 && (data[++index] == 'd')) {
2797                                                 return TokenNameprotected;
2798                                         } else
2799                                                 return TokenNameIdentifier;
2800
2801                                 default :
2802                                         return TokenNameIdentifier;
2803                         }
2804
2805                 case 'r' : //return
2806                         if (length == 6) {
2807                                 if ((data[++index] == 'e')
2808                                         && (data[++index] == 't')
2809                                         && (data[++index] == 'u')
2810                                         && (data[++index] == 'r')
2811                                         && (data[++index] == 'n')) {
2812                                         return TokenNamereturn;
2813                                 }
2814                         }
2815                         return TokenNameIdentifier;
2816
2817                 case 's' : //short static super switch synchronized strictfp
2818                         switch (length) {
2819                                 case 5 :
2820                                         if (data[++index] == 'h')
2821                                                 if ((data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 't'))
2822                                                         return TokenNameshort;
2823                                                 else
2824                                                         return TokenNameIdentifier;
2825                                         else
2826                                                 if ((data[index] == 'u')
2827                                                         && (data[++index] == 'p')
2828                                                         && (data[++index] == 'e')
2829                                                         && (data[++index] == 'r'))
2830                                                         return TokenNamesuper;
2831                                                 else
2832                                                         return TokenNameIdentifier;
2833
2834                                 case 6 :
2835                                         if (data[++index] == 't')
2836                                                 if ((data[++index] == 'a')
2837                                                         && (data[++index] == 't')
2838                                                         && (data[++index] == 'i')
2839                                                         && (data[++index] == 'c')) {
2840                                                         return TokenNamestatic;
2841                                                 } else
2842                                                         return TokenNameIdentifier;
2843                                         else
2844                                                 if ((data[index] == 'w')
2845                                                         && (data[++index] == 'i')
2846                                                         && (data[++index] == 't')
2847                                                         && (data[++index] == 'c')
2848                                                         && (data[++index] == 'h'))
2849                                                         return TokenNameswitch;
2850                                                 else
2851                                                         return TokenNameIdentifier;
2852                                 case 8 :
2853                                         if ((data[++index] == 't')
2854                                                 && (data[++index] == 'r')
2855                                                 && (data[++index] == 'i')
2856                                                 && (data[++index] == 'c')
2857                                                 && (data[++index] == 't')
2858                                                 && (data[++index] == 'f')
2859                                                 && (data[++index] == 'p'))
2860                                                 return TokenNamestrictfp;
2861                                         else
2862                                                 return TokenNameIdentifier;
2863                                 case 12 :
2864                                         if ((data[++index] == 'y')
2865                                                 && (data[++index] == 'n')
2866                                                 && (data[++index] == 'c')
2867                                                 && (data[++index] == 'h')
2868                                                 && (data[++index] == 'r')
2869                                                 && (data[++index] == 'o')
2870                                                 && (data[++index] == 'n')
2871                                                 && (data[++index] == 'i')
2872                                                 && (data[++index] == 'z')
2873                                                 && (data[++index] == 'e')
2874                                                 && (data[++index] == 'd')) {
2875                                                 return TokenNamesynchronized;
2876                                         } else
2877                                                 return TokenNameIdentifier;
2878                                 default :
2879                                         return TokenNameIdentifier;
2880                         }
2881
2882                 case 't' : //try throw throws transient this true
2883                         switch (length) {
2884                                 case 3 :
2885                                         if ((data[++index] == 'r') && (data[++index] == 'y'))
2886                                                 return TokenNametry;
2887                                         else
2888                                                 return TokenNameIdentifier;
2889                                 case 4 :
2890                                         if (data[++index] == 'h')
2891                                                 if ((data[++index] == 'i') && (data[++index] == 's'))
2892                                                         return TokenNamethis;
2893                                                 else
2894                                                         return TokenNameIdentifier;
2895                                         else
2896                                                 if ((data[index] == 'r') && (data[++index] == 'u') && (data[++index] == 'e'))
2897                                                         return TokenNametrue;
2898                                                 else
2899                                                         return TokenNameIdentifier;
2900                                 case 5 :
2901                                         if ((data[++index] == 'h')
2902                                                 && (data[++index] == 'r')
2903                                                 && (data[++index] == 'o')
2904                                                 && (data[++index] == 'w'))
2905                                                 return TokenNamethrow;
2906                                         else
2907                                                 return TokenNameIdentifier;
2908                                 case 6 :
2909                                         if ((data[++index] == 'h')
2910                                                 && (data[++index] == 'r')
2911                                                 && (data[++index] == 'o')
2912                                                 && (data[++index] == 'w')
2913                                                 && (data[++index] == 's'))
2914                                                 return TokenNamethrows;
2915                                         else
2916                                                 return TokenNameIdentifier;
2917                                 case 9 :
2918                                         if ((data[++index] == 'r')
2919                                                 && (data[++index] == 'a')
2920                                                 && (data[++index] == 'n')
2921                                                 && (data[++index] == 's')
2922                                                 && (data[++index] == 'i')
2923                                                 && (data[++index] == 'e')
2924                                                 && (data[++index] == 'n')
2925                                                 && (data[++index] == 't')) {
2926                                                 return TokenNametransient;
2927                                         } else
2928                                                 return TokenNameIdentifier;
2929
2930                                 default :
2931                                         return TokenNameIdentifier;
2932                         }
2933
2934                 case 'v' : //void volatile
2935                         switch (length) {
2936                                 case 4 :
2937                                         if ((data[++index] == 'o') && (data[++index] == 'i') && (data[++index] == 'd'))
2938                                                 return TokenNamevoid;
2939                                         else
2940                                                 return TokenNameIdentifier;
2941                                 case 8 :
2942                                         if ((data[++index] == 'o')
2943                                                 && (data[++index] == 'l')
2944                                                 && (data[++index] == 'a')
2945                                                 && (data[++index] == 't')
2946                                                 && (data[++index] == 'i')
2947                                                 && (data[++index] == 'l')
2948                                                 && (data[++index] == 'e')) {
2949                                                 return TokenNamevolatile;
2950                                         } else
2951                                                 return TokenNameIdentifier;
2952
2953                                 default :
2954                                         return TokenNameIdentifier;
2955                         }
2956
2957                 case 'w' : //while widefp
2958                         switch (length) {
2959                                 case 5 :
2960                                         if ((data[++index] == 'h')
2961                                                 && (data[++index] == 'i')
2962                                                 && (data[++index] == 'l')
2963                                                 && (data[++index] == 'e'))
2964                                                 return TokenNamewhile;
2965                                         else
2966                                                 return TokenNameIdentifier;
2967                                         //case 6:if ( (data[++index] =='i') && (data[++index]=='d') && (data[++index]=='e') && (data[++index]=='f')&& (data[++index]=='p'))
2968                                         //return TokenNamewidefp ;
2969                                         //else
2970                                         //return TokenNameIdentifier;
2971                                 default :
2972                                         return TokenNameIdentifier;
2973                         }
2974
2975                 default :
2976                         return TokenNameIdentifier;
2977         }
2978 }
2979 public int scanNumber(boolean dotPrefix) throws InvalidInputException {
2980
2981         //when entering this method the currentCharacter is the first
2982         //digit of the number. It may be preceeded by a '.' when
2983         //dotPrefix is true
2984
2985         boolean floating = dotPrefix;
2986         if ((!dotPrefix) && (this.currentCharacter == '0')) {
2987                 if (getNextChar('x', 'X') >= 0) { //----------hexa-----------------
2988                         //force the first char of the hexa number do exist...
2989                         // consume next character
2990                         this.unicodeAsBackSlash = false;
2991                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2992                                 && (this.source[this.currentPosition] == 'u')) {
2993                                 getNextUnicodeChar();
2994                         } else {
2995                                 if (this.withoutUnicodePtr != 0) {
2996                                         unicodeStoreAt(++this.withoutUnicodePtr);
2997                                 }
2998                         }
2999                         if (Character.digit(this.currentCharacter, 16) == -1)
3000                                 throw new InvalidInputException(INVALID_HEXA);
3001                         //---end forcing--
3002                         while (getNextCharAsDigit(16)){/*empty*/}
3003                         if (getNextChar('l', 'L') >= 0)
3004                                 return TokenNameLongLiteral;
3005                         else
3006                                 return TokenNameIntegerLiteral;
3007                 }
3008
3009                 //there is x or X in the number
3010                 //potential octal ! ... some one may write 000099.0 ! thus 00100 < 00078.0 is true !!!!! crazy language
3011                 if (getNextCharAsDigit()) { //-------------potential octal-----------------
3012                         while (getNextCharAsDigit()){/*empty*/}
3013
3014                         if (getNextChar('l', 'L') >= 0) {
3015                                 return TokenNameLongLiteral;
3016                         }
3017
3018                         if (getNextChar('f', 'F') >= 0) {
3019                                 return TokenNameFloatingPointLiteral;
3020                         }
3021
3022                         if (getNextChar('d', 'D') >= 0) {
3023                                 return TokenNameDoubleLiteral;
3024                         } else { //make the distinction between octal and float ....
3025                                 boolean isInteger = true;
3026                                 if (getNextChar('.')) {
3027                                         isInteger = false;
3028                                         while (getNextCharAsDigit()){/*empty*/}
3029                                 }
3030                                 if (getNextChar('e', 'E') >= 0) { // consume next character
3031                                         isInteger = false;
3032                                         this.unicodeAsBackSlash = false;
3033                                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3034                                                 && (this.source[this.currentPosition] == 'u')) {
3035                                                 getNextUnicodeChar();
3036                                         } else {
3037                                                 if (this.withoutUnicodePtr != 0) {
3038                                                         unicodeStoreAt(++this.withoutUnicodePtr);
3039                                                 }
3040                                         }
3041
3042                                         if ((this.currentCharacter == '-')
3043                                                 || (this.currentCharacter == '+')) { // consume next character
3044                                                 this.unicodeAsBackSlash = false;
3045                                                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3046                                                         && (this.source[this.currentPosition] == 'u')) {
3047                                                         getNextUnicodeChar();
3048                                                 } else {
3049                                                         if (this.withoutUnicodePtr != 0) {
3050                                                                 unicodeStoreAt(++this.withoutUnicodePtr);
3051                                                         }
3052                                                 }
3053                                         }
3054                                         if (!isDigit(this.currentCharacter))
3055                                                 throw new InvalidInputException(INVALID_FLOAT);
3056                                         while (getNextCharAsDigit()){/*empty*/}
3057                                 }
3058                                 if (getNextChar('f', 'F') >= 0)
3059                                         return TokenNameFloatingPointLiteral;
3060                                 if (getNextChar('d', 'D') >= 0 || !isInteger)
3061                                         return TokenNameDoubleLiteral;
3062                                 return TokenNameIntegerLiteral;
3063                         }
3064                 } else {
3065                         /* carry on */
3066                 }
3067         }
3068
3069         while (getNextCharAsDigit()){/*empty*/}
3070
3071         if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3072                 return TokenNameLongLiteral;
3073
3074         if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty
3075                 while (getNextCharAsDigit()){/*empty*/}
3076                 floating = true;
3077         }
3078
3079         //if floating is true both exponant and suffix may be optional
3080
3081         if (getNextChar('e', 'E') >= 0) {
3082                 floating = true;
3083                 // consume next character
3084                 this.unicodeAsBackSlash = false;
3085                 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3086                         && (this.source[this.currentPosition] == 'u')) {
3087                         getNextUnicodeChar();
3088                 } else {
3089                         if (this.withoutUnicodePtr != 0) {
3090                                 unicodeStoreAt(++this.withoutUnicodePtr);
3091                         }
3092                 }
3093
3094                 if ((this.currentCharacter == '-')
3095                         || (this.currentCharacter == '+')) { // consume next character
3096                         this.unicodeAsBackSlash = false;
3097                         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
3098                                 && (this.source[this.currentPosition] == 'u')) {
3099                                 getNextUnicodeChar();
3100                         } else {
3101                                 if (this.withoutUnicodePtr != 0) {
3102                                         unicodeStoreAt(++this.withoutUnicodePtr);
3103                                 }
3104                         }
3105                 }
3106                 if (!isDigit(this.currentCharacter))
3107                         throw new InvalidInputException(INVALID_FLOAT);
3108                 while (getNextCharAsDigit()){/*empty*/}
3109         }
3110
3111         if (getNextChar('d', 'D') >= 0)
3112                 return TokenNameDoubleLiteral;
3113         if (getNextChar('f', 'F') >= 0)
3114                 return TokenNameFloatingPointLiteral;
3115
3116         //the long flag has been tested before
3117
3118         return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral;
3119 }
3120 /**
3121  * Search the line number corresponding to a specific position
3122  * @param position int
3123  * @return int
3124  */
3125 public final int getLineNumber(int position) {
3126
3127         if (this.lineEnds == null)
3128                 return 1;
3129         int length = this.linePtr+1;
3130         if (length == 0)
3131                 return 1;
3132         int g = 0, d = length - 1;
3133         int m = 0;
3134         while (g <= d) {
3135                 m = (g + d) /2;
3136                 if (position < this.lineEnds[m]) {
3137                         d = m-1;
3138                 } else if (position > this.lineEnds[m]) {
3139                         g = m+1;
3140                 } else {
3141                         return m + 1;
3142                 }
3143         }
3144         if (position < this.lineEnds[m]) {
3145                 return m+1;
3146         }
3147         return m+2;
3148 }
3149 public final void setSource(char[] sourceString){
3150         //the source-buffer is set to sourceString
3151
3152         int sourceLength;
3153         if (sourceString == null) {
3154                 this.source = CharOperation.NO_CHAR;
3155                 sourceLength = 0;
3156         } else {
3157                 this.source = sourceString;
3158                 sourceLength = sourceString.length;
3159         }
3160         this.startPosition = -1;
3161         this.eofPosition = sourceLength;
3162         this.initialPosition = this.currentPosition = 0;
3163         this.containsAssertKeyword = false;
3164 }
3165
3166 public String toString() {
3167         if (this.startPosition == this.source.length)
3168                 return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
3169         if (this.currentPosition > this.source.length)
3170                 return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
3171
3172         char front[] = new char[this.startPosition];
3173         System.arraycopy(this.source, 0, front, 0, this.startPosition);
3174
3175         int middleLength = (this.currentPosition - 1) - this.startPosition + 1;
3176         char middle[];
3177         if (middleLength > -1) {
3178                 middle = new char[middleLength];
3179                 System.arraycopy(
3180                         this.source,
3181                         this.startPosition,
3182                         middle,
3183                         0,
3184                         middleLength);
3185         } else {
3186                 middle = CharOperation.NO_CHAR;
3187         }
3188
3189         char end[] = new char[this.source.length - (this.currentPosition - 1)];
3190         System.arraycopy(
3191                 this.source,
3192                 (this.currentPosition - 1) + 1,
3193                 end,
3194                 0,
3195                 this.source.length - (this.currentPosition - 1) - 1);
3196
3197         return new String(front)
3198                 + "\n===============================\nStarts here -->" //$NON-NLS-1$
3199                 + new String(middle)
3200                 + "<-- Ends here\n===============================\n" //$NON-NLS-1$
3201                 + new String(end);
3202 }
3203 public final String toStringAction(int act) {
3204         switch (act) {
3205                 case TokenNameIdentifier :
3206                         return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3207                 case TokenNameabstract :
3208                         return "abstract"; //$NON-NLS-1$
3209                 case TokenNameboolean :
3210                         return "boolean"; //$NON-NLS-1$
3211                 case TokenNamebreak :
3212                         return "break"; //$NON-NLS-1$
3213                 case TokenNamebyte :
3214                         return "byte"; //$NON-NLS-1$
3215                 case TokenNamecase :
3216                         return "case"; //$NON-NLS-1$
3217                 case TokenNamecatch :
3218                         return "catch"; //$NON-NLS-1$
3219                 case TokenNamechar :
3220                         return "char"; //$NON-NLS-1$
3221                 case TokenNameclass :
3222                         return "class"; //$NON-NLS-1$
3223                 case TokenNamecontinue :
3224                         return "continue"; //$NON-NLS-1$
3225                 case TokenNamedefault :
3226                         return "default"; //$NON-NLS-1$
3227                 case TokenNamedo :
3228                         return "do"; //$NON-NLS-1$
3229                 case TokenNamedouble :
3230                         return "double"; //$NON-NLS-1$
3231                 case TokenNameelse :
3232                         return "else"; //$NON-NLS-1$
3233                 case TokenNameextends :
3234                         return "extends"; //$NON-NLS-1$
3235                 case TokenNamefalse :
3236                         return "false"; //$NON-NLS-1$
3237                 case TokenNamefinal :
3238                         return "final"; //$NON-NLS-1$
3239                 case TokenNamefinally :
3240                         return "finally"; //$NON-NLS-1$
3241                 case TokenNamefloat :
3242                         return "float"; //$NON-NLS-1$
3243                 case TokenNamefor :
3244                         return "for"; //$NON-NLS-1$
3245                 case TokenNameif :
3246                         return "if"; //$NON-NLS-1$
3247                 case TokenNameimplements :
3248                         return "implements"; //$NON-NLS-1$
3249                 case TokenNameimport :
3250                         return "import"; //$NON-NLS-1$
3251                 case TokenNameinstanceof :
3252                         return "instanceof"; //$NON-NLS-1$
3253                 case TokenNameint :
3254                         return "int"; //$NON-NLS-1$
3255                 case TokenNameinterface :
3256                         return "interface"; //$NON-NLS-1$
3257                 case TokenNamelong :
3258                         return "long"; //$NON-NLS-1$
3259                 case TokenNamenative :
3260                         return "native"; //$NON-NLS-1$
3261                 case TokenNamenew :
3262                         return "new"; //$NON-NLS-1$
3263                 case TokenNamenull :
3264                         return "null"; //$NON-NLS-1$
3265                 case TokenNamepackage :
3266                         return "package"; //$NON-NLS-1$
3267                 case TokenNameprivate :
3268                         return "private"; //$NON-NLS-1$
3269                 case TokenNameprotected :
3270                         return "protected"; //$NON-NLS-1$
3271                 case TokenNamepublic :
3272                         return "public"; //$NON-NLS-1$
3273                 case TokenNamereturn :
3274                         return "return"; //$NON-NLS-1$
3275                 case TokenNameshort :
3276                         return "short"; //$NON-NLS-1$
3277                 case TokenNamestatic :
3278                         return "static"; //$NON-NLS-1$
3279                 case TokenNamesuper :
3280                         return "super"; //$NON-NLS-1$
3281                 case TokenNameswitch :
3282                         return "switch"; //$NON-NLS-1$
3283                 case TokenNamesynchronized :
3284                         return "synchronized"; //$NON-NLS-1$
3285                 case TokenNamethis :
3286                         return "this"; //$NON-NLS-1$
3287                 case TokenNamethrow :
3288                         return "throw"; //$NON-NLS-1$
3289                 case TokenNamethrows :
3290                         return "throws"; //$NON-NLS-1$
3291                 case TokenNametransient :
3292                         return "transient"; //$NON-NLS-1$
3293                 case TokenNametrue :
3294                         return "true"; //$NON-NLS-1$
3295                 case TokenNametry :
3296                         return "try"; //$NON-NLS-1$
3297                 case TokenNamevoid :
3298                         return "void"; //$NON-NLS-1$
3299                 case TokenNamevolatile :
3300                         return "volatile"; //$NON-NLS-1$
3301                 case TokenNamewhile :
3302                         return "while"; //$NON-NLS-1$
3303
3304                 case TokenNameIntegerLiteral :
3305                         return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3306                 case TokenNameLongLiteral :
3307                         return "Long(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3308                 case TokenNameFloatingPointLiteral :
3309                         return "Float(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3310                 case TokenNameDoubleLiteral :
3311                         return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3312                 case TokenNameCharacterLiteral :
3313                         return "Char(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3314                 case TokenNameStringLiteral :
3315                         return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3316
3317                 case TokenNamePLUS_PLUS :
3318                         return "++"; //$NON-NLS-1$
3319                 case TokenNameMINUS_MINUS :
3320                         return "--"; //$NON-NLS-1$
3321                 case TokenNameEQUAL_EQUAL :
3322                         return "=="; //$NON-NLS-1$
3323                 case TokenNameLESS_EQUAL :
3324                         return "<="; //$NON-NLS-1$
3325                 case TokenNameGREATER_EQUAL :
3326                         return ">="; //$NON-NLS-1$
3327                 case TokenNameNOT_EQUAL :
3328                         return "!="; //$NON-NLS-1$
3329                 case TokenNameLEFT_SHIFT :
3330                         return "<<"; //$NON-NLS-1$
3331                 case TokenNameRIGHT_SHIFT :
3332                         return ">>"; //$NON-NLS-1$
3333                 case TokenNameUNSIGNED_RIGHT_SHIFT :
3334                         return ">>>"; //$NON-NLS-1$
3335                 case TokenNamePLUS_EQUAL :
3336                         return "+="; //$NON-NLS-1$
3337                 case TokenNameMINUS_EQUAL :
3338                         return "-="; //$NON-NLS-1$
3339                 case TokenNameMULTIPLY_EQUAL :
3340                         return "*="; //$NON-NLS-1$
3341                 case TokenNameDIVIDE_EQUAL :
3342                         return "/="; //$NON-NLS-1$
3343                 case TokenNameAND_EQUAL :
3344                         return "&="; //$NON-NLS-1$
3345                 case TokenNameOR_EQUAL :
3346                         return "|="; //$NON-NLS-1$
3347                 case TokenNameXOR_EQUAL :
3348                         return "^="; //$NON-NLS-1$
3349                 case TokenNameREMAINDER_EQUAL :
3350                         return "%="; //$NON-NLS-1$
3351                 case TokenNameLEFT_SHIFT_EQUAL :
3352                         return "<<="; //$NON-NLS-1$
3353                 case TokenNameRIGHT_SHIFT_EQUAL :
3354                         return ">>="; //$NON-NLS-1$
3355                 case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL :
3356                         return ">>>="; //$NON-NLS-1$
3357                 case TokenNameOR_OR :
3358                         return "||"; //$NON-NLS-1$
3359                 case TokenNameAND_AND :
3360                         return "&&"; //$NON-NLS-1$
3361                 case TokenNamePLUS :
3362                         return "+"; //$NON-NLS-1$
3363                 case TokenNameMINUS :
3364                         return "-"; //$NON-NLS-1$
3365                 case TokenNameNOT :
3366                         return "!"; //$NON-NLS-1$
3367                 case TokenNameREMAINDER :
3368                         return "%"; //$NON-NLS-1$
3369                 case TokenNameXOR :
3370                         return "^"; //$NON-NLS-1$
3371                 case TokenNameAND :
3372                         return "&"; //$NON-NLS-1$
3373                 case TokenNameMULTIPLY :
3374                         return "*"; //$NON-NLS-1$
3375                 case TokenNameOR :
3376                         return "|"; //$NON-NLS-1$
3377                 case TokenNameTWIDDLE :
3378                         return "~"; //$NON-NLS-1$
3379                 case TokenNameDIVIDE :
3380                         return "/"; //$NON-NLS-1$
3381                 case TokenNameGREATER :
3382                         return ">"; //$NON-NLS-1$
3383                 case TokenNameLESS :
3384                         return "<"; //$NON-NLS-1$
3385                 case TokenNameLPAREN :
3386                         return "("; //$NON-NLS-1$
3387                 case TokenNameRPAREN :
3388                         return ")"; //$NON-NLS-1$
3389                 case TokenNameLBRACE :
3390                         return "{"; //$NON-NLS-1$
3391                 case TokenNameRBRACE :
3392                         return "}"; //$NON-NLS-1$
3393                 case TokenNameLBRACKET :
3394                         return "["; //$NON-NLS-1$
3395                 case TokenNameRBRACKET :
3396                         return "]"; //$NON-NLS-1$
3397                 case TokenNameSEMICOLON :
3398                         return ";"; //$NON-NLS-1$
3399                 case TokenNameQUESTION :
3400                         return "?"; //$NON-NLS-1$
3401                 case TokenNameCOLON :
3402                         return ":"; //$NON-NLS-1$
3403                 case TokenNameCOMMA :
3404                         return ","; //$NON-NLS-1$
3405                 case TokenNameDOT :
3406                         return "."; //$NON-NLS-1$
3407                 case TokenNameEQUAL :
3408                         return "="; //$NON-NLS-1$
3409                 case TokenNameEOF :
3410                         return "EOF"; //$NON-NLS-1$
3411                 default :
3412                         return "not-a-token"; //$NON-NLS-1$
3413         }
3414 }
3415 public void unicodeInitializeBuffer(int length) {
3416         this.withoutUnicodePtr = length;
3417     if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[length+(1+10)];
3418     int bLength = this.withoutUnicodeBuffer.length;
3419     if (1+length >= bLength) {
3420         System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length + (1+10)], 0, bLength);
3421     }
3422         System.arraycopy(this.source, this.startPosition, this.withoutUnicodeBuffer, 1, length);
3423 }
3424 public void unicodeStoreAt(int pos) {
3425     if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[10];
3426     int length = this.withoutUnicodeBuffer.length;
3427     if (pos == length) {
3428         System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length * 2], 0, length);
3429     }
3430         this.withoutUnicodeBuffer[pos] = this.currentCharacter;
3431 }
3432 }