src/org/eclipse/jdt/internal/compiler/parser/AbstractCommentParser.java

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2004 IBM Corporation and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v10.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  *******************************************************************************/
  11 package org.eclipse.jdt.internal.compiler.parser;
  12
  13 import java.util.ArrayList;
  14 import java.util.List;
  15
  16 import org.eclipse.jdt.core.compiler.CharOperation;
  17 import org.eclipse.jdt.core.compiler.InvalidInputException;
  18 import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
  19
  20 /**
  21  * Parser specialized for decoding javadoc comments
  22  */
  23 public abstract class AbstractCommentParser {
  24
  25         // recognized tags
  26         public static final char[] TAG_DEPRECATED = "deprecated".toCharArray(); //$NON-NLS-1$
  27         public static final char[] TAG_PARAM = "param".toCharArray(); //$NON-NLS-1$
  28         public static final char[] TAG_RETURN = "return".toCharArray(); //$NON-NLS-1$
  29         public static final char[] TAG_THROWS = "throws".toCharArray(); //$NON-NLS-1$
  30         public static final char[] TAG_EXCEPTION = "exception".toCharArray(); //$NON-NLS-1$
  31         public static final char[] TAG_SEE = "see".toCharArray(); //$NON-NLS-1$
  32         public static final char[] TAG_LINK = "link".toCharArray(); //$NON-NLS-1$
  33         public static final char[] TAG_LINKPLAIN = "linkplain".toCharArray(); //$NON-NLS-1$
  34         public static final char[] TAG_INHERITDOC = "inheritDoc".toCharArray(); //$NON-NLS-1$
  35
  36         // tags expected positions
  37         public final static int ORDERED_TAGS_NUMBER = 3;
  38         public final static int PARAM_TAG_EXPECTED_ORDER = 0;
  39         public final static int THROWS_TAG_EXPECTED_ORDER = 1;
  40         public final static int SEE_TAG_EXPECTED_ORDER = 2;
  41
  42         // Kind of comment parser
  43         public final static int COMPIL_PARSER = 0x00000001;
  44         public final static int DOM_PARSER = 0x00000002;
  45
  46         // Public fields
  47         public Scanner scanner;
  48         public boolean checkDocComment = false;
  49
  50         // Protected fields
  51         protected boolean inherited, deprecated;
  52         protected char[] source;
  53         protected int index, endComment, lineEnd;
  54         protected int tokenPreviousPosition, lastIdentifierEndPosition, starPosition;
  55         protected int textStart, memberStart;
  56         protected int tagSourceStart, tagSourceEnd;
  57         protected int inlineTagStart;
  58         protected Parser sourceParser;
  59         protected Object returnStatement;
  60         protected boolean lineStarted = false, inlineTagStarted = false;
  61         protected int kind;
  62         protected int[] lineEnds;
  63
  64         // Private fields
  65         private int currentTokenType = -1;
  66
  67         // Line pointers
  68         private int linePtr, lastLinePtr;
  69
  70         // Identifier stack
  71         protected int identifierPtr;
  72         protected char[][] identifierStack;
  73         protected int identifierLengthPtr;
  74         protected int[] identifierLengthStack;
  75         protected long[] identifierPositionStack;
  76         // Ast stack
  77         protected static int AstStackIncrement = 10;
  78         protected int astPtr;
  79         protected Object[] astStack;
  80         protected int astLengthPtr;
  81         protected int[] astLengthStack;
  82
  83         protected AbstractCommentParser(Parser sourceParser) {
  84                 this.sourceParser = sourceParser;
  85                 this.scanner = new Scanner(false, false, false, ClassFileConstants.JDK1_3, null, null, true/*taskCaseSensitive*/);
  86                 this.identifierStack = new char[20][];
  87                 this.identifierPositionStack = new long[20];
  88                 this.identifierLengthStack = new int[10];
  89                 this.astStack = new Object[30];
  90                 this.astLengthStack = new int[20];
  91         }
  92
  93         /* (non-Javadoc)
  94          * Returns true if tag @deprecated is present in javadoc comment.
  95          *
  96          * If javadoc checking is enabled, will also construct an Javadoc node, which will be stored into Parser.javadoc
  97          * slot for being consumed later on.
  98          */
  99         protected boolean parseComment(int javadocStart, int javadocEnd) {
 100
 101                 boolean validComment = true;
 102                 try {
 103                         // Init scanner position
 104                         this.scanner.resetTo(javadocStart, javadocEnd);
 105                         this.endComment = javadocEnd;
 106                         this.index = javadocStart;
 107                         readChar(); // starting '/'
 108                         int previousPosition = this.index;
 109                         readChar(); // first '*'
 110                         char nextCharacter= readChar(); // second '*'
 111
 112                         // Init local variables
 113                         this.astLengthPtr = -1;
 114                         this.astPtr = -1;
 115                         this.currentTokenType = -1;
 116                         this.inlineTagStarted = false;
 117                         this.inlineTagStart = -1;
 118                         this.lineStarted = false;
 119                         this.returnStatement = null;
 120                         this.inherited = false;
 121                         this.deprecated = false;
 122                         this.linePtr = getLineNumber(javadocStart);
 123                         this.lastLinePtr = getLineNumber(javadocEnd);
 124                         this.lineEnd = (this.linePtr == this.lastLinePtr) ? this.endComment : this.scanner.getLineEnd(this.linePtr);
 125                         this.textStart = -1;
 126                         char previousChar = 0;
 127                         int invalidTagLineEnd = -1;
 128                         int invalidInlineTagLineEnd = -1;
 129
 130                         // Loop on each comment character
 131                         while (this.index < this.endComment) {
 132                                 previousPosition = this.index;
 133                                 previousChar = nextCharacter;
 134
 135                                 // Calculate line end (cannot use this.scanner.linePtr as scanner does not parse line ends again)
 136                                 if (this.index > (this.lineEnd+1)) {
 137                                         updateLineEnd();
 138                                 }
 139
 140                                 // Read next char only if token was consumed
 141                                 if (this.currentTokenType < 0) {
 142                                         nextCharacter = readChar(); // consider unicodes
 143                                 } else {
 144                                         previousPosition = this.scanner.getCurrentTokenStartPosition();
 145                                         switch (this.currentTokenType) {
 146                                                 case TerminalTokens.TokenNameRBRACE:
 147                                                         nextCharacter = '}';
 148                                                         break;
 149                                                 case TerminalTokens.TokenNameMULTIPLY:
 150                                                         nextCharacter = '*';
 151                                                         break;
 152                                         default:
 153                                                         nextCharacter = this.scanner.currentCharacter;
 154                                         }
 155                                         consumeToken();
 156                                 }
 157
 158                                 if (this.index >= this.endComment) {
 159                                         break;
 160                                 }
 161
 162                                 switch (nextCharacter) {
 163                                         case '@' :
 164                                                 boolean valid = false;
 165                                                 // Start tag parsing only if we have a java identifier start character and if we are on line beginning or at inline tag beginning
 166                                                 if ((!this.lineStarted || previousChar == '{')) {
 167                                                         this.lineStarted = true;
 168                                                         if (this.inlineTagStarted) {
 169                                                                 this.inlineTagStarted = false;
 170                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
 171                                                                 // Cannot have @ inside inline comment
 172                                                                 if (this.sourceParser != null) {
 173                                                                         int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
 174                                                                         this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
 175                                                                 }
 176                                                                 validComment = false;
 177                                                                 if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 178                                                                         pushText(this.textStart, previousPosition);
 179                                                                 }
 180                                                                 if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
 181                                                         }
 182                                                         if (previousChar == '{') {
 183                                                                 if (this.textStart != -1 && this.textStart < this.inlineTagStart) {
 184                                                                         pushText(this.textStart, this.inlineTagStart);
 185                                                                 }
 186                                                                 this.inlineTagStarted = true;
 187                                                                 invalidInlineTagLineEnd = this.lineEnd;
 188                                                         } else if (this.textStart != -1 && this.textStart < invalidTagLineEnd) {
 189                                                                 pushText(this.textStart, invalidTagLineEnd);
 190                                                         }
 191                                                         this.scanner.resetTo(this.index, this.endComment);
 192                                                         this.currentTokenType = -1; // flush token cache at line begin
 193                                                         try {
 194                                                                 int token = readTokenAndConsume();
 195                                                                 this.tagSourceStart = this.scanner.getCurrentTokenStartPosition();
 196                                                                 this.tagSourceEnd = this.scanner.getCurrentTokenEndPosition();
 197                                                                 char[] tag = this.scanner.getCurrentIdentifierSource(); // first token is either an identifier or a keyword
 198                                                                 if (this.kind == DOM_PARSER) {
 199                                                                         // For DOM parser, try to get tag name other than java identifier
 200                                                                         // (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51660)
 201                                                                         int tk = token;
 202                                                                         int le = this.lineEnd;
 203                                                                         char pc = peekChar();
 204                                                                         tagNameToken: while (tk != TerminalTokens.TokenNameEOF) {
 205                                                                                 this.tagSourceEnd = this.scanner.getCurrentTokenEndPosition();
 206                                                                                 token = tk;
 207                                                                                 // !, ", #, %, &, ', -, :, <, >, * chars and spaces are not allowed in tag names
 208                                                                                 switch (pc) {
 209                                                                                         case '}':
 210                                                                                         case '!':
 211                                                                                         case '#':
 212                                                                                         case '%':
 213                                                                                         case '&':
 214                                                                                         case '\'':
 215                                                                                         case ':':
 216                                                                                         // case '-': allowed in tag names as this character is often used in doclets (bug 68087)
 217                                                                                         case '<':
 218                                                                                         case '>':
 219                                                                                         case '*': // break for '*' as this is perhaps the end of comment (bug 65288)
 220                                                                                                 break tagNameToken;
 221                                                                                         default:
 222                                                                                                 if (pc == ' ' || Character.isWhitespace(pc)) break tagNameToken;
 223                                                                                 }
 224                                                                                 tk = readTokenAndConsume();
 225                                                                                 pc = peekChar();
 226                                                                         }
 227                                                                         int length = this.tagSourceEnd-this.tagSourceStart+1;
 228                                                                         tag = new char[length];
 229                                                                         System.arraycopy(this.source, this.tagSourceStart, tag, 0, length);
 230                                                                         this.index = this.tagSourceEnd+1;
 231                                                                         this.scanner.currentPosition = this.tagSourceEnd+1;
 232                                                                         this.tagSourceStart = previousPosition;
 233                                                                         this.lineEnd = le;
 234                                                                 }
 235                                                                 switch (token) {
 236                                                                         case TerminalTokens.TokenNameIdentifier :
 237                                                                                 if (CharOperation.equals(tag, TAG_DEPRECATED)) {
 238                                                                                         this.deprecated = true;
 239                                                                                         if (this.kind == DOM_PARSER) {
 240                                                                                                 valid = parseTag();
 241                                                                                         } else {
 242                                                                                                 valid = true;
 243                                                                                         }
 244                                                                                 } else if (CharOperation.equals(tag, TAG_INHERITDOC)) {
 245                                                                                         // inhibits inherited flag when tags have been already stored
 246                                                                                         // see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51606
 247                                                                                         // Note that for DOM_PARSER, nodes stack may be not empty even no '@' tag
 248                                                                                         // was encountered in comment. But it cannot be the case for COMPILER_PARSER
 249                                                                                         // and so is enough as it is only this parser which signals the missing tag warnings...
 250                                                                                         this.inherited = this.astPtr==-1;
 251                                                                                         if (this.kind == DOM_PARSER) {
 252                                                                                                 valid = parseTag();
 253                                                                                         } else {
 254                                                                                                 valid = true;
 255                                                                                         }
 256                                                                                 } else if (CharOperation.equals(tag, TAG_PARAM)) {
 257                                                                                         valid = parseParam();
 258                                                                                 } else if (CharOperation.equals(tag, TAG_EXCEPTION)) {
 259                                                                                         valid = parseThrows(false);
 260                                                                                 } else if (CharOperation.equals(tag, TAG_SEE)) {
 261                                                                                         if (this.inlineTagStarted) {
 262                                                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
 263                                                                                                 // Cannot have @see inside inline comment
 264                                                                                                 valid = false;
 265                                                                                                 if (this.sourceParser != null)
 266                                                                                                         this.sourceParser.problemReporter().javadocUnexpectedTag(this.tagSourceStart, this.tagSourceEnd);
 267                                                                                         } else {
 268                                                                                                 valid = parseSee(false);
 269                                                                                         }
 270                                                                                 } else if (CharOperation.equals(tag, TAG_LINK)) {
 271                                                                                         if (this.inlineTagStarted) {
 272                                                                                                 valid = parseSee(false);
 273                                                                                         } else {
 274                                                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
 275                                                                                                 // Cannot have @link outside inline comment
 276                                                                                                 valid = false;
 277                                                                                                 if (this.sourceParser != null)
 278                                                                                                         this.sourceParser.problemReporter().javadocUnexpectedTag(this.tagSourceStart, this.tagSourceEnd);
 279                                                                                         }
 280                                                                                 } else if (CharOperation.equals(tag, TAG_LINKPLAIN)) {
 281                                                                                         if (this.inlineTagStarted) {
 282                                                                                                 valid = parseSee(true);
 283                                                                                         } else {
 284                                                                                                 valid = parseTag();
 285                                                                                         }
 286                                                                                 } else {
 287                                                                                         valid = parseTag();
 288                                                                                 }
 289                                                                                 break;
 290                                                                         case TerminalTokens.TokenNamereturn :
 291                                                                                 valid = parseReturn();
 292                                                                                 // verify characters after return tag (we're expecting text description)
 293                                                                                 if(!verifyCharsAfterReturnTag(this.index)) {
 294                                                                                         if (this.sourceParser != null) {
 295                                                                                                 int end = this.starPosition == -1 || this.lineEnd<this.starPosition ? this.lineEnd : this.starPosition;
 296                                                                                                 this.sourceParser.problemReporter().javadocInvalidTag(this.tagSourceStart, end);
 297                                                                                         }
 298                                                                                 }
 299                                                                                 break;
 300                                                                         case TerminalTokens.TokenNamethrows :
 301                                                                                 valid = parseThrows(true);
 302                                                                                 break;
 303                                                                         default:
 304                                                                                 if (this.kind == DOM_PARSER) {
 305                                                                                         switch (token) {
 306                                                                                                 case TerminalTokens.TokenNameabstract:
 307                                                                                                 case TerminalTokens.TokenNameassert:
 308                                                                                                 case TerminalTokens.TokenNameboolean:
 309                                                                                                 case TerminalTokens.TokenNamebreak:
 310                                                                                                 case TerminalTokens.TokenNamebyte:
 311                                                                                                 case TerminalTokens.TokenNamecase:
 312                                                                                                 case TerminalTokens.TokenNamecatch:
 313                                                                                                 case TerminalTokens.TokenNamechar:
 314                                                                                                 case TerminalTokens.TokenNameclass:
 315                                                                                                 case TerminalTokens.TokenNamecontinue:
 316                                                                                                 case TerminalTokens.TokenNamedefault:
 317                                                                                                 case TerminalTokens.TokenNamedo:
 318                                                                                                 case TerminalTokens.TokenNamedouble:
 319                                                                                                 case TerminalTokens.TokenNameelse:
 320                                                                                                 case TerminalTokens.TokenNameextends:
 321                                                                                                 case TerminalTokens.TokenNamefalse:
 322                                                                                                 case TerminalTokens.TokenNamefinal:
 323                                                                                                 case TerminalTokens.TokenNamefinally:
 324                                                                                                 case TerminalTokens.TokenNamefloat:
 325                                                                                                 case TerminalTokens.TokenNamefor:
 326                                                                                                 case TerminalTokens.TokenNameif:
 327                                                                                                 case TerminalTokens.TokenNameimplements:
 328                                                                                                 case TerminalTokens.TokenNameimport:
 329                                                                                                 case TerminalTokens.TokenNameinstanceof:
 330                                                                                                 case TerminalTokens.TokenNameint:
 331                                                                                                 case TerminalTokens.TokenNameinterface:
 332                                                                                                 case TerminalTokens.TokenNamelong:
 333                                                                                                 case TerminalTokens.TokenNamenative:
 334                                                                                                 case TerminalTokens.TokenNamenew:
 335                                                                                                 case TerminalTokens.TokenNamenull:
 336                                                                                                 case TerminalTokens.TokenNamepackage:
 337                                                                                                 case TerminalTokens.TokenNameprivate:
 338                                                                                                 case TerminalTokens.TokenNameprotected:
 339                                                                                                 case TerminalTokens.TokenNamepublic:
 340                                                                                                 case TerminalTokens.TokenNameshort:
 341                                                                                                 case TerminalTokens.TokenNamestatic:
 342                                                                                                 case TerminalTokens.TokenNamestrictfp:
 343                                                                                                 case TerminalTokens.TokenNamesuper:
 344                                                                                                 case TerminalTokens.TokenNameswitch:
 345                                                                                                 case TerminalTokens.TokenNamesynchronized:
 346                                                                                                 case TerminalTokens.TokenNamethis:
 347                                                                                                 case TerminalTokens.TokenNamethrow:
 348                                                                                                 case TerminalTokens.TokenNametransient:
 349                                                                                                 case TerminalTokens.TokenNametrue:
 350                                                                                                 case TerminalTokens.TokenNametry:
 351                                                                                                 case TerminalTokens.TokenNamevoid:
 352                                                                                                 case TerminalTokens.TokenNamevolatile:
 353                                                                                                 case TerminalTokens.TokenNamewhile:
 354                                                                                                         valid = parseTag();
 355                                                                                                         break;
 356                                                                                         }
 357                                                                                 }
 358                                                                 }
 359                                                                 this.textStart = this.index;
 360                                                                 if (!valid) {
 361                                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
 362                                                                         // do not stop the inline tag when error is encountered to get text after
 363                                                                         validComment = false;
 364                                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
 365                                                                         // for DOM AST node, store tag as text in case of invalid syntax
 366                                                                         if (this.kind == DOM_PARSER) {
 367                                                                                 parseTag();
 368                                                                                 this.textStart = this.tagSourceEnd+1;
 369                                                                                 invalidTagLineEnd  = this.lineEnd;
 370                                                                         }
 371                                                                 }
 372                                                         } catch (InvalidInputException e) {
 373                                                                 consumeToken();
 374                                                         }
 375                                                 }
 376                                                 break;
 377                                         case '\r':
 378                                         case '\n':
 379                                                 if (this.lineStarted && this.textStart < previousPosition) {
 380                                                         pushText(this.textStart, previousPosition);
 381                                                 }
 382                                                 this.lineStarted = false;
 383                                                 // Fix bug 51650
 384                                                 this.textStart = -1;
 385                                                 break;
 386                                         case '}' :
 387                                                 if (this.inlineTagStarted) {
 388                                                         if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 389                                                                 pushText(this.textStart, previousPosition);
 390                                                         }
 391                                                         if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
 392                                                         this.textStart = this.index;
 393                                                         this.inlineTagStarted = false;
 394                                                 } else {
 395                                                         if (!this.lineStarted) {
 396                                                                 this.textStart = previousPosition;
 397                                                         }
 398                                                 }
 399                                                 this.lineStarted = true;
 400                                                 break;
 401                                         case '{' :
 402                                                 if (this.inlineTagStarted) {
 403                                                         this.inlineTagStarted = false;
 404                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
 405                                                         // Cannot have opening brace in inline comment
 406                                                         if (this.sourceParser != null) {
 407                                                                 int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
 408                                                                 this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
 409                                                         }
 410                                                         if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 411                                                                 pushText(this.textStart, previousPosition);
 412                                                         }
 413                                                         if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
 414                                                 }
 415                                                 if (!this.lineStarted) {
 416                                                         this.textStart = previousPosition;
 417                                                 }
 418                                                 this.lineStarted = true;
 419                                                 this.inlineTagStart = previousPosition;
 420                                                 break;
 421                                         case '*' :
 422                                         case '\u000c' : /* FORM FEED               */
 423                                         case ' ' :                      /* SPACE                   */
 424                                         case '\t' :                     /* HORIZONTAL TABULATION   */
 425                                                 // do nothing for space or '*' characters
 426                                                 break;
 427                                         default :
 428                                                 if (!this.lineStarted) {
 429                                                         this.textStart = previousPosition;
 430                                                 }
 431                                                 this.lineStarted = true;
 432                                                 break;
 433                                 }
 434                         }
 435                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
 436                         // Cannot leave comment inside inline comment
 437                         if (this.inlineTagStarted) {
 438                                 this.inlineTagStarted = false;
 439                                 if (this.sourceParser != null) {
 440                                         int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
 441                                         if (this.index >= this.endComment) end = invalidInlineTagLineEnd;
 442                                         this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
 443                                 }
 444                                 if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 445                                         pushText(this.textStart, previousPosition);
 446                                 }
 447                                 if (this.kind == DOM_PARSER) {
 448                                         refreshInlineTagPosition(previousPosition);
 449                                 }
 450                         } else if (this.lineStarted && this.textStart < previousPosition) {
 451                                 pushText(this.textStart, previousPosition);
 452                         }
 453                         updateDocComment();
 454                 } catch (Exception ex) {
 455                         validComment = false;
 456                 }
 457                 return validComment;
 458         }
 459
 460         private void consumeToken() {
 461                 this.currentTokenType = -1; // flush token cache
 462                 updateLineEnd();
 463         }
 464
 465         protected abstract Object createArgumentReference(char[] name, int dim, Object typeRef, long[] dimPos, long argNamePos) throws InvalidInputException;
 466         protected abstract Object createFieldReference(Object receiver) throws InvalidInputException;
 467         protected abstract Object createMethodReference(Object receiver, List arguments) throws InvalidInputException;
 468         protected Object createReturnStatement() { return null; }
 469         protected abstract Object createTypeReference(int primitiveToken);
 470
 471         private int getEndPosition() {
 472                 if (this.scanner.getCurrentTokenEndPosition() > this.lineEnd) {
 473                         return this.lineEnd;
 474                 } else {
 475                         return this.scanner.getCurrentTokenEndPosition();
 476                 }
 477         }
 478
 479         /*
 480          * Search the source position corresponding to the end of a given line number.
 481          * Warning: returned position is 1-based index!
 482          * @see Scanner#getLineEnd(int) We cannot directly use this method
 483          * when linePtr field is not initialized.
 484          *
 485         private int getLineEnd(int lineNumber) {
 486
 487                 if (this.scanner.linePtr != -1) {
 488                         return this.scanner.getLineEnd(lineNumber);
 489                 }
 490                 if (this.lineEnds == null)
 491                         return -1;
 492                 if (lineNumber > this.lineEnds.length+1)
 493                         return -1;
 494                 if (lineNumber <= 0)
 495                         return -1;
 496                 if (lineNumber == this.lineEnds.length + 1)
 497                         return this.scanner.eofPosition;
 498                 return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
 499         }
 500         */
 501
 502         /**
 503          * Search the line number corresponding to a specific position.
 504          * Warning: returned position is 1-based index!
 505          * @see Scanner#getLineNumber(int) We cannot directly use this method
 506          * when linePtr field is not initialized.
 507          */
 508         private int getLineNumber(int position) {
 509
 510                 if (this.scanner.linePtr != -1) {
 511                         return this.scanner.getLineNumber(position);
 512                 }
 513                 if (this.lineEnds == null)
 514                         return 1;
 515                 int length = this.lineEnds.length;
 516                 if (length == 0)
 517                         return 1;
 518                 int g = 0, d = length - 1;
 519                 int m = 0;
 520                 while (g <= d) {
 521                         m = (g + d) /2;
 522                         if (position < this.lineEnds[m]) {
 523                                 d = m-1;
 524                         } else if (position > this.lineEnds[m]) {
 525                                 g = m+1;
 526                         } else {
 527                                 return m + 1;
 528                         }
 529                 }
 530                 if (position < this.lineEnds[m]) {
 531                         return m+1;
 532                 }
 533                 return m+2;
 534         }
 535
 536         /*
 537          * Parse argument in @see tag method reference
 538          */
 539         private Object parseArguments(Object receiver) throws InvalidInputException {
 540
 541                 // Init
 542                 int modulo = 0; // should be 2 for (Type,Type,...) or 3 for (Type arg,Type arg,...)
 543                 int iToken = 0;
 544                 char[] argName = null;
 545                 List arguments = new ArrayList(10);
 546                 int start = this.scanner.getCurrentTokenStartPosition();
 547
 548                 // Parse arguments declaration if method reference
 549                 nextArg : while (this.index < this.scanner.eofPosition) {
 550
 551                         // Read argument type reference
 552                         Object typeRef;
 553                         try {
 554                                 typeRef = parseQualifiedName(false);
 555                         } catch (InvalidInputException e) {
 556                                 break nextArg;
 557                         }
 558                         boolean firstArg = modulo == 0;
 559                         if (firstArg) { // verify position
 560                                 if (iToken != 0)
 561                                         break nextArg;
 562                         } else if ((iToken % modulo) != 0) {
 563                                         break nextArg;
 564                         }
 565                         if (typeRef == null) {
 566                                 if (firstArg && this.currentTokenType == TerminalTokens.TokenNameRPAREN) {
 567                                         // verify characters after arguments declaration (expecting white space or end comment)
 568                                         if (!verifySpaceOrEndComment()) {
 569                                                 int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 570                                                 if (this.source[end]=='\n') end--;
 571                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
 572                                                 return null;
 573                                         }
 574                                         this.lineStarted = true;
 575                                         return createMethodReference(receiver, null);
 576                                 }
 577                                 break nextArg;
 578                         }
 579                         iToken++;
 580
 581                         // Read possible array declaration
 582                         int dim = 0;
 583                         long[] dimPositions = new long[20]; // assume that there won't be more than 20 dimensions...
 584                         if (readToken() == TerminalTokens.TokenNameLBRACKET) {
 585                                 int dimStart = this.scanner.getCurrentTokenStartPosition();
 586                                 while (readToken() == TerminalTokens.TokenNameLBRACKET) {
 587                                         consumeToken();
 588                                         if (readToken() != TerminalTokens.TokenNameRBRACKET) {
 589                                                 break nextArg;
 590                                         }
 591                                         consumeToken();
 592                                         dimPositions[dim++] = (((long) dimStart) << 32) + this.scanner.getCurrentTokenEndPosition();
 593                                 }
 594                         }
 595
 596                         // Read argument name
 597                         long argNamePos = -1;
 598                         if (readToken() == TerminalTokens.TokenNameIdentifier) {
 599                                 consumeToken();
 600                                 if (firstArg) { // verify position
 601                                         if (iToken != 1)
 602                                                 break nextArg;
 603                                 } else if ((iToken % modulo) != 1) {
 604                                                 break nextArg;
 605                                 }
 606                                 if (argName == null) { // verify that all arguments name are declared
 607                                         if (!firstArg) {
 608                                                 break nextArg;
 609                                         }
 610                                 }
 611                                 argName = this.scanner.getCurrentIdentifierSource();
 612                                 argNamePos = (((long)this.scanner.getCurrentTokenStartPosition())<<32)+this.scanner.getCurrentTokenEndPosition();
 613                                 iToken++;
 614                         } else if (argName != null) { // verify that no argument name is declared
 615                                 break nextArg;
 616                         }
 617
 618                         // Verify token position
 619                         if (firstArg) {
 620                                 modulo = iToken + 1;
 621                         } else {
 622                                 if ((iToken % modulo) != (modulo - 1)) {
 623                                         break nextArg;
 624                                 }
 625                         }
 626
 627                         // Read separator or end arguments declaration
 628                         int token = readToken();
 629                         char[] name = argName == null ? new char[0] : argName;
 630                         if (token == TerminalTokens.TokenNameCOMMA) {
 631                                 // Create new argument
 632                                 Object argument = createArgumentReference(name, dim, typeRef, dimPositions, argNamePos);
 633                                 arguments.add(argument);
 634                                 consumeToken();
 635                                 iToken++;
 636                         } else if (token == TerminalTokens.TokenNameRPAREN) {
 637                                 // verify characters after arguments declaration (expecting white space or end comment)
 638                                 if (!verifySpaceOrEndComment()) {
 639                                         int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 640                                         if (this.source[end]=='\n') end--;
 641                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
 642                                         return null;
 643                                 }
 644                                 // Create new argument
 645                                 Object argument = createArgumentReference(name, dim, typeRef, dimPositions, argNamePos);
 646                                 arguments.add(argument);
 647                                 consumeToken();
 648                                 return createMethodReference(receiver, arguments);
 649                         } else {
 650                                 break nextArg;
 651                         }
 652                 }
 653
 654                 // Something wrong happened => Invalid input
 655                 throw new InvalidInputException();
 656         }
 657
 658         /*
 659          * Parse an URL link reference in @see tag
 660          */
 661         private boolean parseHref() throws InvalidInputException {
 662                 int start = this.scanner.getCurrentTokenStartPosition();
 663                 if (Character.toLowerCase(readChar()) == 'a') {
 664                         this.scanner.currentPosition = this.index;
 665                         if (readToken() == TerminalTokens.TokenNameIdentifier) {
 666                                 this.currentTokenType = -1; // do not update line end
 667                                 try {
 668                                         if (CharOperation.equals(this.scanner.getCurrentIdentifierSource(), new char[]{'h', 'r', 'e', 'f'}, false) &&
 669                                                 readToken() == TerminalTokens.TokenNameEQUAL) {
 670                                                 this.currentTokenType = -1; // do not update line end
 671                                                 if (readToken() == TerminalTokens.TokenNameStringLiteral) {
 672                                                         this.currentTokenType = -1; // do not update line end
 673                                                         // Skip all characters after string literal until closing '>' (see bug 68726)
 674                                                         while (this.index <= this.lineEnd && readToken() != TerminalTokens.TokenNameGREATER) {
 675                                                                 this.currentTokenType = -1; // do not update line end
 676                                                         }
 677                                                         if (this.currentTokenType == TerminalTokens.TokenNameGREATER) {
 678                                                                 consumeToken(); // update line end as new lines are allowed in URL description
 679                                                                 while (readToken() != TerminalTokens.TokenNameLESS) {
 680                                                                         if (this.scanner.currentPosition >= this.scanner.eofPosition || this.scanner.currentCharacter == '@') {
 681                                                                                 // Reset position: we want to rescan last token
 682                                                                                 this.index = this.tokenPreviousPosition;
 683                                                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
 684                                                                                 this.currentTokenType = -1;
 685                                                                                 // Signal syntax error
 686                                                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(start, this.lineEnd);
 687                                                                                 return false;
 688                                                                         }
 689                                                                         consumeToken();
 690                                                                 }
 691                                                                 this.currentTokenType = -1; // do not update line end
 692                                                                 if (readChar() == '/') {
 693                                                                         if (Character.toLowerCase(readChar()) == 'a') {
 694                                                                                 if (readChar() == '>') {
 695                                                                                         // Valid href
 696                                                                                         return true;
 697                                                                                 }
 698                                                                         }
 699                                                                 }
 700                                                         }
 701                                                 }
 702                                         }
 703                                 } catch (InvalidInputException ex) {
 704                                         // Do nothing as we want to keep positions for error message
 705                                 }
 706                         }
 707                 }
 708                 // Reset position: we want to rescan last token
 709                 this.index = this.tokenPreviousPosition;
 710                 this.scanner.currentPosition = this.tokenPreviousPosition;
 711                 this.currentTokenType = -1;
 712                 // Signal syntax error
 713                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(start, this.lineEnd);
 714                 return false;
 715         }
 716
 717         /*
 718          * Parse a method reference in @see tag
 719          */
 720         private Object parseMember(Object receiver) throws InvalidInputException {
 721                 // Init
 722                 this.identifierPtr = -1;
 723                 this.identifierLengthPtr = -1;
 724                 int start = this.scanner.getCurrentTokenStartPosition();
 725                 this.memberStart = start;
 726
 727                 // Get member identifier
 728                 if (readToken() == TerminalTokens.TokenNameIdentifier) {
 729                         consumeToken();
 730                         pushIdentifier(true);
 731                         // Look for next token to know whether it's a field or method reference
 732                         int previousPosition = this.index;
 733                         if (readToken() == TerminalTokens.TokenNameLPAREN) {
 734                                 consumeToken();
 735                                 start = this.scanner.getCurrentTokenStartPosition();
 736                                 try {
 737                                         return parseArguments(receiver);
 738                                 } catch (InvalidInputException e) {
 739                                         int end = this.scanner.getCurrentTokenEndPosition() < this.lineEnd ?
 740                                                         this.scanner.getCurrentTokenEndPosition() :
 741                                                         this.scanner.getCurrentTokenStartPosition();
 742                                         end = end < this.lineEnd ? end : this.lineEnd;
 743                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReferenceArgs(start, end);
 744                                 }
 745                                 return null;
 746                         }
 747
 748                         // Reset position: we want to rescan last token
 749                         this.index = previousPosition;
 750                         this.scanner.currentPosition = previousPosition;
 751                         this.currentTokenType = -1;
 752
 753                         // Verify character(s) after identifier (expecting space or end comment)
 754                         if (!verifySpaceOrEndComment()) {
 755                                 int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 756                                 if (this.source[end]=='\n') end--;
 757                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
 758                                 return null;
 759                         }
 760                         return createFieldReference(receiver);
 761                 }
 762                 int end = getEndPosition() - 1;
 763                 end = start > end ? start : end;
 764                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, end);
 765                 // Reset position: we want to rescan last token
 766                 this.index = this.tokenPreviousPosition;
 767                 this.scanner.currentPosition = this.tokenPreviousPosition;
 768                 this.currentTokenType = -1;
 769                 return null;
 770         }
 771
 772         /*
 773          * Parse @param tag declaration
 774          */
 775         protected boolean parseParam() {
 776
 777                 // Store current token state
 778                 int start = this.tagSourceStart;
 779                 int end = this.tagSourceEnd;
 780
 781                 try {
 782                         // Push identifier next
 783                         int token = readToken();
 784                         switch (token) {
 785                                 case TerminalTokens.TokenNameIdentifier :
 786                                         consumeToken();
 787                                         return pushParamName();
 788                                 case TerminalTokens.TokenNameEOF :
 789                                         break;
 790                                 default :
 791                                         start = this.scanner.getCurrentTokenStartPosition();
 792                                         end = getEndPosition();
 793                                         if (end < start) start = this.tagSourceStart;
 794                                         break;
 795                         }
 796                 } catch (InvalidInputException e) {
 797                         end = getEndPosition();
 798                 }
 799
 800                 // Reset position to avoid missing tokens when new line was encountered
 801                 this.index = this.tokenPreviousPosition;
 802                 this.scanner.currentPosition = this.tokenPreviousPosition;
 803                 this.currentTokenType = -1;
 804
 805                 // Report problem
 806                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingParamName(start, end);
 807                 return false;
 808         }
 809
 810         /*
 811          * Parse a qualified name and built a type reference if the syntax is valid.
 812          */
 813         protected Object parseQualifiedName(boolean reset) throws InvalidInputException {
 814
 815                 // Reset identifier stack if requested
 816                 if (reset) {
 817                         this.identifierPtr = -1;
 818                         this.identifierLengthPtr = -1;
 819                 }
 820
 821                 // Scan tokens
 822                 int primitiveToken = -1;
 823                 nextToken : for (int iToken = 0; ; iToken++) {
 824                         int token = readToken();
 825                         switch (token) {
 826                                 case TerminalTokens.TokenNameIdentifier :
 827                                         if (((iToken % 2) > 0)) { // identifiers must be odd tokens
 828                                                 break nextToken;
 829                                         }
 830                                         pushIdentifier(iToken == 0);
 831                                         consumeToken();
 832                                         break;
 833
 834                                 case TerminalTokens.TokenNameDOT :
 835                                         if ((iToken % 2) == 0) { // dots must be even tokens
 836                                                 throw new InvalidInputException();
 837                                         }
 838                                         consumeToken();
 839                                         break;
 840
 841                                 case TerminalTokens.TokenNamevoid :
 842                                 case TerminalTokens.TokenNameboolean :
 843                                 case TerminalTokens.TokenNamebyte :
 844                                 case TerminalTokens.TokenNamechar :
 845                                 case TerminalTokens.TokenNamedouble :
 846                                 case TerminalTokens.TokenNamefloat :
 847                                 case TerminalTokens.TokenNameint :
 848                                 case TerminalTokens.TokenNamelong :
 849                                 case TerminalTokens.TokenNameshort :
 850                                         if (iToken > 0) {
 851                                                 throw new InvalidInputException();
 852                                         }
 853                                         pushIdentifier(true);
 854                                         primitiveToken = token;
 855                                         consumeToken();
 856                                         break nextToken;
 857
 858                                 default :
 859                                         if (iToken == 0) {
 860                                                 return null;
 861                                         }
 862                                         if ((iToken % 2) == 0) { // cannot leave on a dot
 863                                                 // Reset position: we want to rescan last token
 864                                                 if (this.kind == DOM_PARSER && this.currentTokenType != -1) {
 865                                                         this.index = this.tokenPreviousPosition;
 866                                                         this.scanner.currentPosition = this.tokenPreviousPosition;
 867                                                         this.currentTokenType = -1;
 868                                                 }
 869                                                 throw new InvalidInputException();
 870                                         }
 871                                         break nextToken;
 872                         }
 873                 }
 874                 // Reset position: we want to rescan last token
 875                 if (this.currentTokenType != -1) {
 876                         this.index = this.tokenPreviousPosition;
 877                         this.scanner.currentPosition = this.tokenPreviousPosition;
 878                         this.currentTokenType = -1;
 879                 }
 880                 this.lastIdentifierEndPosition = (int) this.identifierPositionStack[this.identifierPtr];
 881                 return createTypeReference(primitiveToken);
 882         }
 883
 884         /*
 885          * Parse a reference in @see tag
 886          */
 887         protected boolean parseReference(boolean plain) throws InvalidInputException {
 888                 Object typeRef = null;
 889                 Object reference = null;
 890                 int previousPosition = -1;
 891                 int typeRefStartPosition = -1;
 892                 nextToken : while (this.index < this.scanner.eofPosition) {
 893                         previousPosition = this.index;
 894                         int token = readToken();
 895                         switch (token) {
 896                                 case TerminalTokens.TokenNameStringLiteral : // @see "string"
 897                                         int start = this.scanner.getCurrentTokenStartPosition();
 898                                         consumeToken();
 899                                         // If typeRef != null we may raise a warning here to let user know there's an unused reference...
 900                                         // Currently as javadoc 1.4.2 ignore it, we do the same (see bug 69302)
 901                                         if (typeRef != null) {
 902                                                 start = this.tagSourceEnd+1;
 903                                                 previousPosition = start;
 904                                                 typeRef = null;
 905                                         }
 906                                         // verify end line (expecting empty or end comment)
 907                                         if (verifyEndLine(previousPosition)) {
 908                                                 return true;
 909                                         }
 910                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, this.lineEnd);
 911                                         return false;
 912                                 case TerminalTokens.TokenNameLESS : // @see "<a href="URL#Value">label</a>
 913                                         consumeToken();
 914                                         start = this.scanner.getCurrentTokenStartPosition();
 915                                         if (parseHref()) {
 916                                                 consumeToken();
 917                                                 // If typeRef != null we may raise a warning here to let user know there's an unused reference...
 918                                                 // Currently as javadoc 1.4.2 ignore it, we do the same (see bug 69302)
 919                                                 if (typeRef != null) {
 920                                                         start = this.tagSourceEnd+1;
 921                                                         previousPosition = start;
 922                                                         typeRef = null;
 923                                                 }
 924                                                 // verify end line (expecting empty or end comment)
 925                                                 if (verifyEndLine(previousPosition)) {
 926                                                         return true;
 927                                                 }
 928                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, this.lineEnd);
 929                                         }
 930                                         return false;
 931                                 case TerminalTokens.TokenNameERROR :
 932                                         if (this.scanner.currentCharacter == '#') { // @see ...#member
 933                                                 consumeToken();
 934                                                 reference = parseMember(typeRef);
 935                                                 if (reference != null) {
 936                                                         return pushSeeRef(reference, plain);
 937                                                 }
 938                                                 return false;
 939                                         }
 940                                         break nextToken;
 941                                 case TerminalTokens.TokenNameIdentifier :
 942                                         if (typeRef == null) {
 943                                                 typeRefStartPosition = this.scanner.getCurrentTokenStartPosition();
 944                                                 typeRef = parseQualifiedName(true);
 945                                                 break;
 946                                         }
 947                                         break nextToken;
 948                                 default :
 949                                         break nextToken;
 950                         }
 951                 }
 952
 953                 // Verify that we got a reference
 954                 if (reference == null) reference = typeRef;
 955                 if (reference == null) {
 956                         this.index = this.tokenPreviousPosition;
 957                         this.scanner.currentPosition = this.tokenPreviousPosition;
 958                         this.currentTokenType = -1;
 959                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingSeeReference(this.tagSourceStart, this.tagSourceEnd);
 960                         return false;
 961                 }
 962
 963                 // Reset position at the end of type reference
 964                 this.index = this.lastIdentifierEndPosition+1;
 965                 this.scanner.currentPosition = this.index;
 966                 this.currentTokenType = -1;
 967
 968                 // Verify that line end does not start with an open parenthese (which could be a constructor reference wrongly written...)
 969                 // See bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=47215
 970                 char ch = peekChar();
 971                 if (ch == '(') {
 972                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(typeRefStartPosition, this.lineEnd);
 973                         return false;
 974                 }
 975
 976                 // Verify that we get white space after reference
 977                 if (!verifySpaceOrEndComment()) {
 978                         this.index = this.tokenPreviousPosition;
 979                         this.scanner.currentPosition = this.tokenPreviousPosition;
 980                         this.currentTokenType = -1;
 981                         int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 982                         if (this.source[end]=='\n') end--;
 983                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(typeRefStartPosition, end);
 984                         return false;
 985                 }
 986
 987                 // Everything is OK, store reference
 988                 return pushSeeRef(reference, plain);
 989         }
 990
 991         /*
 992          * Parse @return tag declaration
 993          */
 994         protected abstract boolean parseReturn();
 995
 996         /*
 997          * Parse @see tag declaration
 998          */
 999         protected boolean parseSee(boolean plain) {
1000                 int start = this.scanner.currentPosition;
1001                 try {
1002                         return parseReference(plain);
1003                 } catch (InvalidInputException ex) {
1004                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, getEndPosition());
1005                 }
1006                 // Reset position to avoid missing tokens when new line was encountered
1007                 this.index = this.tokenPreviousPosition;
1008                 this.scanner.currentPosition = this.tokenPreviousPosition;
1009                 this.currentTokenType = -1;
1010                 return false;
1011         }
1012
1013         /*
1014          * Parse @return tag declaration
1015          */
1016         protected abstract boolean parseTag();
1017
1018         /*
1019          * Parse @throws tag declaration
1020          */
1021         protected boolean parseThrows(boolean real) {
1022                 int start = this.scanner.currentPosition;
1023                 try {
1024                         Object typeRef = parseQualifiedName(true);
1025                         if (typeRef == null) {
1026                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingThrowsClassName(this.tagSourceStart, this.tagSourceEnd);
1027                         } else {
1028                                 return pushThrowName(typeRef, real);
1029                         }
1030                 } catch (InvalidInputException ex) {
1031                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidThrowsClass(start, getEndPosition());
1032                 }
1033                 return false;
1034         }
1035
1036         /*
1037          * Return current character without move index position.
1038          */
1039         private char peekChar() {
1040                 int idx = this.index;
1041                 char c = this.source[idx++];
1042                 if (c == '\\' && this.source[idx] == 'u') {
1043                         int c1, c2, c3, c4;
1044                         idx++;
1045                         while (this.source[idx] == 'u')
1046                                 idx++;
1047                         if (!(((c1 = Character.getNumericValue(this.source[idx++])) > 15 || c1 < 0)
1048                                         || ((c2 = Character.getNumericValue(this.source[idx++])) > 15 || c2 < 0)
1049                                         || ((c3 = Character.getNumericValue(this.source[idx++])) > 15 || c3 < 0) || ((c4 = Character.getNumericValue(this.source[idx++])) > 15 || c4 < 0))) {
1050                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1051                         }
1052                 }
1053                 return c;
1054         }
1055
1056         /*
1057          * push the consumeToken on the identifier stack. Increase the total number of identifier in the stack.
1058          */
1059         protected void pushIdentifier(boolean newLength) {
1060
1061                 int stackLength = this.identifierStack.length;
1062                 if (++this.identifierPtr >= stackLength) {
1063                         System.arraycopy(
1064                                 this.identifierStack, 0,
1065                                 this.identifierStack = new char[stackLength + 10][], 0,
1066                                 stackLength);
1067                         System.arraycopy(
1068                                 this.identifierPositionStack, 0,
1069                                 this.identifierPositionStack = new long[stackLength + 10], 0,
1070                                 stackLength);
1071                 }
1072                 this.identifierStack[this.identifierPtr] = this.scanner.getCurrentIdentifierSource();
1073                 this.identifierPositionStack[this.identifierPtr] = (((long) this.scanner.startPosition) << 32) + (this.scanner.currentPosition - 1);
1074
1075                 if (newLength) {
1076                         stackLength = this.identifierLengthStack.length;
1077                         if (++this.identifierLengthPtr >= stackLength) {
1078                                 System.arraycopy(
1079                                         this.identifierLengthStack, 0,
1080                                         this.identifierLengthStack = new int[stackLength + 10], 0,
1081                                         stackLength);
1082                         }
1083                         this.identifierLengthStack[this.identifierLengthPtr] = 1;
1084                 } else {
1085                         this.identifierLengthStack[this.identifierLengthPtr]++;
1086                 }
1087         }
1088
1089         /*
1090          * Add a new obj on top of the ast stack.
1091          * If new length is required, then add also a new length in length stack.
1092          */
1093         protected void pushOnAstStack(Object node, boolean newLength) {
1094
1095                 if (node == null) {
1096                         this.astLengthStack[++this.astLengthPtr] = 0;
1097                         return;
1098                 }
1099
1100                 int stackLength = this.astStack.length;
1101                 if (++this.astPtr >= stackLength) {
1102                         System.arraycopy(
1103                                 this.astStack, 0,
1104                                 this.astStack = new Object[stackLength + AstStackIncrement], 0,
1105                                 stackLength);
1106                         this.astPtr = stackLength;
1107                 }
1108                 this.astStack[this.astPtr] = node;
1109
1110                 if (newLength) {
1111                         stackLength = this.astLengthStack.length;
1112                         if (++this.astLengthPtr >= stackLength) {
1113                                 System.arraycopy(
1114                                         this.astLengthStack, 0,
1115                                         this.astLengthStack = new int[stackLength + AstStackIncrement], 0,
1116                                         stackLength);
1117                         }
1118                         this.astLengthStack[this.astLengthPtr] = 1;
1119                 } else {
1120                         this.astLengthStack[this.astLengthPtr]++;
1121                 }
1122         }
1123
1124         /*
1125          * Push a param name in ast node stack.
1126          */
1127         protected abstract boolean pushParamName();
1128
1129         /*
1130          * Push a reference statement in ast node stack.
1131          */
1132         protected abstract boolean pushSeeRef(Object statement, boolean plain);
1133
1134         /*
1135          * Push a text element in ast node stack
1136          */
1137         protected abstract void pushText(int start, int end);
1138
1139         /*
1140          * Push a throws type ref in ast node stack.
1141          */
1142         protected abstract boolean pushThrowName(Object typeRef, boolean real);
1143
1144         /*
1145          * Read current character and move index position.
1146          * Warning: scanner position is unchanged using this method!
1147          */
1148         protected char readChar() {
1149
1150                 char c = this.source[this.index++];
1151                 if (c == '\\' && this.source[this.index] == 'u') {
1152                         int c1, c2, c3, c4;
1153                         int pos = this.index;
1154                         this.index++;
1155                         while (this.source[this.index] == 'u')
1156                                 this.index++;
1157                         if (!(((c1 = Character.getNumericValue(this.source[this.index++])) > 15 || c1 < 0)
1158                                         || ((c2 = Character.getNumericValue(this.source[this.index++])) > 15 || c2 < 0)
1159                                         || ((c3 = Character.getNumericValue(this.source[this.index++])) > 15 || c3 < 0) || ((c4 = Character.getNumericValue(this.source[this.index++])) > 15 || c4 < 0))) {
1160                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1161                         } else {
1162                                 // TODO (frederic) currently reset to previous position, perhaps signal a syntax error would be more appropriate
1163                                 this.index = pos;
1164                         }
1165                 }
1166                 return c;
1167         }
1168
1169         /*
1170          * Read token only if previous was consumed
1171          */
1172         private int readToken() throws InvalidInputException {
1173                 if (this.currentTokenType < 0) {
1174                         this.tokenPreviousPosition = this.scanner.currentPosition;
1175                         this.currentTokenType = this.scanner.getNextToken();
1176                         if (this.scanner.currentPosition > (this.lineEnd+1)) { // be sure to be on next line (lineEnd is still on the same line)
1177                                 this.lineStarted = false;
1178                                 while (this.currentTokenType == TerminalTokens.TokenNameMULTIPLY) {
1179                                         this.currentTokenType = this.scanner.getNextToken();
1180                                 }
1181                         }
1182                         this.index = this.scanner.currentPosition;
1183                         this.lineStarted = true; // after having read a token, line is obviously started...
1184                 }
1185                 return this.currentTokenType;
1186         }
1187
1188         private int readTokenAndConsume() throws InvalidInputException {
1189                 int token = readToken();
1190                 consumeToken();
1191                 return token;
1192         }
1193
1194         /*
1195          * Refresh start position and length of an inline tag.
1196          */
1197         protected void refreshInlineTagPosition(int previousPosition) {
1198                 // do nothing by default
1199         }
1200
1201         public String toString() {
1202                 StringBuffer buffer = new StringBuffer();
1203                 int startPos = this.scanner.currentPosition<this.index ? this.scanner.currentPosition : this.index;
1204                 int endPos = this.scanner.currentPosition<this.index ? this.index : this.scanner.currentPosition;
1205                 if (startPos == this.source.length)
1206                         return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
1207                 if (endPos > this.source.length)
1208                         return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
1209
1210                 char front[] = new char[startPos];
1211                 System.arraycopy(this.source, 0, front, 0, startPos);
1212
1213                 int middleLength = (endPos - 1) - startPos + 1;
1214                 char middle[];
1215                 if (middleLength > -1) {
1216                         middle = new char[middleLength];
1217                         System.arraycopy(
1218                                 this.source,
1219                                 startPos,
1220                                 middle,
1221                                 0,
1222                                 middleLength);
1223                 } else {
1224                         middle = CharOperation.NO_CHAR;
1225                 }
1226
1227                 char end[] = new char[this.source.length - (endPos - 1)];
1228                 System.arraycopy(
1229                         this.source,
1230                         (endPos - 1) + 1,
1231                         end,
1232                         0,
1233                         this.source.length - (endPos - 1) - 1);
1234
1235                 buffer.append(front);
1236                 if (this.scanner.currentPosition<this.index) {
1237                         buffer.append("\n===============================\nScanner current position here -->"); //$NON-NLS-1$
1238                 } else {
1239                         buffer.append("\n===============================\nParser index here -->"); //$NON-NLS-1$
1240                 }
1241                 buffer.append(middle);
1242                 if (this.scanner.currentPosition<this.index) {
1243                         buffer.append("<-- Parser index here\n===============================\n"); //$NON-NLS-1$
1244                 } else {
1245                         buffer.append("<-- Scanner current position here\n===============================\n"); //$NON-NLS-1$
1246                 }
1247                 buffer.append(end);
1248
1249                 return buffer.toString();
1250         }
1251
1252         /*
1253          * Update
1254          */
1255         protected abstract void updateDocComment();
1256
1257         /*
1258          * Update line end
1259          */
1260         protected void updateLineEnd() {
1261                 while (this.index > (this.lineEnd+1)) { // be sure to be on next line (lineEnd is still on the same line)
1262                         if (this.linePtr < this.lastLinePtr) {
1263                                 this.lineEnd = this.scanner.getLineEnd(++this.linePtr) - 1;
1264                         } else {
1265                                 this.lineEnd = this.endComment;
1266                                 return;
1267                         }
1268                 }
1269         }
1270
1271         /*
1272          * Verify that end of the line only contains space characters or end of comment.
1273          * Note that end of comment may be preceeding by several contiguous '*' chars.
1274          */
1275         private boolean verifyEndLine(int textPosition) {
1276                 int startPosition = this.index;
1277                 int previousPosition = this.index;
1278                 this.starPosition = -1;
1279                 char ch = readChar();
1280                 nextChar: while (true) {
1281                         switch (ch) {
1282                                 case '\r':
1283                                 case '\n':
1284                                         if (this.kind == DOM_PARSER) {
1285                                                 parseTag();
1286                                                 pushText(textPosition, previousPosition);
1287                                         }
1288                                         this.index = previousPosition;
1289                                         return true;
1290                                 case '\u000c' : /* FORM FEED               */
1291                                 case ' ' :                      /* SPACE                   */
1292                                 case '\t' :                     /* HORIZONTAL TABULATION   */
1293                                         if (this.starPosition >= 0) break nextChar;
1294                                         break;
1295                                 case '*':
1296                                         this.starPosition = previousPosition;
1297                                         break;
1298                                 case '/':
1299                                         if (this.starPosition >= textPosition) {
1300                                                 if (this.kind == DOM_PARSER) {
1301                                                         parseTag();
1302                                                         pushText(textPosition, this.starPosition);
1303                                                 }
1304                                                 return true;
1305                                         }
1306                                 default :
1307                                         // leave loop
1308                                         break nextChar;
1309
1310                         }
1311                         previousPosition = this.index;
1312                         ch = readChar();
1313                 }
1314                 this.index = startPosition;
1315                 return false;
1316         }
1317
1318         /*
1319          * Verify that some text exists after a @return tag. Text must be different than
1320          * end of comment which may be preceeding by several '*' chars.
1321          */
1322         private boolean verifyCharsAfterReturnTag(int startPosition) {
1323                 // Whitespace or inline tag closing brace
1324                 int previousPosition = this.index;
1325                 char ch = readChar();
1326                 boolean malformed = true;
1327                 while (Character.isWhitespace(ch)) {
1328                         malformed = false;
1329                         previousPosition = this.index;
1330                         ch = readChar();
1331                 }
1332                 // End of comment
1333                 this.starPosition = -1;
1334                 nextChar: while (this.index<this.source.length) {
1335                         switch (ch) {
1336                                 case '*':
1337                                         // valid whatever the number of star before last '/'
1338                                         this.starPosition = previousPosition;
1339                                         break;
1340                                 case '/':
1341                                         if (this.starPosition >= startPosition) { // valid only if a star was previous character
1342                                                 return false;
1343                                         }
1344                                 default :
1345                                         // valid if any other character is encountered, even white spaces
1346                                         this.index = startPosition;
1347                                         return !malformed;
1348
1349                         }
1350                         previousPosition = this.index;
1351                         ch = readChar();
1352                 }
1353                 this.index = startPosition;
1354                 return false;
1355         }
1356
1357         /*
1358          * Verify characters after a name matches one of following conditions:
1359          *      1- first character is a white space
1360          *      2- first character is a closing brace *and* we're currently parsing an inline tag
1361          *      3- are the end of comment (several contiguous star ('*') characters may be
1362          *          found before the last slash ('/') character).
1363          */
1364         private boolean verifySpaceOrEndComment() {
1365                 int startPosition = this.index;
1366                 // Whitespace or inline tag closing brace
1367                 char ch = peekChar();
1368                 switch (ch) {
1369                         case '}':
1370                                 return this.inlineTagStarted;
1371                         default:
1372                                 if (Character.isWhitespace(ch)) {
1373                                         return true;
1374                                 }
1375                 }
1376                 // End of comment
1377                 int previousPosition = this.index;
1378                 this.starPosition = -1;
1379                 ch = readChar();
1380                 nextChar: while (this.index<this.source.length) {
1381                         switch (ch) {
1382                                 case '*':
1383                                         // valid whatever the number of star before last '/'
1384                                         this.starPosition = previousPosition;
1385                                         break;
1386                                 case '/':
1387                                         if (this.starPosition >= startPosition) { // valid only if a star was previous character
1388                                                 return true;
1389                                         }
1390                                 default :
1391                                         // invalid whatever other character, even white spaces
1392                                         this.index = startPosition;
1393                                         return false;
1394
1395                         }
1396                         previousPosition = this.index;
1397                         ch = readChar();
1398                 }
1399                 this.index = startPosition;
1400                 return false;
1401         }
1402 }