[283055] HTMLHeadTokenizer can get stuck in an infinite loop
diff --git a/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizer.java b/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizer.java index f100697..6f03350 100644 --- a/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizer.java +++ b/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizer.java
@@ -1,7 +1,7 @@ -/* The following code was generated by JFlex 1.2.2 on 7/28/08 5:19 PM */ +/* The following code was generated by JFlex 1.2.2 on 6/29/09 9:58 AM */ /******************************************************************************* - * Copyright (c) 2004, 2008 IBM Corporation and others. + * Copyright (c) 2004, 2009 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -24,8 +24,8 @@ /** * This class is a scanner generated by * <a href="http://www.informatik.tu-muenchen.de/~kleing/jflex/">JFlex</a> 1.2.2 - * on 7/28/08 5:19 PM from the specification file - * <tt>file:/D:/workspaces/wtp301/workspace/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/HTMLHeadTokenizer.jFlex</tt> + * on 6/29/09 9:58 AM from the specification file + * <tt>file:/D:/dev/workspaces/wtp3M/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/HTMLHeadTokenizer.jFlex</tt> */ public class HTMLHeadTokenizer { @@ -116,7 +116,7 @@ private int yy_endRead; /** number of newlines encountered up to the start of the matched text */ -// private int yyline; + //private int yyline; /** the number of characters up to the start of the matched text */ private int yychar; @@ -125,7 +125,7 @@ * the number of characters from the last newline up to the start of the * matched text */ -// private int yycolumn; + //private int yycolumn; /** * yy_atBOL == true <=> the scanner is currently at the beginning of a line @@ -188,7 +188,7 @@ yy_endRead = 0; /* number of newlines encountered up to the start of the matched text */ -// yyline = 0; + // yyline = 0; /* the number of characters up to the start of the matched text */ yychar = 0; @@ -197,7 +197,7 @@ * the number of characters from the last newline up to the start * of the matched text */ -// yycolumn = 0; + // yycolumn = 0; /** * yy_atBOL == true <=> the scanner is currently at the beginning @@ -229,8 +229,16 @@ public final HeadParserToken getNextToken() throws IOException { String context = null; - context = primGetNextToken(); HeadParserToken result = null; + try { + context = primGetNextToken(); + } + catch (IllegalStateException e) { + hasMore = false; + result = createToken(HTMLHeadTokenizerConstants.UNDEFINED, yychar, yytext()); + while(yy_advance() != YYEOF) {} + return result; + } if (valueText != null) { result = createToken(context, yychar, valueText); valueText = null; @@ -347,7 +355,7 @@ return yy_buffer[yy_currentPos++]; } - + /** * Closes the input stream. */ @@ -397,13 +405,14 @@ */ private void yy_ScanError(int errorCode) { try { - System.out.println(YY_ERROR_MSG[errorCode]); + IllegalStateException ise = new IllegalStateException("Instance: " + System.identityHashCode(this) + " offset:" + yychar + " state:" + yystate()); + System.out.println(YY_ERROR_MSG[errorCode] + "\n" + ise); + throw ise; } catch (ArrayIndexOutOfBoundsException e) { System.out.println(YY_ERROR_MSG[YY_UNKNOWN_ERROR]); } -// System.exit(1); } @@ -473,7 +482,7 @@ yy_forAction: { while (true) { - + yy_input = yy_advance(); if ( yy_input == YYEOF ) break yy_forAction;
diff --git a/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizerConstants.java b/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizerConstants.java index 938bcde..2600b07 100644 --- a/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizerConstants.java +++ b/bundles/org.eclipse.wst.html.core/src/org/eclipse/wst/html/core/internal/contenttype/HTMLHeadTokenizerConstants.java
@@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2004 IBM Corporation and others. + * Copyright (c) 2004, 2009 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -16,5 +16,5 @@ String MetaTagEnd = "MetaTagEnd"; //$NON-NLS-1$ String MetaTagStart = "MetaTagStart"; //$NON-NLS-1$ String MetaTagContentType = "MetaTagContentType"; //$NON-NLS-1$ - + String UNDEFINED = "UNDEFINED"; //$NON-NLS-1$ } \ No newline at end of file
diff --git a/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/HTMLHeadTokenizer.jFlex b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/HTMLHeadTokenizer.jFlex index fd5dc23..fce07bc 100644 --- a/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/HTMLHeadTokenizer.jFlex +++ b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/HTMLHeadTokenizer.jFlex
@@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2004, 2008 IBM Corporation and others. + * Copyright (c) 2004, 2009 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -70,7 +70,7 @@ yy_endRead = 0; /* number of newlines encountered up to the start of the matched text */ - yyline = 0; + // yyline = 0; /* the number of characters up to the start of the matched text */ yychar = 0; @@ -79,7 +79,7 @@ * the number of characters from the last newline up to the start * of the matched text */ - yycolumn = 0; + // yycolumn = 0; /** * yy_atBOL == true <=> the scanner is currently at the beginning @@ -111,8 +111,16 @@ public final HeadParserToken getNextToken() throws IOException { String context = null; - context = primGetNextToken(); HeadParserToken result = null; + try { + context = primGetNextToken(); + } + catch (IllegalStateException e) { + hasMore = false; + result = createToken(HTMLHeadTokenizerConstants.UNDEFINED, yychar, yytext()); + while(yy_advance() != YYEOF) {} + return result; + } if (valueText != null) { result = createToken(context, yychar, valueText); valueText = null;
diff --git a/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/skeleton b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/skeleton new file mode 100644 index 0000000..4a9b3f7 --- /dev/null +++ b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/skeleton
@@ -0,0 +1,268 @@ + + /** this character denotes the end of file */ + final public static int YYEOF = -1; + + /** lexical states */ +--- lexical states, charmap + + /* error codes */ + final private static int YY_UNKNOWN_ERROR = 0; + final private static int YY_ILLEGAL_STATE = 1; + final private static int YY_NO_MATCH = 2; + final private static int YY_PUSHBACK_2BIG = 3; + + /* error messages for the codes above */ + final private static String YY_ERROR_MSG[] = { + "Unkown internal scanner error", + "Internal error: unknown state", + "Error: could not match input", + "Error: pushback value was too large" + }; + +--- isFinal list + /** the input device */ + private java.io.Reader yy_reader; + + /** the current state of the DFA */ + private int yy_state; + + /** the current lexical state */ + private int yy_lexical_state = YYINITIAL; + + /** this buffer contains the current text to be matched and is + the source of the yytext() string */ + private char yy_buffer[] = new char[16384]; + + /** the textposition at the last accepting state */ + private int yy_markedPos; + + /** the textposition at the last state to be included in yytext */ + private int yy_pushbackPos; + + /** the current text position in the buffer */ + private int yy_currentPos; + + /** startRead marks the beginning of the yytext() string in the buffer */ + private int yy_startRead; + + /** endRead marks the last character in the buffer, that has been read + from input */ + private int yy_endRead; + + /** number of newlines encountered up to the start of the matched text */ + //private int yyline; + + /** the number of characters up to the start of the matched text */ + private int yychar; + + /** + * the number of characters from the last newline up to the start of the + * matched text + */ + //private int yycolumn; + + /** + * yy_atBOL == true <=> the scanner is currently at the beginning of a line + */ + private boolean yy_atBOL; + + /** yy_atEOF == true <=> the scanner has returned a value for EOF */ + private boolean yy_atEOF; + +--- user class code + + /** + * Creates a new scanner + * There is also a java.io.InputStream version of this constructor. + * + * @param in the java.io.Reader to read input from. + */ +--- constructor declaration + + + /** + * Gets the next input character. + * + * @return the next character of the input stream, EOF if the + * end of the stream is reached. + * @exception IOException if any I/O-Error occurs + */ + private int yy_advance() throws java.io.IOException { + + /* standard case */ + if (yy_currentPos < yy_endRead) return yy_buffer[yy_currentPos++]; + + /* if the eof is reached, we don't need to work hard */ + if (yy_atEOF) return YYEOF; + + /* otherwise: need to refill the buffer */ + + /* first: make room (if you can) */ + if (yy_startRead > 0) { + System.arraycopy(yy_buffer, yy_startRead, + yy_buffer, 0, + yy_endRead-yy_startRead); + + /* translate stored positions */ + yy_endRead-= yy_startRead; + yy_currentPos-= yy_startRead; + yy_markedPos-= yy_startRead; + yy_pushbackPos-= yy_startRead; + yy_startRead = 0; + } + + /* is the buffer big enough? */ + if (yy_currentPos >= yy_buffer.length) { + /* if not: blow it up */ + char newBuffer[] = new char[yy_currentPos*2]; + System.arraycopy(yy_buffer, 0, newBuffer, 0, yy_buffer.length); + yy_buffer = newBuffer; + } + + /* finally: fill the buffer with new input */ + int numRead = yy_reader.read(yy_buffer, yy_endRead, + yy_buffer.length-yy_endRead); + + if ( numRead == -1 ) return YYEOF; + + yy_endRead+= numRead; + + return yy_buffer[yy_currentPos++]; + } + + + /** + * Closes the input stream. + */ + final public void yyclose() throws java.io.IOException { + yy_atEOF = true; /* indicate end of file */ + yy_endRead = yy_startRead; /* invalidate buffer */ + yy_reader.close(); + } + + + /** + * Returns the current lexical state. + */ + final public int yystate() { + return yy_lexical_state; + } + + /** + * Enters a new lexical state + * + * @param newState the new lexical state + */ + final public void yybegin(int newState) { + yy_lexical_state = newState; + } + + + /** + * Returns the text matched by the current regular expression. + */ + final public String yytext() { + return new String( yy_buffer, yy_startRead, yy_markedPos-yy_startRead ); + } + + /** + * Returns the length of the matched text region. + */ + final public int yylength() { + return yy_markedPos-yy_startRead; + } + + + /** + * Reports an error that occured while scanning. + * + * @param errorCode the code of the errormessage to display + */ + private void yy_ScanError(int errorCode) { + try { + IllegalStateException ise = new IllegalStateException("Instance: " + System.identityHashCode(this) + " offset:" + yychar + " state:" + yystate()); + System.out.println(YY_ERROR_MSG[errorCode] + "\n" + ise); + throw ise; + } + catch (ArrayIndexOutOfBoundsException e) { + System.out.println(YY_ERROR_MSG[YY_UNKNOWN_ERROR]); + } + + } + + + /** + * Pushes the specified amount of characters back into the input stream. + * + * They will be read again by then next call of the scanning method + * + * @param number the number of characters to be read again. + * This number must not be greater than yylength()! + */ + private void yypushback(int number) { + if ( number > yylength() ) + yy_ScanError(YY_PUSHBACK_2BIG); + + yy_markedPos -= number; + } + + +--- yy_doEof + /** + * Resumes scanning until the next regular expression is matched, + * the end of input is encountered or an I/O-Error occurs. + * + * @return the next token + * @exception IOException if any I/O-Error occurs + */ +--- yylex declaration + int yy_input; + int yy_action; + +--- local declarations + + while (true) { + +--- start admin (line, char, col count) + yy_action = -1; + + yy_currentPos = yy_startRead = yy_markedPos; + +--- start admin (lexstate etc) + + yy_forAction: { + while (true) { + + yy_input = yy_advance(); + + if ( yy_input == YYEOF ) break yy_forAction; + +--- line, col, char count, next transition, isFinal action + yy_action = yy_state; + yy_markedPos = yy_currentPos; +--- line count update + } + + } + } + +--- char count update + + switch (yy_action) { + +--- actions + default: + if (yy_input == YYEOF && yy_startRead == yy_currentPos) { + yy_atEOF = true; +--- eofvalue + } + else { +--- no match + } + } + } + } + +--- main + +}
diff --git a/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/skeleton.readme b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/skeleton.readme new file mode 100644 index 0000000..eb89700 --- /dev/null +++ b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/HeadParsers/HTMLHeadTokenizer/skeleton.readme
@@ -0,0 +1,6 @@ +The skeleton.sed file contains the modified JFlex 1.2.2 skeleton file with +changes for use with the tokenizers within the org.eclipse.wst.sse.core.xml and +org.eclipse.wst.sse.core.jsp plugins. + +The skeleton file's method definitions are copied into the generated output +directly.