Regenerated lexer; committed patch to add whitetext to fixed form tokens (Bugzilla 284597)
diff --git a/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex b/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex
index 7760749..42ddd05 100644
--- a/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex
+++ b/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex
@@ -52,6 +52,8 @@
%type IToken
%{
+ private IToken prevToken = null;
+
private FixedFormLexerPrepass prepass;
private TokenFactory tokenFactory;
@@ -80,17 +82,73 @@
private IToken token(Terminal terminal)
{
+ //For some there are 2 terminals of type Terminal.END_OF_INPUT that get here in a row
+ // so, technically, the function below sets the whitespaceAfter on the first one it sees,
+ // which is not really the expected behavior, but that token's whitespace is not used for
+ // anything anyway, so this should be OK
+ if(prevToken != null && terminal == Terminal.END_OF_INPUT)
+ {
+ //We need to manually set this, because the input string to the lexer does not
+ // have any whitespace, or at the very list has it trimmed, so we are loosing all
+ // of the trailing whitespace on any refactoring. That is why we assign it as
+ // whitespaceAfter to the last END_OF_STATEMENT token before END_OF_INPUT
+ String whiteAfter = prepass.getTrailingWhitespace();
+ prevToken.setWhiteAfter(whiteAfter);
+ }
+
lastTokenLine = prepass.getLine(yychar)+1;
lastTokenCol = prepass.getColumn(yychar)+1;
lastTokenFileOffset = prepass.getOffset(yychar);
lastTokenStreamOffset = prepass.getOffset(yychar);
lastTokenLength = prepass.getOffset(yychar+yylength()-1)-prepass.getOffset(yychar)+1;
- return tokenFactory.createToken(terminal,
- "",
- terminal == Terminal.T_SCON || terminal == Terminal.T_HCON
- ? stringBuffer.toString()
- : yytext(),
+
+ //For some reason the author of above code needed to add 1 to the line/col values
+ // for my code, I actually need the original value of token positions, so
+ // I added those variables for that
+ int tokLine = lastTokenLine-1;
+ int tokCol = lastTokenCol-1;
+ int tokOff = lastTokenFileOffset;
+
+ String tokenText = "";
+ //If it is the end of statement, use text from original string to get the line separator.
+ // For some reason the text returned by yytext() in this case is always '/n', while the
+ // actual separator is '/r/n'
+ if(terminal == Terminal.T_EOS)
+ {
+ tokenText = prepass.getFileEOL();
+ }
+ //If it is the end of input, use the Lexer's text.
+ else if(terminal == Terminal.END_OF_INPUT)
+ {
+ tokenText = yytext();
+ }
+ //If it is a quote, use text accumulated in the buffer string
+ else if(terminal == Terminal.T_SCON || terminal == Terminal.T_HCON)
+ {
+ tokenText = stringBuffer.toString();
+ //lastTokenColumn and lastTokenOffset of quoted strings actually IS the position of the last character
+ // of that string (for some reason w/o the final quote). So, in order to correctly map it to the whitespace
+ // before it, we need to shift the column and offset we are using by 1 less than the length of the string
+ tokCol = tokCol - (tokenText.length() - 1);
+ tokOff = tokOff - (tokenText.length() - 1);
+ }
+ //Otherwise, use the text directly from file (that way all the whitespace is preserved in the
+ // tokens text)
+ else
+ tokenText = prepass.getTokenText(lastTokenFileOffset, lastTokenLength);
+
+ prevToken = tokenFactory.createToken(terminal,
+ prepass.getWhitespaceBefore(tokLine, tokCol, tokOff),
+ tokenText,
"");
+
+ if(!tokenText.equals(yytext()) && terminal != Terminal.T_EOS)
+ {
+ prevToken.setPreprocessorDirective(tokenText);
+ prevToken.setText(yytext());
+ }
+
+ return prevToken;
}
/*
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
index c73e451..512e8a5 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 5/26/09 7:47 PM */
+/* The following code was generated by JFlex 1.4.1 on 7/27/09 3:59 PM */
/*******************************************************************************
* Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
@@ -44,7 +44,7 @@
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 5/26/09 7:47 PM from the specification file
+ * on 7/27/09 3:59 PM from the specification file
* <tt>FixedFormLexerPhase1.flex</tt>
*/
class FixedFormLexerPhase1 implements ILexer {
@@ -838,6 +838,8 @@
private boolean zzAtEOF;
/* user code: */
+ private IToken prevToken = null;
+
private FixedFormLexerPrepass prepass;
private TokenFactory tokenFactory;
@@ -866,17 +868,73 @@
private IToken token(Terminal terminal)
{
+ //For some there are 2 terminals of type Terminal.END_OF_INPUT that get here in a row
+ // so, technically, the function below sets the whitespaceAfter on the first one it sees,
+ // which is not really the expected behavior, but that token's whitespace is not used for
+ // anything anyway, so this should be OK
+ if(prevToken != null && terminal == Terminal.END_OF_INPUT)
+ {
+ //We need to manually set this, because the input string to the lexer does not
+ // have any whitespace, or at the very list has it trimmed, so we are loosing all
+ // of the trailing whitespace on any refactoring. That is why we assign it as
+ // whitespaceAfter to the last END_OF_STATEMENT token before END_OF_INPUT
+ String whiteAfter = prepass.getTrailingWhitespace();
+ prevToken.setWhiteAfter(whiteAfter);
+ }
+
lastTokenLine = prepass.getLine(yychar)+1;
lastTokenCol = prepass.getColumn(yychar)+1;
lastTokenFileOffset = prepass.getOffset(yychar);
lastTokenStreamOffset = prepass.getOffset(yychar);
lastTokenLength = prepass.getOffset(yychar+yylength()-1)-prepass.getOffset(yychar)+1;
- return tokenFactory.createToken(terminal,
- "",
- terminal == Terminal.T_SCON || terminal == Terminal.T_HCON
- ? stringBuffer.toString()
- : yytext(),
+
+ //For some reason the author of above code needed to add 1 to the line/col values
+ // for my code, I actually need the original value of token positions, so
+ // I added those variables for that
+ int tokLine = lastTokenLine-1;
+ int tokCol = lastTokenCol-1;
+ int tokOff = lastTokenFileOffset;
+
+ String tokenText = "";
+ //If it is the end of statement, use text from original string to get the line separator.
+ // For some reason the text returned by yytext() in this case is always '/n', while the
+ // actual separator is '/r/n'
+ if(terminal == Terminal.T_EOS)
+ {
+ tokenText = prepass.getFileEOL();
+ }
+ //If it is the end of input, use the Lexer's text.
+ else if(terminal == Terminal.END_OF_INPUT)
+ {
+ tokenText = yytext();
+ }
+ //If it is a quote, use text accumulated in the buffer string
+ else if(terminal == Terminal.T_SCON || terminal == Terminal.T_HCON)
+ {
+ tokenText = stringBuffer.toString();
+ //lastTokenColumn and lastTokenOffset of quoted strings actually IS the position of the last character
+ // of that string (for some reason w/o the final quote). So, in order to correctly map it to the whitespace
+ // before it, we need to shift the column and offset we are using by 1 less than the length of the string
+ tokCol = tokCol - (tokenText.length() - 1);
+ tokOff = tokOff - (tokenText.length() - 1);
+ }
+ //Otherwise, use the text directly from file (that way all the whitespace is preserved in the
+ // tokens text)
+ else
+ tokenText = prepass.getTokenText(lastTokenFileOffset, lastTokenLength);
+
+ prevToken = tokenFactory.createToken(terminal,
+ prepass.getWhitespaceBefore(tokLine, tokCol, tokOff),
+ tokenText,
"");
+
+ if(!tokenText.equals(yytext()) && terminal != Terminal.T_EOS)
+ {
+ prevToken.setPreprocessorDirective(tokenText);
+ prevToken.setText(yytext());
+ }
+
+ return prevToken;
}
/*
@@ -1353,41 +1411,47 @@
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_NON_OVERRIDABLE);
}
case 243: break;
- case 22:
- { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): String literal spans multiple lines without continuation");
- }
- case 244: break;
case 96:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_THEN);
}
- case 245: break;
+ case 244: break;
case 123:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_READEQ);
}
- case 246: break;
+ case 245: break;
case 86:
{ storeNonTreeToken();
}
- case 247: break;
+ case 246: break;
case 158:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DIRECTEQ);
}
- case 248: break;
+ case 247: break;
case 87:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NONE);
}
- case 249: break;
+ case 248: break;
case 232:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ASYNCHRONOUSEQ);
}
- case 250: break;
+ case 249: break;
case 139:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IMPORT);
}
- case 251: break;
+ case 250: break;
case 207:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INTRINSIC);
}
+ case 251: break;
+ case 43:
+ { stringBuffer = new StringBuffer();
+ String text = yytext();
+ stringBuffer.append(text);
+ hollerithLength=Integer.parseInt(text.substring(0,text.length()-1));
+ if (hollerithLength==0)
+ throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Invalid length of hollerith literal: 0");
+ yybegin(HOLLERITH);
+ }
case 252: break;
case 62:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ERREQ);
@@ -1409,161 +1473,157 @@
{ wantEos = true; unsetSOL(); return token(Terminal.T_LPAREN);
}
case 257: break;
- case 26:
- { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Hollerith literal spans multiple lines without continuation");
- }
- case 258: break;
case 157:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DEFAULT);
}
- case 259: break;
+ case 258: break;
case 46:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RCON);
}
- case 260: break;
+ case 259: break;
case 227:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDBLOCKDATA);
}
- case 261: break;
+ case 260: break;
case 24:
{ stringBuffer.append('\"');
yybegin(YYSTANDARD);
wantEos = true;
return token(Terminal.T_SCON);
}
- case 262: break;
+ case 261: break;
case 30:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DO);
}
- case 263: break;
+ case 262: break;
case 128:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_VALUE);
}
- case 264: break;
+ case 263: break;
case 60:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ENUM);
}
- case 265: break;
+ case 264: break;
case 108:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BLOCK);
}
- case 266: break;
+ case 265: break;
case 20:
{ wantEos = true; unsetSOL(); return token(Terminal.T_SLASH);
}
- case 267: break;
+ case 266: break;
case 84:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_PASS);
}
- case 268: break;
+ case 267: break;
case 138:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INTENT);
}
- case 269: break;
+ case 268: break;
case 233:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_NON_INTRINSIC);
}
- case 270: break;
+ case 269: break;
case 99:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_KIND);
}
- case 271: break;
+ case 270: break;
case 3:
{ wantEos = true; unsetSOL(); return token(Terminal.T_ASTERISK);
}
- case 272: break;
+ case 271: break;
case 181:
{ wantEos = true; yybegin(OPERATORorFORMAT); return token(Terminal.T_OPERATOR);
}
- case 273: break;
+ case 272: break;
case 162:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ACTIONEQ);
}
- case 274: break;
+ case 273: break;
case 92:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CALL);
}
- case 275: break;
+ case 274: break;
case 187:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FUNCTION);
}
- case 276: break;
+ case 275: break;
case 205:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IOLENGTHEQ);
}
- case 277: break;
+ case 276: break;
case 11:
{ wantEos = true; unsetSOL(); return token(Terminal.T_COMMA);
}
- case 278: break;
+ case 277: break;
case 144:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_NOPASS);
}
- case 279: break;
+ case 278: break;
case 66:
{ wantEos = true; unsetSOL(); return token(Terminal.T_GE);
}
- case 280: break;
+ case 279: break;
case 193:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CONTINUE);
}
- case 281: break;
+ case 280: break;
case 19:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PERCENT);
}
- case 282: break;
+ case 281: break;
case 50:
{ stringBuffer.append('\"');
}
- case 283: break;
+ case 282: break;
case 175:
{ wantEos = true; unsetSOL(); return token(Terminal.T_COMPLEX);
}
- case 284: break;
+ case 283: break;
case 90:
{ wantEos = true; unsetSOL(); return token(Terminal.T_LENEQ);
}
- case 285: break;
+ case 284: break;
case 4:
{ wantEos = true; unsetSOL(); return token(Terminal.T_ICON);
}
- case 286: break;
+ case 285: break;
case 104:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENTRY);
}
- case 287: break;
+ case 286: break;
case 223:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDFUNCTION);
}
- case 288: break;
+ case 287: break;
case 168:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_GENERIC);
}
- case 289: break;
+ case 288: break;
case 78:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FMTEQ);
}
- case 290: break;
+ case 289: break;
case 174:
{ wantEos = true; unsetSOL(); return token(Terminal.T_LOGICAL);
}
- case 291: break;
+ case 290: break;
case 5:
{ wantEos = true; unsetSOL(); return token(Terminal.T_PLUS);
}
- case 292: break;
+ case 291: break;
case 186:
{ wantEos = true; yybegin(IMPLICIT); return token(Terminal.T_IMPLICIT);
}
- case 293: break;
+ case 292: break;
case 91:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CASE);
}
- case 294: break;
+ case 293: break;
case 77:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FILE);
}
- case 295: break;
+ case 294: break;
case 25:
{ hollerithLength--;
stringBuffer.append(yytext());
@@ -1573,22 +1633,26 @@
return token(Terminal.T_HCON);
}
}
- case 296: break;
+ case 295: break;
case 41:
{ wantEos = true; unsetSOL(); return token(Terminal.T_SLASHEQ);
}
- case 297: break;
+ case 296: break;
case 132:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DELIMEQ);
}
- case 298: break;
+ case 297: break;
case 69:
{ wantEos = true; unsetSOL(); return token(Terminal.T_LE);
}
- case 299: break;
+ case 298: break;
case 171:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PRIVATE);
}
+ case 299: break;
+ case 26:
+ { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Hollerith literal spans multiple lines without continuation");
+ }
case 300: break;
case 98:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_WAIT);
@@ -1606,526 +1670,520 @@
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WHILE);
}
case 304: break;
+ case 22:
+ { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): String literal spans multiple lines without continuation");
+ }
+ case 305: break;
case 65:
{ wantEos = true; unsetSOL(); return token(Terminal.T_OR);
}
- case 305: break;
+ case 306: break;
case 200:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELSEWHERE);
}
- case 306: break;
+ case 307: break;
case 27:
{ wantEos = true; unsetSOL(); return token(Terminal.T_UNEXPECTED_CHARACTER);
}
- case 307: break;
+ case 308: break;
case 134:
{ wantEos = true; unsetSOL(); return token(Terminal.T_NEQV);
}
- case 308: break;
+ case 309: break;
case 124:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RECLEQ);
}
- case 309: break;
+ case 310: break;
case 170:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PROGRAM);
}
- case 310: break;
+ case 311: break;
case 42:
{ wantEos = true; unsetSOL(); return token(Terminal.T_SLASHSLASH);
}
- case 311: break;
+ case 312: break;
case 164:
{ wantEos = true; unsetSOL(); return token(Terminal.T_INTEGER);
}
- case 312: break;
+ case 313: break;
case 122:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNITEQ);
}
- case 313: break;
+ case 314: break;
case 89:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NMLEQ);
}
- case 314: break;
+ case 315: break;
case 15:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EQUALS);
}
- case 315: break;
+ case 316: break;
case 209:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PARAMETER);
}
- case 316: break;
+ case 317: break;
case 7:
{ stringBuffer = new StringBuffer();
stringBuffer.append('\'');
yybegin(QUOTED);
}
- case 317: break;
+ case 318: break;
case 215:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDPROGRAM);
}
- case 318: break;
+ case 319: break;
case 222:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_READWRITEEQ);
}
- case 319: break;
+ case 320: break;
case 160:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OPENEDEQ);
}
- case 320: break;
+ case 321: break;
case 29:
{ wantEos = true; unsetSOL(); return token(Terminal.T_POW);
}
- case 321: break;
+ case 322: break;
case 113:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SIZEEQ);
}
- case 322: break;
+ case 323: break;
case 167:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_STREAMEQ);
}
- case 323: break;
+ case 324: break;
case 71:
{ wantEos = true; unsetSOL(); return token(Terminal.T_BCON);
}
- case 324: break;
+ case 325: break;
case 14:
{ wantEos = true; return token(Terminal.T_RBRACKET);
}
- case 325: break;
+ case 326: break;
case 197:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDMODULE);
}
- case 326: break;
+ case 327: break;
case 196:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDSELECT);
}
- case 327: break;
+ case 328: break;
case 189:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_PENDINGEQ);
}
- case 328: break;
+ case 329: break;
case 133:
{ wantEos = true; unsetSOL(); return token(Terminal.T_DOUBLE);
}
- case 329: break;
+ case 330: break;
case 18:
{ wantEos = true; unsetSOL(); return token(Terminal.T_LESSTHAN);
}
- case 330: break;
+ case 331: break;
case 190:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NEXTRECEQ);
}
- case 331: break;
+ case 332: break;
case 185:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ALLOCATE);
}
- case 332: break;
+ case 333: break;
case 83:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PADEQ);
}
- case 333: break;
+ case 334: break;
case 172:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NULLIFY);
}
- case 334: break;
+ case 335: break;
case 204:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ASSOCIATE);
}
- case 335: break;
+ case 336: break;
case 2:
{ yybegin(YYINITIAL); boolean b = wantEos; wantEos = false; if (b) return token(Terminal.T_EOS); else storeNonTreeToken();
}
- case 336: break;
+ case 337: break;
case 105:
{ wantEos = true; unsetSOL(); return token(Terminal.T_EQV);
}
- case 337: break;
+ case 338: break;
case 156:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDTYPE);
}
- case 338: break;
+ case 339: break;
case 182:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OPTIONAL);
}
- case 339: break;
+ case 340: break;
case 188:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SEQUENCE);
}
- case 340: break;
+ case 341: break;
case 103:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDIF);
}
- case 341: break;
+ case 342: break;
case 81:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_GOTO);
}
- case 342: break;
+ case 343: break;
case 55:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_LEN);
}
- case 343: break;
+ case 344: break;
case 155:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDFILE);
}
- case 344: break;
+ case 345: break;
case 140:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FORALL);
}
- case 345: break;
+ case 346: break;
case 54:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IDEQ);
}
- case 346: break;
+ case 347: break;
case 58:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EXIT);
}
- case 347: break;
+ case 348: break;
case 219:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FORMATTEDEQ);
}
- case 348: break;
+ case 349: break;
case 23:
{ stringBuffer.append('\'');
yybegin(YYSTANDARD);
wantEos = true;
return token(Terminal.T_SCON);
}
- case 349: break;
+ case 350: break;
case 107:
{ wantEos = true; unsetSOL(); return token(Terminal.T_NOT);
}
- case 350: break;
+ case 351: break;
case 57:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DCON);
}
- case 351: break;
+ case 352: break;
case 210:
{ wantEos = true; unsetSOL(); return token(Terminal.T_PRECISION);
}
- case 352: break;
+ case 353: break;
case 74:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OPEN);
}
- case 353: break;
+ case 354: break;
case 115:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_STATEQ);
}
- case 354: break;
+ case 355: break;
case 28:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IDENT);
}
- case 355: break;
+ case 356: break;
case 148:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RETURN);
}
- case 356: break;
+ case 357: break;
case 184:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ABSTRACT);
}
- case 357: break;
+ case 358: break;
case 39:
{ wantEos = true; unsetSOL(); return token(Terminal.T_LESSTHANEQ);
}
- case 358: break;
+ case 359: break;
case 220:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SELECTCASE);
}
- case 359: break;
+ case 360: break;
case 218:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ASSIGNMENT);
}
- case 360: break;
+ case 361: break;
case 213:
{ wantEos = true; unsetSOL(); return token(Terminal.T_CHARACTER);
}
- case 361: break;
+ case 362: break;
case 40:
{ wantEos = true; return token(Terminal.T_NE);
}
- case 362: break;
+ case 363: break;
case 36:
{ wantEos = true; unsetSOL(); return token(Terminal.T_EQEQ);
}
- case 363: break;
+ case 364: break;
case 8:
{ stringBuffer = new StringBuffer();
stringBuffer.append('\"');
yybegin(DBLQUOTED);
}
- case 364: break;
+ case 365: break;
case 180:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_DECIMALEQ);
}
- case 365: break;
+ case 366: break;
case 67:
{ wantEos = true; unsetSOL(); return token(Terminal.T_GT);
}
- case 366: break;
+ case 367: break;
case 137:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IOMSGEQ);
}
- case 367: break;
+ case 368: break;
case 153:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WRITEEQ);
}
- case 368: break;
+ case 369: break;
case 80:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_STOP);
}
- case 369: break;
+ case 370: break;
case 141:
{ wantEos = true; yybegin(OPERATORorFORMAT); return token(Terminal.T_FORMAT);
}
- case 370: break;
+ case 371: break;
case 53:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OUT);
}
- case 371: break;
+ case 372: break;
case 146:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_COMMON);
}
- case 372: break;
+ case 373: break;
case 119:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CLOSE);
}
- case 373: break;
+ case 374: break;
case 35:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_TO);
}
- case 374: break;
+ case 375: break;
case 166:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_STATUSEQ);
}
- case 375: break;
+ case 376: break;
case 44:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PCON);
}
- case 376: break;
+ case 377: break;
case 21:
{ stringBuffer.append( yytext() );
}
- case 377: break;
+ case 378: break;
case 178:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDWHERE);
}
- case 378: break;
+ case 379: break;
case 9:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNDERSCORE);
}
- case 379: break;
+ case 380: break;
case 93:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_READ);
}
- case 380: break;
+ case 381: break;
case 127:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WRITE);
}
- case 381: break;
+ case 382: break;
case 76:
{ wantEos = true; unsetSOL(); return token(Terminal.T_ZCON);
}
- case 382: break;
+ case 383: break;
case 225:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ALLOCATABLE);
}
- case 383: break;
+ case 384: break;
case 6:
{ wantEos = true; unsetSOL(); return token(Terminal.T_MINUS);
}
- case 384: break;
+ case 385: break;
case 199:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELEMENTAL);
}
- case 385: break;
+ case 386: break;
case 221:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SUBROUTINE);
}
- case 386: break;
+ case 387: break;
case 202:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BACKSPACE);
}
- case 387: break;
+ case 388: break;
case 38:
{ wantEos = true; unsetSOL(); return token(Terminal.T_GREATERTHANEQ);
}
- case 388: break;
+ case 389: break;
case 70:
{ wantEos = true; unsetSOL(); return token(Terminal.T_LT);
}
- case 389: break;
+ case 390: break;
case 64:
{ wantEos = true; unsetSOL(); return token(Terminal.T_EQ);
}
- case 390: break;
+ case 391: break;
case 97:
{ wantEos = true; unsetSOL(); return token(Terminal.T_TYPE);
}
- case 391: break;
+ case 392: break;
case 165:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INQUIRE);
}
- case 392: break;
+ case 393: break;
case 206:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INTERFACE);
}
- case 393: break;
+ case 394: break;
case 61:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELSE);
}
- case 394: break;
+ case 395: break;
case 45:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_XCON);
}
- case 395: break;
+ case 396: break;
case 173:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NUMBEREQ);
}
- case 396: break;
+ case 397: break;
case 129:
{ wantEos = true; unsetSOL(); return token(Terminal.T_KINDEQ);
}
- case 397: break;
+ case 398: break;
case 230:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNFORMATTEDEQ);
}
- case 398: break;
+ case 399: break;
case 95:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RECEQ);
}
- case 399: break;
+ case 400: break;
case 234:
{ wantEos = true; unsetSOL(); return token(Terminal.T_DOUBLEPRECISION);
}
- case 400: break;
+ case 401: break;
case 201:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DIMENSION);
}
- case 401: break;
+ case 402: break;
case 68:
{ wantEos = true; unsetSOL(); return token(Terminal.T_NE);
}
- case 402: break;
+ case 403: break;
case 47:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_LPARENSLASH);
}
- case 403: break;
+ case 404: break;
case 33:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IN);
}
- case 404: break;
+ case 405: break;
case 135:
{ wantEos = true; unsetSOL(); return token(Terminal.T_TRUE);
}
- case 405: break;
+ case 406: break;
case 125:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WHERE);
}
- case 406: break;
+ case 407: break;
case 161:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ACCESSEQ);
}
- case 407: break;
+ case 408: break;
case 208:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_POSITIONEQ);
}
- case 408: break;
+ case 409: break;
case 228:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDINTERFACE);
}
- case 409: break;
+ case 410: break;
case 16:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_COLON);
}
- case 410: break;
+ case 411: break;
case 85:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PURE);
}
- case 411: break;
+ case 412: break;
case 211:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PROCEDURE);
}
- case 412: break;
+ case 413: break;
case 63:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DATA);
}
- case 413: break;
+ case 414: break;
case 152:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_TARGET);
}
- case 414: break;
+ case 415: break;
case 34:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_GO);
}
- case 415: break;
+ case 416: break;
case 73:
{ wantEos = true; unsetSOL(); return token(Terminal.T_OCON);
}
- case 416: break;
+ case 417: break;
case 131:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELSEIF);
}
- case 417: break;
+ case 418: break;
case 118:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NAMEEQ);
}
- case 418: break;
+ case 419: break;
case 216:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ENUMERATOR);
}
- case 419: break;
+ case 420: break;
case 13:
{ wantEos = true; return token(Terminal.T_LBRACKET);
}
- case 420: break;
+ case 421: break;
case 116:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PAUSE);
}
- case 421: break;
+ case 422: break;
case 31:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IF);
}
- case 422: break;
+ case 423: break;
case 32:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IS);
}
- case 423: break;
+ case 424: break;
case 102:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDDO);
}
- case 424: break;
+ case 425: break;
case 109:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INOUT);
}
- case 425: break;
+ case 426: break;
case 52:
{ wantEos = true; unsetSOL(); return token(Terminal.T_XDOP);
}
- case 426: break;
+ case 427: break;
case 88:
{ wantEos = true; unsetSOL(); return token(Terminal.T_NULL);
}
- case 427: break;
+ case 428: break;
case 121:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CYCLE);
}
- case 428: break;
+ case 429: break;
case 136:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BLANKEQ);
}
- case 429: break;
+ case 430: break;
case 72:
{ wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_BIND);
}
- case 430: break;
- case 43:
- { stringBuffer = new StringBuffer();
- String text = yytext();
- stringBuffer.append(text);
- hollerithLength=Integer.parseInt(text.substring(0,text.length()-1));
- if (hollerithLength==0)
- throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Invalid length of hollerith literal: 0");
- yybegin(HOLLERITH);
- }
case 431: break;
case 117:
{ wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PRINT);
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
index a06424b..6e74740 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
@@ -31,6 +31,7 @@
public FixedFormLexerPhase2(InputStream in, IFile file, String filename, TokenFactory tokenFactory)
{
+ in = new LineAppendingInputStream(in);
final FixedFormLexerPrepass prepass = new FixedFormLexerPrepass(in);
InputStream prepassReader = new InputStream()
{
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java
index 366e084..b1c0f57 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java
@@ -15,6 +15,8 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.util.HashMap;
+import java.util.Iterator;
/**
* FixedFormLexerPrepass preprocesses the input stream. It discards all
@@ -31,6 +33,9 @@
static final int inDblQuoteEnd=3;
static final int inQuote=4;
static final int inQuoteEnd=5;
+
+ //private String fileContent = "";
+ StringBuilder strBuilder = new StringBuilder();
private int state = inStart;
int hollerithLength = -2; //-1: hollerith could start, -2: hollerith cant start
@@ -43,6 +48,11 @@
private DynamicIntArray columnMapping = new DynamicIntArray(1000);
private DynamicIntArray offsetMapping = new DynamicIntArray(1000);
+ //Maps whitespace(string) to position in file (line, col, offset) tuple
+ private HashMap whiteSpaceMapping = new HashMap();
+ //Used to accumulate whitespace between lines (multi-line comments, etc) because
+ // the string is processed on per-line basis
+ private String prevWhiteSpace = "";
private int EOFLinePos=0;
private int EOFColPos=0;
@@ -74,42 +84,90 @@
int lastCharPos = lineMapping.length() - 1;
return offsetMapping.get(Math.min(absCharPos, lastCharPos));
}
-
+
public int read() throws Exception {
int c = internalRead();
//System.out.print((char)c);
return c;
}
-
+
+ public String getTokenText(int offset, int length)
+ {
+ int strLen = strBuilder.length();
+ if(offset >= 0 && length > 0 && offset < strLen && length <= strLen && offset+length < strLen)
+ {
+ String res = strBuilder.substring(offset, offset+length);
+ return res;
+ }
+ return "";
+ }
+
+ public String getTokenText(int offset)
+ {
+ if(offset >= 0 && offset < strBuilder.length())
+ {
+ String res = strBuilder.substring(offset);
+ return res;
+ }
+ return "";
+ }
+
+ public String getTrailingWhitespace()
+ {
+ String trimmed = strBuilder.toString().trim();
+ //This gets the index of the beginning of the whitespace AFTER the first "end of line" symbol on the last line
+ // with actual text of the file
+ int start = strBuilder.indexOf(trimmed) + trimmed.length() + in.getFileEOL().length();
+ String res = strBuilder.substring(start);
+ return res;
+ }
+
private void markPosition(int line, int col, int offset) {
offsetMapping.pushBack(offset);
lineMapping.pushBack(line);
columnMapping.pushBack(col);
}
- private int internalRead() throws Exception {
-
- for (;;) {
- if (actLine==null) {
- actLine=getNextNonCommentLine();
- if (actLine==null) {
+ private int internalRead() throws Exception
+ {
+ PreLexerLine prevLine = null;
+ for (;;)
+ {
+ if (actLine==null)
+ {
+ actLine=getNextLine();
+ if (actLine==null)
+ {
markPosition(EOFLinePos,EOFColPos,EOFOffsetPos);
return -1;
}
- else actLinePos=0;
- } else if (actLinePos==actLine.length()
- || actLinePos==PreLexerLine.COLWIDTH) { //test if continuation-line follows, else send \n
-
- PreLexerLine prevLine=actLine;
- actLine=getNextNonCommentLine();
- if (actLine==null) { //end of file
+ else
+ actLinePos=0;
+ }
+ else if(actLinePos==actLine.length() ||
+ actLinePos==PreLexerLine.COLWIDTH)
+ { //test if continuation-line follows, else send \n
+ prevLine=actLine;
+ actLine=getNextLine();
+ if (actLine==null)
+ { //end of file
hollerithLength=-2;
state=inStart;
markPosition(prevLine.linePos,actLinePos,prevLine.offset+actLinePos+1);
return '\n';
- } else if (actLine.type==PreLexerLine.CONTINUATION) {
+ }
+ else if (actLine.type==PreLexerLine.CONTINUATION)
+ {
actLinePos=6;
- } else {
+ }
+ else if(actLine.type==PreLexerLine.COMMENT)
+ {
+ prevWhiteSpace=prevWhiteSpace.concat(actLine.getText());
+ prevWhiteSpace=prevWhiteSpace.concat(in.getFileEOL());
+ actLinePos = actLine.length();
+ }
+ else
+ {
actLinePos=0;
hollerithLength=-2;
state=inStart;
@@ -119,109 +177,286 @@
}
actLinePos = getNextSigPos(actLine,actLinePos);
- //TODO: save non-tree tokens
- if (actLinePos<0) {
+ if (actLinePos<0)
+ {
actLinePos=actLine.length();
- } else {
+ }
+ else
+ {
markPosition(actLine.linePos,actLinePos,actLine.offset+actLinePos);
return actLine.charAt(actLinePos++);
}
}
}
- private PreLexerLine getNextNonCommentLine() {
- for (;;) {
- PreLexerLine line = getNextLine();
- if (line==null) return null;
- if (line.type!=PreLexerLine.COMMENT) return line;
- else {
- //TODO: save non-tree tokens
- }
- }
- }
-
+// private PreLexerLine getNextNonCommentLine() {
+// for (;;) {
+// PreLexerLine line = getNextLine();
+// if (line==null) return null;
+// if (line.type!=PreLexerLine.COMMENT) return line;
+// else {
+// //TODO: save non-tree tokens
+// }
+// }
+// }
+
private PreLexerLine getNextLine() {
try {
int actOffset=in.getOffset();
String line = in.readLine();
- if (line==null) return null;
- EOFLinePos=in.getLineNumber()-1;
- EOFColPos=line.length();
- EOFOffsetPos=actOffset;
+ if (line==null)
+ return null;
+
+ strBuilder.append(line);
+ strBuilder.append(in.getFileEOL());
+ //fileContent = fileContent.concat(line);
+ //fileContent = fileContent.concat(in.getFileEOL());
+ EOFLinePos=in.getLineNumber()+1;//-1; //Move that token past the last line
+ EOFColPos=0;//line.length();
+ EOFOffsetPos=actOffset+line.length()+in.getFileEOL().length();//To accomodate for End-of-line statement
PreLexerLine pll = new PreLexerLine(line,in.getLineNumber()-1,actOffset);
return pll;
} catch (IOException e) {
return null;
}
}
-
+
+ private boolean isWhitespace(char c)
+ {
+ return c==' ' || c=='\t' || c=='\r' || c=='\n';
+ }
+
+ public String getWhitespaceBefore(int ln, int lastCol, int lastOffset)
+ {
+ int colBefore = lastCol;
+ int offsetBefore = lastOffset;
+
+ if(colBefore < 0 || offsetBefore < 0)
+ return "";
+
+ //Create a positionInFile object, with line,col and offset set to the END of the potential whitespace
+ PositionInFile posInFile = new PositionInFile(ln, colBefore, offsetBefore, false);
+ String result = (String)whiteSpaceMapping.get(posInFile);
+ /* Iterator iter = whiteSpaceMapping.keySet().iterator();
+ while(iter.hasNext())
+ {
+ PositionInFile temp = (PositionInFile)iter.next();
+ if(posInFile.isSameEnd(temp))
+ {
+ return (String)whiteSpaceMapping.get(temp);
+ }
+ }*/
+ if(result==null)
+ return "";
+
+ return result;
+ }
+
+ private int extractWhitespace(PreLexerLine line, int startPos)
+ {
+ String whiteAgg = "";
+ int charPos = startPos;
+ int startWhitespace = -1;
+ int length = line.length();
+
+ if(line.type == PreLexerLine.COMMENT)
+ {
+ if(startWhitespace == -1)
+ startWhitespace = charPos;
+ //Append current line to prevWhiteSpace
+ prevWhiteSpace = prevWhiteSpace.concat(line.getText().substring(charPos));
+ //Since PreLexerLine throws away whitespace, attach a "new line" character to our whitespace
+ prevWhiteSpace = prevWhiteSpace.concat(in.getFileEOL());
+ charPos = -1; //Finished line
+ //Don't insert the white-space because full-line comments are associated with whatever token
+ // you find on the NEXT line, so don't add them to the map yet
+ return charPos;
+ }
+
+ for(;charPos<line.length();charPos++)
+ {
+ char c = line.charAt(charPos);
+ //If it is a continuation line, treat character at position 6 as whitespace
+ if(line.type == PreLexerLine.CONTINUATION && charPos == 6)
+ {
+ if(startWhitespace == -1)
+ startWhitespace = 0;
+ String prevLineWhite = in.getFileEOL().concat(line.getText().substring(0, 6));
+ whiteAgg = whiteAgg.concat(prevLineWhite);
+ }
+
+ if(isWhitespace(c))
+ {
+ if(startWhitespace == -1)
+ startWhitespace = charPos;
+ whiteAgg = whiteAgg.concat(String.valueOf(c));
+ }
+ else if(c=='!' || charPos >= PreLexerLine.COLWIDTH) //It a comment, grab the rest of the line
+ {
+ if(startWhitespace == -1)
+ startWhitespace = charPos;
+ whiteAgg = whiteAgg.concat(line.getText().substring(charPos));
+ charPos = length; //Finished line
+ break;
+ }
+ else //Not a whitespace character
+ {
+ break;
+ }
+ }
+ if((whiteAgg.length() != 0 || prevWhiteSpace.length() != 0) && startWhitespace != -1)
+ {
+ PositionInFile posInFile = new PositionInFile(line.linePos,
+ startWhitespace,
+ charPos,
+ line.offset+startWhitespace,
+ line.offset+charPos);
+ String combinedWhite = prevWhiteSpace.concat(whiteAgg);
+ whiteSpaceMapping.put(posInFile, combinedWhite);
+ prevWhiteSpace = "";
+
+ //If we moved into the comments, return -1 since we gobbled those up
+ if(charPos >= PreLexerLine.COLWIDTH)
+ charPos = -1;
+ }
+ if(charPos >= length) //If we "gobbled up" the entire line, return -1 to
+ return -1; //signify the end of the line
+
+ return charPos;
+ }
+
// return: -1 : end of line reached
private int getNextSigPos(PreLexerLine line, int startPos) {
- for (int charPos=startPos;charPos<line.length();++charPos) {
+ for (int charPos=startPos;charPos<line.length();++charPos)
+ {
char c = line.charAt(charPos);
- if (line.type==PreLexerLine.CPPDIRECTIVE) return charPos;
- if (line.type==PreLexerLine.CONTINUATION && charPos<=5) continue;
+ if (line.type==PreLexerLine.CPPDIRECTIVE)
+ return charPos;
+ if (line.type==PreLexerLine.CONTINUATION && charPos<=5)
+ continue;
- if (state==inStart) {
- if (c==' ' || c=='\t') continue;
- else if (c=='!') {
- return -1; // rest of line is comment
- } else if (charPos<=4 && !Character.isDigit(c)) continue; //only allow digits(label) in column 0-4
- else if (c=='\'') {
+ //A bit ugly. This pretty much goes through and stores
+ // all the whitespace from a given character until the
+ // first non-whitespace character in a map PositionInFile -> String,
+ // so that it can later be attached to appropriate tokens.
+ if (state==inStart &&
+ (
+ isWhitespace(c) ||
+ c=='!' ||
+ line.type == PreLexerLine.COMMENT ||
+ charPos >= PreLexerLine.COLWIDTH ||
+ line.type == PreLexerLine.CONTINUATION
+ ))
+ {
+ charPos = extractWhitespace(line, charPos);
+ //If we got to the end of the line, no need to continue
+ if(charPos >= 0 && charPos < line.length())
+ c = line.charAt(charPos);
+ else
+ return -1;
+ }
+
+ if (state==inStart)
+ {
+ if (charPos<=4 && !Character.isDigit(c))
+ continue; //only allow digits(label) in column 0-4
+ else if (c=='\'')
+ {
hollerithLength=-1;
state=inQuote;
- } else if (c=='\"') {
+ }
+ else if (c=='\"')
+ {
hollerithLength=-1;
state=inDblQuote;
- } else if ((c=='h') || (c=='H')) {
- if (hollerithLength>0) state=inHollerith;
- else if (hollerithLength<0) hollerithLength=-2;
- } else if (hollerithLength!=-2 && Character.isDigit(c)) {
- if (hollerithLength==-1) hollerithLength=Character.digit(c,10);
- else hollerithLength=hollerithLength*10+Character.digit(c,10);
- } else if (Character.isLetter(c) || c=='_') {
+ }
+ else if ((c=='h') || (c=='H'))
+ {
+ if (hollerithLength>0)
+ state=inHollerith;
+ else if (hollerithLength<0)
+ hollerithLength=-2;
+ }
+ else if (hollerithLength!=-2 && Character.isDigit(c))
+ {
+ if (hollerithLength==-1)
+ hollerithLength=Character.digit(c,10);
+ else
+ hollerithLength=hollerithLength*10+Character.digit(c,10);
+ }
+ else if (Character.isLetter(c) || c=='_')
+ {
hollerithLength=-2;
- } else {
- if (charPos==0) hollerithLength=-2;// ignore label at start of line
- else hollerithLength=-1;
+ }
+ else
+ {
+ if (charPos==0)
+ hollerithLength=-2;// ignore label at start of line
+ else
+ hollerithLength=-1;
}
return charPos;
- } else if (state==inQuote) {
- if (c=='\'') state=inQuoteEnd;
+ }
+ else if (state==inQuote)
+ {
+ if (c=='\'')
+ state=inQuoteEnd;
return charPos;
- } else if (state==inQuoteEnd) {
- if (c=='\'') {
+ } else if (state==inQuoteEnd)
+ {
+ if (c=='\'')
+ {
state=inQuote;
return charPos;
- } else {
+ }
+ else
+ {
state=inStart;
charPos--;
}
- } else if (state==inDblQuote) {
- if (c=='\"') state=inQuoteEnd;
+ }
+ else if (state==inDblQuote)
+ {
+ if (c=='\"')
+ state=inQuoteEnd;
return charPos;
- } else if (state==inDblQuoteEnd) {
- if (c=='\"') {
+ }
+ else if (state==inDblQuoteEnd)
+ {
+ if (c=='\"')
+ {
state=inDblQuote;
return charPos;
- } else {
+ }
+ else
+ {
state=inStart;
charPos--;
}
- } else if (state==inHollerith) {
+ }
+ else if (state==inHollerith)
+ {
hollerithLength--;
- if (hollerithLength==0) state=inStart;
+ if (hollerithLength==0)
+ state=inStart;
return charPos;
- } else { //undefined state
+ }
+ else
+ { //undefined state
throw new RuntimeException("Undefined state in FixedFormPreLexer");
}
}
return -1; //end of line reached
}
+
+ public String getFileEOL()
+ {
+ return in.getFileEOL();
+ }
}
class DynamicIntArray {
@@ -286,11 +521,12 @@
this.offset=offset;
/* truncate anything beyond 72 characters */
- if (_lineText.length()>COLWIDTH) {
+ /*if (_lineText.length()>COLWIDTH) {
lineText=_lineText.substring(0,COLWIDTH);
} else {
lineText=_lineText;
- }
+ }*/
+ lineText = _lineText;
String trimmedText=lineText.trim();
@@ -331,18 +567,106 @@
return lineText.charAt(pos);
}
+ public String getText()
+ {
+ return this.lineText;
+ }
public String toString() {
return "Line "+linePos+": "+lineText;
}
}
+class PositionInFile
+{
+ private int line = -1;
+ private int startCol = -1;
+ private int startOffset = -1;
+ private int endCol = -1;
+ private int endOffset = -1;
+
+ public PositionInFile(int ln, int stCl, int endCl, int stOfst, int endOfst)
+ {
+ this.line = ln;
+ this.startCol = stCl;
+ this.endCol = endCl;
+ this.startOffset = stOfst;
+ this.endOffset = endOfst;
+ }
+
+ public PositionInFile(int ln, int cl, int ofst, boolean isStart)
+ {
+ this.line = ln;
+ if(isStart)
+ {
+ this.startCol = cl;
+ this.startOffset = ofst;
+ }
+ else
+ {
+ this.endCol = cl;
+ this.endOffset = ofst;
+ }
+ }
+
+ public int getLine()
+ {
+ return this.line;
+ }
+
+ public int getStartCol()
+ {
+ return this.startCol;
+ }
+
+ public int getStartOffset()
+ {
+ return this.startOffset;
+ }
+
+ public int getEndCol()
+ {
+ return this.startCol;
+ }
+
+ public int getEndOffset()
+ {
+ return this.startOffset;
+ }
+
+ public boolean isSameStart(PositionInFile other)
+ {
+ return (other.line == this.line &&
+ other.startCol == this.startCol &&
+ other.startOffset == this.startOffset);
+ }
+
+ public boolean isSameEnd(PositionInFile other)
+ {
+ return (other.line == this.line &&
+ other.endCol == this.endCol &&
+ other.endOffset == this.endOffset);
+ }
+
+ //Override
+ public int hashCode()
+ {
+ return this.endOffset;
+ }
+
+ public boolean equals(Object obj)
+ {
+ return ((PositionInFile)obj).endOffset == this.endOffset;
+ }
+}
class OffsetLineReader {
private BufferedReader bReader;
private StringBuffer sBuf;
private int lineNumber=0;
private int offset=0;
+ private String fileEOL = null;
+
private int charBuf=-1;
@@ -366,12 +690,21 @@
sBuf.setLength(0);
while(true) {
if (charBuf=='\n') {
+ if (fileEOL == null) fileEOL = "\n";
charBuf=getNextChar();
lineNumber++;
break;
} else if (charBuf=='\r') {
charBuf=getNextChar();
- if (charBuf=='\n') charBuf=getNextChar();
+ if (charBuf=='\n')
+ {
+ if (fileEOL == null) fileEOL = "\r\n";
+ charBuf=getNextChar();
+ }
+ else
+ {
+ if (fileEOL == null) fileEOL = "\r";
+ }
lineNumber++;
break;
} else if (charBuf==-1) {
@@ -392,4 +725,8 @@
if (charBuf<0) return offset;
else return offset-1;
}
+
+ public String getFileEOL() {
+ return fileEOL;
+ }
}
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java
index 8ce70e3..de777af 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 7/24/09 10:51 AM */
+/* The following code was generated by JFlex 1.4.1 on 7/27/09 3:59 PM */
/*******************************************************************************
* Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
@@ -39,7 +39,7 @@
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 7/24/09 10:51 AM from the specification file
+ * on 7/27/09 3:59 PM from the specification file
* <tt>FreeFormLexerPhase1.flex</tt>
*/
public class FreeFormLexerPhase1 implements ILexer {
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/IToken.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/IToken.java
index 9ec60ae..8f8b2db 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/IToken.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/IToken.java
@@ -82,6 +82,9 @@
public int getLength();
public void setLength(int length);
+ public String getPreprocessorDirective();
+ public void setPreprocessorDirective(String preprocessorDirective);
+
// public Object getBinding()
// {
// return binding;
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/SimpleToken.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/SimpleToken.java
index 4b1e720..7e82c99 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/SimpleToken.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/SimpleToken.java
@@ -32,6 +32,11 @@
* The token text
*/
protected String text = "";
+
+ /**
+ * The guarding directive
+ */
+ protected String preprocessorDirective = null;
///////////////////////////////////////////////////////////////////////////
// Additional Fields - Not updated when refactoring
@@ -94,6 +99,10 @@
* Sets whitespace and whitetext appearing after this token that should be associated with this token, not the next
*/
public void setWhiteAfter(String value) {;}
+
+ public String getPreprocessorDirective() { return preprocessorDirective; }
+
+ public void setPreprocessorDirective(String preprocessorDirective) { this.preprocessorDirective = preprocessorDirective; }
public int getLine()
{
