| /******************************************************************************* |
| * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others. |
| * All rights reserved. This program and the accompanying materials |
| * are made available under the terms of the Eclipse Public License v1.0 |
| * which accompanies this distribution, and is available at |
| * http://www.eclipse.org/legal/epl-v10.html |
| * |
| * Contributors: |
| * UIUC - Initial API and implementation |
| *******************************************************************************/ |
| |
| /* |
| * First phase of fixed form lexical analysis for Fortran 95 parser |
| * |
| * FixedFormLexerPhase1 acts as a "token stream" feeding FreeFormLexerPhase2 |
| * and is feeded by FixedFormLexerPrepass |
| * (See FixedFormLexerPrepass and FixedFormLexerPhase2.java and f95t.bnf) |
| * |
| * @author Jeffrey Overbey and Dirk Rossow |
| * |
| * @see FixedFormLexerPrepass |
| * @see FixedFormLexerPhase2 |
| * @see Parser |
| * |
| * NOTE: Get rid of (space out) "yybegin(YYSTANDARD);" in the lines |
| * for any tokens that can appear in an IMPLICIT statement |
| * (It was also omitted in the lines for T_SLASH so that |
| * INTERFACE OPERATOR (/) would tokenize correctly.) |
| * |
| * Changes: |
| * 29.06.2005 Jeff Overbey: Added Fortran INCLUDE and CPP directives |
| */ |
| |
| package org.eclipse.photran.internal.core.lexer; |
| |
| import java.util.regex.Pattern; |
| import java.io.Reader; |
| import org.eclipse.core.resources.IFile; |
| |
| @SuppressWarnings("all") |
| %% |
| |
| %class FixedFormLexerPhase1 |
| %throws LexerException |
| %char |
| %line |
| %column |
| %ignorecase |
| //%8bit |
| %unicode |
| %implements ILexer |
| %type IToken |
| |
| %{ |
| private IToken prevToken = null; |
| |
| private FixedFormLexerPrepass prepass; |
| |
| public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass) { |
| this(new LineAppendingReader(in)); |
| this.prepass=_prepass; |
| } |
| |
| //unset start of line state |
| private void unsetSOL() { |
| if (yystate()==YYINITIAL) yybegin(YYSTANDARD); |
| } |
| |
| // JO |
| // IMPORTANT: If this is called on a new type of statement, |
| // PreLexerLine#hollerithsOK() in FixedFormLexerPrepass |
| // must also be modified to recognize the statement. |
| // This ensures that spaces will be removed from the |
| // text it would otherwise treat as a Hollerith. (This |
| // can lead to errors like "extraneous character: (space)" |
| // and "Hollerith spans multiple lines without continuation") |
| private void disallowHolleriths() { |
| if (yystate()==YYSTANDARD) yybegin(YYSTANDARD_NOHOLLERITH); |
| } |
| |
| public int getLine() { |
| return prepass.getLine(yychar); |
| } |
| |
| public int getCol() { |
| return prepass.getColumn(yychar); |
| } |
| |
| protected FileOrIFile lastTokenFile = null; |
| protected int lastTokenLine = 1, lastTokenCol = 1, lastTokenFileOffset = 0, lastTokenStreamOffset = 0, lastTokenLength = 0; |
| |
| private static final Pattern eol = Pattern.compile("(\\r|\\n)+"); |
| |
| protected boolean isEOL(String string) |
| { |
| return eol.matcher(string).matches(); |
| } |
| |
| private IToken token(Terminal terminal) |
| { |
| //For some there are 2 terminals of type Terminal.END_OF_INPUT that get here in a row |
| // so, technically, the function below sets the whitespaceAfter on the first one it sees, |
| // which is not really the expected behavior, but that token's whitespace is not used for |
| // anything anyway, so this should be OK |
| if(prevToken != null && terminal == Terminal.END_OF_INPUT) |
| { |
| //We need to manually set this, because the input string to the lexer does not |
| // have any whitespace, or at the very list has it trimmed, so we are loosing all |
| // of the trailing whitespace on any refactoring. That is why we assign it as |
| // whitespaceAfter to the last END_OF_STATEMENT token before END_OF_INPUT |
| String whiteAfter = prepass.getTrailingWhitespace(); |
| prevToken.setWhiteAfter(whiteAfter); |
| } |
| |
| lastTokenLine = prepass.getLine(yychar)+1; |
| lastTokenCol = prepass.getColumn(yychar)+1; |
| lastTokenFileOffset = prepass.getOffset(yychar); |
| lastTokenStreamOffset = prepass.getOffset(yychar); |
| lastTokenLength = prepass.getOffset(yychar+yylength()-1)-prepass.getOffset(yychar)+1; |
| |
| //For some reason the author of above code needed to add 1 to the line/col values |
| // for my code, I actually need the original value of token positions, so |
| // I added those variables for that |
| int tokLine = lastTokenLine-1; |
| int tokCol = lastTokenCol-1; |
| int tokOff = lastTokenFileOffset; |
| |
| String tokenText = ""; |
| //If it is the end of statement, use text from original string to get the line separator. |
| // For some reason the text returned by yytext() in this case is always '/n', while the |
| // actual separator is '/r/n' |
| if(terminal == Terminal.T_EOS) |
| { |
| tokenText = yytext(); |
| if (isEOL(tokenText)) |
| tokenText = prepass.getFileEOL(); |
| } |
| //If it is the end of input, use the Lexer's text. |
| else if(terminal == Terminal.END_OF_INPUT) |
| { |
| tokenText = yytext(); |
| } |
| //If it is a quote, use text accumulated in the buffer string |
| else if(terminal == Terminal.T_SCON || terminal == Terminal.T_HCON) |
| { |
| tokenText = stringBuffer.toString(); |
| //lastTokenColumn and lastTokenOffset of quoted strings actually IS the position of the last character |
| // of that string (for some reason w/o the final quote). So, in order to correctly map it to the whitespace |
| // before it, we need to shift the column and offset we are using by 1 less than the length of the string |
| tokCol = tokCol - (tokenText.length() - 1); |
| tokOff = tokOff - (tokenText.length() - 1); |
| } |
| //Otherwise, use the text directly from file (that way all the whitespace is preserved in the |
| // tokens text) |
| else |
| tokenText = prepass.getTokenText(lastTokenFileOffset, lastTokenLength); |
| |
| prevToken = new Token(terminal, |
| prepass.getWhitespaceBefore(tokLine, tokCol, tokOff), |
| tokenText, |
| ""); |
| |
| if(!tokenText.equals(yytext()) && terminal != Terminal.T_EOS) |
| { |
| prevToken.setPreprocessorDirective(new FixedFormReplacement(tokenText)); |
| prevToken.setText(yytext()); |
| } |
| |
| return prevToken; |
| } |
| |
| /* |
| private IToken token(Terminal terminal) |
| { |
| IToken t = new Token(); |
| t.setTerminal(terminal); |
| t.setFilename(this.filename); |
| t.setFileOffset(prepass.getOffset(yychar)); |
| t.setStreamOffset(prepass.getOffset(yychar)); |
| t.setLength(prepass.getOffset(yychar+yylength()-1)-prepass.getOffset(yychar)+1); |
| t.setText(terminal == Terminal.T_SCON || terminal == Terminal.T_HCON |
| ? stringBuffer.toString() |
| : yytext()); |
| t.setStartLine(prepass.getLine(yychar)+1); |
| t.setStartCol(prepass.getColumn(yychar)+1); |
| t.setEndLine(prepass.getLine(yychar+yylength()-1)+1); |
| t.setEndCol(prepass.getColumn(yychar+yylength()-1)+1); |
| |
| return t; |
| } |
| */ |
| |
| private StringBuffer stringBuffer = null; |
| private int hollerithLength = 0; |
| |
| private boolean wantEos = false; |
| |
| private String filename = "<stdin>"; |
| |
| public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, IFile file, String filename) |
| { |
| this(in, _prepass); |
| this.lastTokenFile = new FileOrIFile(file); |
| this.filename = filename; |
| } |
| |
| public String getFilename() |
| { |
| return filename; |
| } |
| |
| public int getLastTokenLine() |
| { |
| return lastTokenLine; |
| } |
| |
| public int getLastTokenCol() |
| { |
| return lastTokenCol; |
| } |
| |
| public FileOrIFile getLastTokenFile() |
| { |
| return lastTokenFile; |
| } |
| |
| public int getLastTokenFileOffset() |
| { |
| return lastTokenFileOffset; |
| } |
| |
| public int getLastTokenStreamOffset() |
| { |
| return lastTokenStreamOffset; |
| } |
| |
| public int getLastTokenLength() |
| { |
| return lastTokenLength; |
| } |
| |
| public void setTokenAsCurrent(IToken token) |
| { |
| //lastTokenFile = token.getFile(); |
| lastTokenLine = token.getLine(); |
| lastTokenCol = token.getCol(); |
| lastTokenFile = token.getPhysicalFile(); |
| lastTokenFileOffset = token.getFileOffset(); |
| lastTokenStreamOffset = token.getStreamOffset(); |
| lastTokenLength = token.getLength(); |
| } |
| |
| // private List/*<NonTreeToken>*/ nonTreeTokens = new LinkedList(); |
| // public List/*<NonTreeToken>*/ getNonTreeTokens() |
| // { |
| // return nonTreeTokens; |
| // } |
| // |
| private void storeNonTreeToken() |
| { |
| // nonTreeTokens.add(new NonTreeToken(this.filename, |
| // prepass.getOffset(yychar), // int offset |
| // prepass.getLine(yychar)+1, // int row |
| // prepass.getColumn(yychar)+1, // int col |
| // yytext())); |
| } |
| %} |
| |
| LineTerminator = \r\n|\n|\r |
| |
| StarredType = ("*"[0-9]+)? |
| |
| Dig = [0-9]+ |
| EExp = ("E"|"e")("+"|"-")?{Dig} |
| DExp = ("D"|"d")("+"|"-")?{Dig} |
| Sig1 = ({Dig} "." [0-9]+ | "." {Dig}) |
| Sig2 = {Dig} "." |
| Sig = {Sig1} | {Sig2} |
| DdotD = {Dig} "." {Dig} |
| Efw = [Ee] {Dig} |
| Range = [a-zA-Z](-[a-zA-Z])? |
| |
| Bcon = ( B ' [0-1]+ ' | B \" [0-1]+ \" ) |
| Ocon = ( O ' [0-7]+ ' | O \" [0-7]+ \" ) |
| Zcon = ( Z ' [0-9A-Fa-f]+ ' | Z \" [0-9A-Fa-f]+ \" ) |
| Hcon = {Dig}(H|h) |
| |
| Dcon1 = {Dig}{DExp} | {Sig}{DExp} | {Sig1} |
| Dcon2 = {Sig2} |
| Fcon = [IiBbOoZzFfDd] {DdotD} | (E [NnSs]? | e [NnSs]? | G | g) {DdotD} {Efw}? |
| Icon = ( {Dig} | {Dig} ("E"|"e") {Dig} "." {Dig} ) |
| Pcon = ("+"|"-")? {Dig} (P|p) |
| Rcon1 = {Dig}{EExp} | {Sig}{EExp} | {Sig1} |
| Rcon2 = {Sig2} |
| Xcon = {Dig} (X|x) |
| Xdop = \. [A-Za-z]+ \. |
| Ident = [A-Za-z$][A-Za-z0-9$_]* |
| | "@" [A-Za-z$][A-Za-z0-9$_]* ":T_IDENT" /* for AST pattern matching */ |
| xImpl = "(" {Range} ("," {Range})* ")" |
| xImplLkahead = (","|";"|{LineTerminator}|"!") |
| |
| NumDotLkahead = [^A-Za-z] |
| |
| CppIfdef="#"[ \t]*"ifdef"[^\r\n]*{LineTerminator} |
| CppIfndef="#"[ \t]*"ifndef"[^\r\n]*{LineTerminator} |
| CppIf="#"[ \t]*"if"[^\r\n]*{LineTerminator} |
| CppElse="#"[ \t]*"else"[^\r\n]*{LineTerminator} |
| CppElif="#"[ \t]*"elif"[^\r\n]*{LineTerminator} |
| CppEndIf="#"[ \t]*"endif"[^\r\n]*{LineTerminator} |
| CppInclude="#"[ \t]*"include"[^\r\n]*{LineTerminator} |
| CppDefine="#"[ \t]*"define"[^\r\n]*{LineTerminator} |
| CppUndef="#"[ \t]*"undef"[^\r\n]*{LineTerminator} |
| CppLine="#"[ \t]*"line"[^\r\n]*{LineTerminator} |
| CppError="#"[ \t]*"error"[^\r\n]*{LineTerminator} |
| CppPragma="#"[ \t]*"pragma"[^\r\n]*{LineTerminator} |
| CppDirective={CppIfdef}|{CppIfndef}|{CppIf}|{CppElse}|{CppElif}|{CppEndIf}|{CppInclude}|{CppDefine}|{CppUndef}|{CppLine}|{CppError}|{CppPragma} |
| |
| FortranInclude="INCLUDE"[ \t]*[\'\"][^\r\n]*[\'\"]{LineTerminator} |
| |
| %state IMPLICIT |
| %state QUOTED |
| %state DBLQUOTED |
| %state HOLLERITH |
| %state YYSTANDARD |
| %state YYSTANDARD_NOHOLLERITH |
| %state OPERATORorFORMAT |
| %state IDENT |
| |
| %% |
| |
| /* Lexical rules */ |
| |
| /* YYSTANDARD and YYSTANDARD_NOHOLLERITH are similar to YYINITIAL but exist to ignore some tokens on start of line */ |
| /* YYSTANDARD_NOHOLLERITH prevents recognition of Holleriths in certain contexts */ |
| /* Incidentally, this is also a state where we don't want to recognize {Pcon} (e.g., DO 1874 p = 1, N) */ |
| |
| <YYSTANDARD,IMPLICIT,OPERATORorFORMAT> { |
| {Hcon} { stringBuffer = new StringBuffer(); |
| String text = yytext(); |
| stringBuffer.append(text); |
| hollerithLength=Integer.parseInt(text.substring(0,text.length()-1)); |
| if (hollerithLength==0) |
| throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Invalid length of hollerith literal: 0"); |
| yybegin(HOLLERITH); |
| } |
| {Pcon} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PCON); } |
| } |
| |
| /* ignore some tokens on start of line*/ |
| <YYSTANDARD_NOHOLLERITH,YYSTANDARD,IMPLICIT,OPERATORorFORMAT> { |
| {Rcon1} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RCON); } |
| {Rcon2}/{NumDotLkahead} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RCON); } |
| {Dcon1} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DCON); } |
| {Dcon2}/{NumDotLkahead} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DCON); } |
| {Fcon} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FCON); } |
| {Xcon} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_XCON); } |
| } |
| |
| <IDENT> { |
| {Ident} { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IDENT); } |
| } |
| |
| |
| <YYSTANDARD_NOHOLLERITH,YYSTANDARD,YYINITIAL,IMPLICIT,OPERATORorFORMAT> { |
| // Intel Extension |
| "CONVERT"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_CONVERTEQ); } |
| // New for Fortran 2008 ////////////////////////////////// |
| "CONCURRENT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CONCURRENT); } |
| "NEWUNIT="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NEWUNITEQ); } |
| "SUBMODULE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SUBMODULE); } |
| "ENDSUBMODULE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDSUBMODULE); } |
| "ENDPROCEDURE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDPROCEDURE); } |
| "IMPURE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IMPURE); } |
| "CODIMENSION" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CODIMENSION); } |
| "CONTIGUOUS" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CONTIGUOUS); } |
| "CRITICAL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CRITICAL); } |
| "ENDCRITICAL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDCRITICAL); } |
| "ALL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ALL); } |
| "ALLSTOP" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ALLSTOP); } |
| "SYNC" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SYNC); } |
| "SYNCALL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SYNCALL); } |
| "SYNCIMAGES" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SYNCIMAGES); } |
| "IMAGES" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IMAGES); } |
| "SYNCMEMORY" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SYNCMEMORY); } |
| "MEMORY" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_MEMORY); } |
| "LOCK" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_LOCK); } |
| "UNLOCK" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNLOCK); } |
| // New for Fortran 2003 ////////////////////////////////// |
| "IMPORT" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IMPORT); } |
| "NON_INTRINSIC" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_NON_INTRINSIC); } |
| "WAIT" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_WAIT); } |
| "[" { wantEos = true; return token(Terminal.T_LBRACKET); } |
| "]" { wantEos = true; return token(Terminal.T_RBRACKET); } |
| "STREAM"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_STREAMEQ); } |
| "PENDING"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_PENDINGEQ); } |
| "POS"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_POSEQ); } |
| "ID"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IDEQ); } |
| "SIGN"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_SIGNEQ); } |
| "ROUND"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ROUNDEQ); } |
| "IOMSG"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IOMSGEQ); } |
| "ENCODING"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ENCODINGEQ); } |
| "DECIMAL"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_DECIMALEQ); } |
| "ASYNCHRONOUS"[ \t]*"="/[^>] { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ASYNCHRONOUSEQ); } |
| "IS" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_IS); } |
| "ASSOCIATE" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ASSOCIATE); } |
| "EXTENDS" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_EXTENDS); } |
| "ABSTRACT" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ABSTRACT); } |
| "BIND" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_BIND); } |
| "PASS" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_PASS); } |
| "NOPASS" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_NOPASS); } |
| "GENERIC" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_GENERIC); } |
| "NON_OVERRIDABLE" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_NON_OVERRIDABLE); } |
| "DEFERRED" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_DEFERRED); } |
| "FINAL" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_FINAL); } |
| "KIND" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_KIND); } |
| "LEN" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_LEN); } |
| "ENUM" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ENUM); } |
| "ENUMERATOR" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ENUMERATOR); } |
| "CLASS" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_CLASS); } |
| "ASYNCHRONOUS" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_ASYNCHRONOUS); } |
| "PROTECTED" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_PROTECTED); } |
| "VALUE" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_VALUE); } |
| "VOLATILE" { wantEos = true; yybegin(YYINITIAL); return token(Terminal.T_VOLATILE); } |
| // Same as for Fortran 77/90/95 ////////////////////////////////// |
| "ACCESS="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ACCESSEQ); } |
| "ACTION="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ACTIONEQ); } |
| "ADVANCE="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ADVANCEEQ); } |
| "ALLOCATABLE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ALLOCATABLE); } |
| "ALLOCATE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ALLOCATE); } |
| "ASSIGN" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ASSIGN); } |
| "ASSIGNMENT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ASSIGNMENT); } |
| "BACKSPACE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BACKSPACE); } |
| "BLANK="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BLANKEQ); } |
| "BLOCK" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BLOCK); } |
| "BLOCKDATA" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_BLOCKDATA); } |
| "CALL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CALL); } |
| "CASE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CASE); } |
| "CHARACTER"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_CHARACTER); } |
| "CLOSE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CLOSE); } |
| "COMMON" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_COMMON); } |
| "COMPLEX"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_COMPLEX); } |
| "CONTAINS" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CONTAINS); } |
| "CONTINUE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CONTINUE); } |
| "CYCLE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_CYCLE); } |
| "DATA" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DATA); } |
| "DEALLOCATE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DEALLOCATE); } |
| "DEFAULT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DEFAULT); } |
| "DELIM="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DELIMEQ); } |
| "DIMENSION" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DIMENSION); } |
| "DIRECT="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_DIRECTEQ); } |
| "DO" { wantEos = true; yybegin(YYSTANDARD); disallowHolleriths(); return token(Terminal.T_DO); } |
| "DOUBLE"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_DOUBLE); } |
| "DOUBLECOMPLEX"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_DOUBLECOMPLEX); } |
| "DOUBLEPRECISION"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_DOUBLEPRECISION); } |
| "ELEMENTAL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELEMENTAL); } |
| "ELSE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELSE); } |
| "ELSEIF" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELSEIF); } |
| "ELSEWHERE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ELSEWHERE); } |
| "END" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_END); } |
| "ENDBLOCK" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDBLOCK); } |
| "ENDBLOCKDATA" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDBLOCKDATA); } |
| "ENDDO" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDDO); } |
| "END="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDEQ); } |
| "ENDFILE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDFILE); } |
| "ENDFORALL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDFORALL); } |
| "ENDFUNCTION" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDFUNCTION); } |
| "ENDIF" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDIF); } |
| "ENDINTERFACE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDINTERFACE); } |
| "ENDMODULE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDMODULE); } |
| "ENDPROGRAM" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDPROGRAM); } |
| "ENDSELECT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDSELECT); } |
| "ENDSUBROUTINE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDSUBROUTINE); } |
| "ENDTYPE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDTYPE); } |
| "ENDWHERE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENDWHERE); } |
| "ENTRY" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ENTRY); } |
| "EQUIVALENCE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EQUIVALENCE); } |
| "ERR="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ERREQ); } |
| "EXIST="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EXISTEQ); } |
| "EXIT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EXIT); } |
| "EXTERNAL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EXTERNAL); } |
| "FILE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FILE); } |
| "FILE="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FILEEQ); } |
| "FMT="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FMTEQ); } |
| "FORALL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FORALL); } |
| "FORMAT" { wantEos = true; yybegin(OPERATORorFORMAT); return token(Terminal.T_FORMAT); } |
| "FORMATTED="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FORMATTEDEQ); } |
| "FORM="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FORMEQ); } |
| "FUNCTION" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_FUNCTION); } |
| "GO" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_GO); } |
| "GOTO" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_GOTO); } |
| "IF" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IF); } |
| "IMPLICIT" { wantEos = true; yybegin(IMPLICIT); return token(Terminal.T_IMPLICIT); } |
| "IN" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IN); } |
| "INOUT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INOUT); } |
| "INQUIRE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INQUIRE); } |
| "INTEGER"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_INTEGER); } |
| "INTENT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INTENT); } |
| "INTERFACE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INTERFACE); } |
| "INTRINSIC" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_INTRINSIC); } |
| "IOLENGTH="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IOLENGTHEQ); } |
| "IOSTAT="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_IOSTATEQ); } |
| "KIND="/[^>] { wantEos = true; unsetSOL(); return token(Terminal.T_KINDEQ); } |
| "LEN="/[^>] { wantEos = true; unsetSOL(); return token(Terminal.T_LENEQ); } |
| "LOGICAL"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_LOGICAL); } |
| "MODULE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_MODULE); } |
| "NAMED="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NAMEDEQ); } |
| "NAME="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NAMEEQ); } |
| "NAMELIST" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NAMELIST); } |
| "NEXTREC="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NEXTRECEQ); } |
| "NML="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NMLEQ); } |
| "NONE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NONE); } |
| "NULL" { wantEos = true; unsetSOL(); return token(Terminal.T_NULL); } |
| "NULLIFY" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NULLIFY); } |
| "NUMBER="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_NUMBEREQ); } |
| "ONLY" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_ONLY); } |
| "OPEN" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OPEN); } |
| "OPENED="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OPENEDEQ); } |
| "OPERATOR" { wantEos = true; yybegin(OPERATORorFORMAT); return token(Terminal.T_OPERATOR); } |
| "OPTIONAL" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OPTIONAL); } |
| "OUT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_OUT); } |
| "PAD="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PADEQ); } |
| "PARAMETER" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PARAMETER); } |
| "PAUSE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PAUSE); } |
| "POINTER" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_POINTER); } |
| "POSITION="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_POSITIONEQ); } |
| "PRECISION" { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_PRECISION); } |
| "PRINT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PRINT); } |
| "PRIVATE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PRIVATE); } |
| "PROCEDURE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PROCEDURE); } |
| "PROGRAM" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PROGRAM); } |
| "PUBLIC" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PUBLIC); } |
| "PURE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PURE); } |
| "READ" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_READ); } |
| "READ="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_READEQ); } |
| "READWRITE="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_READWRITEEQ); } |
| "REAL"{StarredType} { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_REAL); } |
| "REC="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RECEQ); } |
| "RECL="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RECLEQ); } |
| "RECURSIVE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RECURSIVE); } |
| "RESULT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RESULT); } |
| "RETURN" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_RETURN); } |
| "REWIND" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_REWIND); } |
| "SAVE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SAVE); } |
| "SELECT" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SELECT); } |
| "SELECTCASE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SELECTCASE); } |
| "SEQUENCE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SEQUENCE); } |
| "SEQUENTIAL="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SEQUENTIALEQ); } |
| "SIZE="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SIZEEQ); } |
| "STAT="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_STATEQ); } |
| "STATUS="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_STATUSEQ); } |
| "STOP" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_STOP); } |
| "SUBROUTINE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SUBROUTINE); } |
| "TARGET" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_TARGET); } |
| "THEN" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_THEN); } |
| "TO" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_TO); } |
| ".TRUE." { wantEos = true; unsetSOL(); return token(Terminal.T_TRUE); } |
| "TYPE" { wantEos = true; unsetSOL(); return token(Terminal.T_TYPE); } |
| "UNFORMATTED="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNFORMATTEDEQ); } |
| "UNIT="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNITEQ); } |
| "USE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_USE); } |
| "WHERE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WHERE); } |
| "WHILE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WHILE); } |
| "WRITE" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WRITE); } |
| "WRITE="/[^>] { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_WRITEEQ); } |
| "*" { wantEos = true; unsetSOL(); disallowHolleriths(); return token(Terminal.T_ASTERISK); } |
| ".AND." { wantEos = true; unsetSOL(); return token(Terminal.T_AND); } |
| ":" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_COLON); } |
| "," { wantEos = true; unsetSOL(); return token(Terminal.T_COMMA); } |
| ".EQ." { wantEos = true; unsetSOL(); return token(Terminal.T_EQ); } |
| "==" { wantEos = true; unsetSOL(); return token(Terminal.T_EQEQ); } |
| "=>" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EQGREATERTHAN); } |
| "=" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_EQUALS); } |
| ".EQV." { wantEos = true; unsetSOL(); return token(Terminal.T_EQV); } |
| ".FALSE." { wantEos = true; unsetSOL(); return token(Terminal.T_FALSE); } |
| ".GE." { wantEos = true; unsetSOL(); return token(Terminal.T_GE); } |
| ">" { wantEos = true; unsetSOL(); return token(Terminal.T_GREATERTHAN); } |
| ">=" { wantEos = true; unsetSOL(); return token(Terminal.T_GREATERTHANEQ); } |
| ".GT." { wantEos = true; unsetSOL(); return token(Terminal.T_GT); } |
| ".LE." { wantEos = true; unsetSOL(); return token(Terminal.T_LE); } |
| "<" { wantEos = true; unsetSOL(); return token(Terminal.T_LESSTHAN); } |
| "<=" { wantEos = true; unsetSOL(); return token(Terminal.T_LESSTHANEQ); } |
| "(" { wantEos = true; unsetSOL(); return token(Terminal.T_LPAREN); } |
| ".LT." { wantEos = true; unsetSOL(); return token(Terminal.T_LT); } |
| "-" { wantEos = true; unsetSOL(); return token(Terminal.T_MINUS); } |
| "<>" { wantEos = true; return token(Terminal.T_NE); } |
| ".NE." { wantEos = true; unsetSOL(); return token(Terminal.T_NE); } |
| ".NEQV." { wantEos = true; unsetSOL(); return token(Terminal.T_NEQV); } |
| ".NOT." { wantEos = true; unsetSOL(); return token(Terminal.T_NOT); } |
| ".OR." { wantEos = true; unsetSOL(); return token(Terminal.T_OR); } |
| "%" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_PERCENT); } |
| "+" { wantEos = true; unsetSOL(); return token(Terminal.T_PLUS); } |
| "**" { wantEos = true; unsetSOL(); return token(Terminal.T_POW); } |
| ")" { wantEos = true; unsetSOL(); return token(Terminal.T_RPAREN); } |
| "/" { wantEos = true; unsetSOL(); return token(Terminal.T_SLASH); } |
| "/=" { wantEos = true; unsetSOL(); return token(Terminal.T_SLASHEQ); } |
| "//" { wantEos = true; unsetSOL(); return token(Terminal.T_SLASHSLASH); } |
| "_" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_UNDERSCORE); } |
| {Bcon} { wantEos = true; unsetSOL(); return token(Terminal.T_BCON); } |
| {Icon} { wantEos = true; unsetSOL(); return token(Terminal.T_ICON); } |
| {Ocon} { wantEos = true; unsetSOL(); return token(Terminal.T_OCON); } |
| {Xdop} { wantEos = true; unsetSOL(); return token(Terminal.T_XDOP); } |
| {Zcon} { wantEos = true; unsetSOL(); return token(Terminal.T_ZCON); } |
| "'" { stringBuffer = new StringBuffer(); |
| stringBuffer.append('\''); |
| yybegin(QUOTED); } |
| \" { stringBuffer = new StringBuffer(); |
| stringBuffer.append('\"'); |
| yybegin(DBLQUOTED); |
| } |
| {CppDirective} { storeNonTreeToken(); } |
| {FortranInclude} { storeNonTreeToken(); } |
| {LineTerminator}|";" { yybegin(YYINITIAL); boolean b = wantEos; wantEos = false; if (b) return token(Terminal.T_EOS); else storeNonTreeToken(); } |
| <<EOF>> { wantEos = false; yybegin(YYSTANDARD); return token(Terminal.END_OF_INPUT); } |
| . { yypushback(1); |
| int state=yystate(); |
| yybegin(IDENT); |
| IToken token = yylex(); |
| yybegin(state); |
| return token; |
| } |
| } |
| |
| |
| <IMPLICIT> { |
| {xImpl}/{xImplLkahead} { wantEos = true; return token(Terminal.T_X_IMPL); } |
| {xImpl}$ { wantEos = true; return token(Terminal.T_X_IMPL); } |
| } |
| |
| <YYSTANDARD_NOHOLLERITH,YYSTANDARD,IMPLICIT> { |
| "(/" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_LPARENSLASH); } |
| "/)" { wantEos = true; yybegin(YYSTANDARD); return token(Terminal.T_SLASHRPAREN); } |
| } |
| |
| <QUOTED> { |
| "''" { stringBuffer.append("''"); } |
| "'" { stringBuffer.append("'"); |
| yybegin(YYSTANDARD); |
| wantEos = true; |
| return token(Terminal.T_SCON); } |
| [^\n\r']+ { stringBuffer.append( yytext() ); } |
| [\n\r] { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): String literal spans multiple lines without continuation"); } |
| <<EOF>> { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): End of file encountered before string literal terminated"); } |
| } |
| |
| <DBLQUOTED> { |
| \"\" { stringBuffer.append("\"\""); } |
| \" { stringBuffer.append('\"'); |
| yybegin(YYSTANDARD); |
| wantEos = true; |
| return token(Terminal.T_SCON); } |
| [^\n\r\"]+ { stringBuffer.append( yytext() ); } |
| [\n\r] { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): String literal spans multiple lines without continuation"); } |
| <<EOF>> { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): End of file encountered before string literal terminated"); } |
| } |
| |
| <HOLLERITH> { |
| [^\n\r] { hollerithLength--; |
| stringBuffer.append(yytext()); |
| if (hollerithLength==0) { |
| yybegin(YYSTANDARD); |
| wantEos = true; |
| return token(Terminal.T_HCON); |
| } |
| } |
| [\n\r] { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Hollerith literal spans multiple lines without continuation"); } |
| <<EOF>> { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): End of file encountered before hollerith literal terminated"); } |
| } |
| |
| |
| /* Error */ |
| /*. { throw new LexerException(this, "Lexer Error (line " + (getLine()+1) + ", col " + (getCol()+1) + "): Unexpected character " + yytext()); }*/ |
| . { wantEos = true; unsetSOL(); return token(Terminal.T_UNEXPECTED_CHARACTER); } |
