Fixed replacement of semi-colons by newlines when refactoring fixed files
Fixed doubling-up of = sign for certain tokens when refactoring fixed files
diff --git a/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex b/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex
index c6f91bc..88934f6 100644
--- a/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex
+++ b/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex
@@ -33,6 +33,7 @@
package org.eclipse.photran.internal.core.lexer;
+import java.util.regex.Pattern;
import java.io.InputStream;
import org.eclipse.core.resources.IFile;
@@ -79,6 +80,13 @@
protected FileOrIFile lastTokenFile = null;
protected int lastTokenLine = 1, lastTokenCol = 1, lastTokenFileOffset = 0, lastTokenStreamOffset = 0, lastTokenLength = 0;
+ private static final Pattern eol = Pattern.compile("(\\r|\\n)+");
+
+ protected boolean isEOL(String string)
+ {
+ return eol.matcher(string).matches();
+ }
+
private IToken token(Terminal terminal)
{
//For some there are 2 terminals of type Terminal.END_OF_INPUT that get here in a row
@@ -114,7 +122,9 @@
// actual separator is '/r/n'
if(terminal == Terminal.T_EOS)
{
- tokenText = prepass.getFileEOL();
+ tokenText = yytext();
+ if (isEOL(tokenText))
+ tokenText = prepass.getFileEOL();
}
//If it is the end of input, use the Lexer's text.
else if(terminal == Terminal.END_OF_INPUT)
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
index f778169..248c5bf 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 3/5/10 10:46 AM */
+/* The following code was generated by JFlex 1.4.1 on 3/16/10 3:22 PM */
/*******************************************************************************
* Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
@@ -35,6 +35,7 @@
package org.eclipse.photran.internal.core.lexer;
+import java.util.regex.Pattern;
import java.io.InputStream;
import org.eclipse.core.resources.IFile;
@@ -43,7 +44,7 @@
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 3/5/10 10:46 AM from the specification file
+ * on 3/16/10 3:22 PM from the specification file
* <tt>FixedFormLexerPhase1.flex</tt>
*/
class FixedFormLexerPhase1 implements ILexer {
@@ -937,6 +938,13 @@
protected FileOrIFile lastTokenFile = null;
protected int lastTokenLine = 1, lastTokenCol = 1, lastTokenFileOffset = 0, lastTokenStreamOffset = 0, lastTokenLength = 0;
+ private static final Pattern eol = Pattern.compile("(\\r|\\n)+");
+
+ protected boolean isEOL(String string)
+ {
+ return eol.matcher(string).matches();
+ }
+
private IToken token(Terminal terminal)
{
//For some there are 2 terminals of type Terminal.END_OF_INPUT that get here in a row
@@ -972,7 +980,9 @@
// actual separator is '/r/n'
if(terminal == Terminal.T_EOS)
{
- tokenText = prepass.getFileEOL();
+ tokenText = yytext();
+ if (isEOL(tokenText))
+ tokenText = prepass.getFileEOL();
}
//If it is the end of input, use the Lexer's text.
else if(terminal == Terminal.END_OF_INPUT)
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
index e30cc89..c85bc7c 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
@@ -50,7 +50,21 @@
};
FixedFormLexerPhase1 fixedLexer1 = new FixedFormLexerPhase1(prepassReader, prepass, file, filename, tokenFactory);
- freeLexer2 = new FreeFormLexerPhase2(fixedLexer1);
+ freeLexer2 = new FreeFormLexerPhase2(fixedLexer1)
+ {
+ @Override
+ protected void modifyPreprocessorDirective(IToken t)
+ {
+ IPreprocessorReplacement ppr = t.getPreprocessorDirective();
+ if(ppr != null && ppr instanceof FixedFormReplacement)
+ {
+ FixedFormReplacement ffr = (FixedFormReplacement)ppr;
+ String replStr = ffr.toString();
+ replStr = replStr.replaceAll("=", "");
+ ffr.setReplacementText(replStr);
+ }
+ }
+ };
}
public IToken yylex() throws IOException, LexerException
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormReplacement.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormReplacement.java
index 7b0ae2c..d7ba7a4 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormReplacement.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormReplacement.java
@@ -37,4 +37,9 @@
{
return replacementText;
}
+
+ public void setReplacementText(String newText)
+ {
+ replacementText = newText;
+ }
}
diff --git a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase2.java b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase2.java
index 17126f1..cd629c9 100644
--- a/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase2.java
+++ b/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase2.java
@@ -1376,6 +1376,11 @@
return parenDepth[tokenPos] == depth;
}
}
+
+ protected void modifyPreprocessorDirective(IToken t)
+ {
+
+ }
/**
* The token must appear in the spec list for an OPEN, CLOSE, READ,
@@ -1609,8 +1614,10 @@
streamOffsets.insertElementAt(new Integer(((Integer)streamOffsets.get(i+j)).intValue()+textWithoutEquals.length()), i+j+1);
lengths.insertElementAt(new Integer(1), i+j+1);
}
+ modifyPreprocessorDirective(t);
}
+
// If, say, "integer*3" was changed into an identifier, split it into
// three tokens: the identifier "integer", the asterisk, and the number 3
for (int j = 0; j < identifiersStarred.size(); j++)
