| /***** BEGIN LICENSE BLOCK ***** |
| * Version: CPL 1.0/GPL 2.0/LGPL 2.1 |
| * |
| * The contents of this file are subject to the Common Public |
| * License Version 1.0 (the "License"); you may not use this file |
| * except in compliance with the License. You may obtain a copy of |
| * the License at http://www.eclipse.org/legal/cpl-v10.html |
| * |
| * Software distributed under the License is distributed on an "AS |
| * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or |
| * implied. See the License for the specific language governing |
| * rights and limitations under the License. |
| * |
| * Copyright (C) 2002 Benoit Cerrina <b.cerrina@wanadoo.fr> |
| * Copyright (C) 2002-2004 Anders Bengtsson <ndrsbngtssn@yahoo.se> |
| * Copyright (C) 2002-2004 Jan Arne Petersen <jpetersen@uni-bonn.de> |
| * Copyright (C) 2004-2006 Thomas E Enebo <enebo@acm.org> |
| * Copyright (C) 2004 Stefan Matthias Aust <sma@3plus4.de> |
| * Copyright (C) 2004-2005 David Corbin <dcorbin@users.sourceforge.net> |
| * Copyright (C) 2005 Zach Dennis <zdennis@mktec.com> |
| * Copyright (C) 2006 Thomas Corbat <tcorbat@hsr.ch> |
| * |
| * Alternatively, the contents of this file may be used under the terms of |
| * either of the GNU General Public License Version 2 or later (the "GPL"), |
| * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), |
| * in which case the provisions of the GPL or the LGPL are applicable instead |
| * of those above. If you wish to allow use of your version of this file only |
| * under the terms of either the GPL or the LGPL, and not to allow others to |
| * use your version of this file under the terms of the CPL, indicate your |
| * decision by deleting the provisions above and replace them with the notice |
| * and other provisions required by the GPL or the LGPL. If you do not delete |
| * the provisions above, a recipient may use your version of this file under |
| * the terms of any one of the CPL, the GPL or the LGPL. |
| ***** END LICENSE BLOCK *****/ |
| package org.jruby.lexer.yacc; |
| |
| import java.io.IOException; |
| import java.math.BigInteger; |
| |
| import org.jruby.ast.BackRefNode; |
| import org.jruby.ast.BignumNode; |
| import org.jruby.ast.CommentNode; |
| import org.jruby.ast.FixnumNode; |
| import org.jruby.ast.FloatNode; |
| import org.jruby.ast.NthRefNode; |
| import org.jruby.common.IRubyWarnings; |
| import org.jruby.parser.BlockStaticScope; |
| import org.jruby.parser.ParserSupport; |
| import org.jruby.parser.StaticScope; |
| import org.jruby.parser.Tokens; |
| import org.jruby.util.IdUtil; |
| |
| /** This is a port of the MRI lexer to Java it is compatible to Ruby 1.8.1. |
| */ |
| public class RubyYaccLexer { |
| // Last token read via yylex(). |
| private int token; |
| |
| // Value of last token which had a value associated with it. |
| Object yaccValue; |
| |
| // Stream of data that yylex() examines. |
| private LexerSource src; |
| |
| // Used for tiny smidgen of grammar in lexer (see setParserSupport()) |
| private ParserSupport parserSupport = null; |
| |
| // What handles warnings |
| private IRubyWarnings warnings; |
| |
| // Additional context surrounding tokens that both the lexer and |
| // grammar use. |
| private LexState lex_state; |
| |
| // Tempory buffer to build up a potential token. Consumer takes responsibility to reset |
| // this before use. |
| private StringBuffer tokenBuffer = new StringBuffer(60); |
| |
| private StackState conditionState = new StackState(); |
| private StackState cmdArgumentState = new StackState(); |
| private StrTerm lex_strterm; |
| private boolean commandStart; |
| |
| // Give a name to a value. Enebo: This should be used more. |
| static final int EOF = 0; |
| |
| // ruby constants for strings (should this be moved somewhere else?) |
| static final int STR_FUNC_ESCAPE=0x01; |
| static final int STR_FUNC_EXPAND=0x02; |
| static final int STR_FUNC_REGEXP=0x04; |
| static final int STR_FUNC_QWORDS=0x08; |
| static final int STR_FUNC_SYMBOL=0x10; |
| static final int STR_FUNC_INDENT=0x20; |
| |
| private final int str_squote = 0; |
| private final int str_dquote = STR_FUNC_EXPAND; |
| private final int str_xquote = STR_FUNC_EXPAND; |
| private final int str_regexp = STR_FUNC_REGEXP | STR_FUNC_ESCAPE | STR_FUNC_EXPAND; |
| private final int str_ssym = STR_FUNC_SYMBOL; |
| private final int str_dsym = STR_FUNC_SYMBOL | STR_FUNC_EXPAND; |
| |
| public RubyYaccLexer() { |
| reset(); |
| } |
| |
| public void reset() { |
| token = 0; |
| yaccValue = null; |
| src = null; |
| lex_state = null; |
| resetStacks(); |
| lex_strterm = null; |
| commandStart = true; |
| } |
| |
| /** |
| * How the parser advances to the next token. |
| * |
| * @return true if not at end of file (EOF). |
| */ |
| public boolean advance() throws IOException { |
| return (token = yylex()) != EOF; |
| } |
| |
| /** |
| * Last token read from the lexer at the end of a call to yylex() |
| * |
| * @return last token read |
| */ |
| public int token() { |
| return token; |
| } |
| |
| public StringBuffer getTokenBuffer() { |
| return tokenBuffer; |
| } |
| |
| /** |
| * Value of last token (if it is a token which has a value). |
| * |
| * @return value of last value-laden token |
| */ |
| public Object value() { |
| return yaccValue; |
| } |
| |
| public ISourcePositionFactory getPositionFactory() { |
| return src.getPositionFactory(); |
| } |
| |
| /** |
| * Get position information for Token/Node that follows node represented by startPosition |
| * and current lexer location. |
| * |
| * @param startPosition previous node/token |
| * @param inclusive include previous node into position information of current node |
| * @return a new position |
| */ |
| public ISourcePosition getPosition(ISourcePosition startPosition, boolean inclusive) { |
| return src.getPosition(startPosition, inclusive); |
| } |
| |
| public ISourcePosition getPosition() { |
| return src.getPosition(null, false); |
| } |
| |
| /** |
| * Parse must pass its support object for some check at bottom of |
| * yylex(). Ruby does it this way as well (i.e. a little parsing |
| * logic in the lexer). |
| * |
| * @param parserSupport |
| */ |
| public void setParserSupport(ParserSupport parserSupport) { |
| this.parserSupport = parserSupport; |
| } |
| |
| /** |
| * Allow the parser to set the source for its lexer. |
| * |
| * @param source where the lexer gets raw data |
| */ |
| public void setSource(LexerSource source) { |
| this.src = source; |
| } |
| |
| public StrTerm getStrTerm() { |
| return lex_strterm; |
| } |
| |
| public void setStrTerm(StrTerm strterm) { |
| this.lex_strterm = strterm; |
| } |
| |
| public void resetStacks() { |
| conditionState.reset(); |
| cmdArgumentState.reset(); |
| } |
| |
| public void setWarnings(IRubyWarnings warnings) { |
| this.warnings = warnings; |
| } |
| |
| |
| public void setState(LexState state) { |
| this.lex_state = state; |
| } |
| |
| public StackState getCmdArgumentState() { |
| return cmdArgumentState; |
| } |
| |
| public StackState getConditionState() { |
| return conditionState; |
| } |
| |
| public void setValue(Object yaccValue) { |
| this.yaccValue = yaccValue; |
| } |
| |
| private boolean isNext_identchar() throws IOException { |
| char c = src.read(); |
| src.unread(c); |
| |
| return c != EOF && (Character.isLetterOrDigit(c) || c == '-'); |
| } |
| |
| private Object getInteger(String value, int radix) { |
| try { |
| return new FixnumNode(getPosition(), Long.parseLong(value, radix)); |
| } catch (NumberFormatException e) { |
| return new BignumNode(getPosition(), new BigInteger(value, radix)); |
| } |
| } |
| |
| /** |
| * Do the next characters from the source match provided String in a case insensitive manner. |
| * If so, then consume those characters and that string. Otherwise, consume none of them and |
| * return null. |
| * |
| * @param s to be matched against |
| * @return string if string matches, null otherwise |
| */ |
| private String isNextNoCase(String s) throws IOException { |
| StringBuffer buf = new StringBuffer(); |
| |
| for (int i = 0; i < s.length(); i++) { |
| char c = s.charAt(i); |
| char r = src.read(); |
| buf.append(r); |
| |
| if (Character.toLowerCase(c) != r && |
| Character.toUpperCase(c) != r) { |
| src.unreadMany(buf); |
| return null; |
| } |
| } |
| |
| return buf.toString(); |
| } |
| |
| /** |
| * @param c the character to test |
| * @return true if character is a hex value (0-9a-f) |
| */ |
| static final boolean isHexChar(char c) { |
| return Character.isDigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F'); |
| } |
| |
| /** |
| * @param c the character to test |
| * @return true if character is an octal value (0-7) |
| */ |
| static final boolean isOctChar(char c) { |
| return '0' <= c && c <= '7'; |
| } |
| |
| /** |
| * @param c is character to be compared |
| * @return whether c is an identifier or not |
| */ |
| private static final boolean isIdentifierChar(char c) { |
| return Character.isLetterOrDigit(c) || c == '_'; |
| } |
| |
| /** |
| * What type/kind of quote are we dealing with? |
| * |
| * @param c first character the the quote construct |
| * @return a token that specifies the quote type |
| */ |
| private int parseQuote(char c) throws IOException { |
| char begin, end; |
| boolean shortHand; |
| |
| // Short-hand (e.g. %{,%.,%!,... versus %Q{). |
| if (!Character.isLetterOrDigit(c)) { |
| begin = c; |
| c = 'Q'; |
| shortHand = true; |
| // Long-hand (e.g. %Q{}). |
| } else { |
| shortHand = false; |
| begin = src.read(); |
| if (Character.isLetterOrDigit(begin) /* no mb || ismbchar(term)*/) { |
| throw new SyntaxException(getPosition(), "unknown type of %string"); |
| } |
| } |
| if (c == EOF || begin == EOF) { |
| throw new SyntaxException(getPosition(), "unterminated quoted string meets end of file"); |
| } |
| |
| // Figure end-char. '\0' is special to indicate begin=end and that no nesting? |
| if (begin == '(') end = ')'; |
| else if (begin == '[') end = ']'; |
| else if (begin == '{') end = '}'; |
| else if (begin == '<') end = '>'; |
| else { end = begin; begin = '\0'; }; |
| |
| switch (c) { |
| case 'Q': |
| lex_strterm = new StringTerm(str_dquote, end, begin); |
| yaccValue = new Token("%"+ (shortHand ? (""+end) : ("" + c + begin)), getPosition()); |
| return Tokens.tSTRING_BEG; |
| |
| case 'q': |
| lex_strterm = new StringTerm(str_squote, end, begin); |
| yaccValue = new Token("%"+c+begin, getPosition()); |
| return Tokens.tSTRING_BEG; |
| |
| case 'W': |
| lex_strterm = new StringTerm(str_dquote | STR_FUNC_QWORDS, end, begin); |
| do {c = src.read();} while (Character.isWhitespace(c)); |
| src.unread(c); |
| yaccValue = new Token("%"+c+begin, getPosition()); |
| return Tokens.tWORDS_BEG; |
| |
| case 'w': |
| lex_strterm = new StringTerm(str_squote | STR_FUNC_QWORDS, end, begin); |
| do {c = src.read();} while (Character.isWhitespace(c)); |
| src.unread(c); |
| yaccValue = new Token("%"+c+begin, getPosition()); |
| return Tokens.tQWORDS_BEG; |
| |
| case 'x': |
| lex_strterm = new StringTerm(str_xquote, end, begin); |
| yaccValue = new Token("%"+c+begin, getPosition()); |
| return Tokens.tXSTRING_BEG; |
| |
| case 'r': |
| lex_strterm = new StringTerm(str_regexp, end, begin); |
| yaccValue = new Token("%"+c+begin, getPosition()); |
| return Tokens.tREGEXP_BEG; |
| |
| case 's': |
| lex_strterm = new StringTerm(str_ssym, end, begin); |
| lex_state = LexState.EXPR_FNAME; |
| yaccValue = new Token("%"+c+begin, getPosition()); |
| return Tokens.tSYMBEG; |
| |
| default: |
| throw new SyntaxException(getPosition(), "Unknown type of %string. Expected 'Q', 'q', 'w', 'x', 'r' or any non letter character, but found '" + c + "'."); |
| } |
| } |
| |
| private int hereDocumentIdentifier() throws IOException { |
| char c = src.read(); |
| int term; |
| |
| int func = 0; |
| if (c == '-') { |
| c = src.read(); |
| func = STR_FUNC_INDENT; |
| } |
| |
| if (c == '\'' || c == '"' || c == '`') { |
| if (c == '\'') { |
| func |= str_squote; |
| } else if (c == '"') { |
| func |= str_dquote; |
| } else { |
| func |= str_xquote; |
| } |
| |
| tokenBuffer.setLength(0); |
| term = c; |
| while ((c = src.read()) != EOF && c != term) { |
| tokenBuffer.append(c); |
| } |
| if (c == EOF) { |
| throw new SyntaxException(getPosition(), "unterminated here document identifier"); |
| } |
| } else { |
| if (!isIdentifierChar(c)) { |
| src.unread(c); |
| if ((func & STR_FUNC_INDENT) != 0) { |
| src.unread('-'); |
| } |
| return 0; |
| } |
| tokenBuffer.setLength(0); |
| term = '"'; |
| func |= str_dquote; |
| do { |
| tokenBuffer.append(c); |
| } while ((c = src.read()) != EOF && isIdentifierChar(c)); |
| src.unread(c); |
| } |
| |
| String line = src.readLine() + '\n'; |
| String tok = tokenBuffer.toString(); |
| lex_strterm = new HeredocTerm(tok, func, line); |
| |
| if (term == '`') { |
| yaccValue = new Token("`", getPosition()); |
| return Tokens.tXSTRING_BEG; |
| } |
| |
| yaccValue = new Token("\"", getPosition()); |
| // Hacky: Advance position to eat newline here.... |
| getPosition(); |
| return Tokens.tSTRING_BEG; |
| } |
| |
| private void arg_ambiguous() { |
| warnings.warning(getPosition(), "Ambiguous first argument; make sure."); |
| } |
| |
| /** |
| * Read a comment up to end of line. When found each comment will get stored away into |
| * the parser result so that any interested party can use them as they seem fit. One idea |
| * is that IDE authors can do distance based heuristics to associate these comments to the |
| * AST node they think they belong to. |
| * |
| * @param c last character read from lexer source |
| * @return newline or eof value |
| */ |
| protected int readComment(char c) throws IOException { |
| ISourcePosition startPosition = src.getPosition(); |
| tokenBuffer.setLength(0); |
| tokenBuffer.append(c); |
| |
| // FIXME: Consider making a better LexerSource.readLine |
| while ((c = src.read()) != '\n') { |
| tokenBuffer.append(c); |
| if (c == EOF) { |
| break; |
| } |
| } |
| src.unread(c); |
| |
| // Store away each comment to parser result so IDEs can do whatever they want with them. |
| ISourcePosition position = startPosition.union(getPosition()); |
| parserSupport.getResult().addComment(new CommentNode(position, tokenBuffer.toString())); |
| |
| return c; |
| } |
| |
| /* |
| * Not normally used, but is left in here since it can be useful in debugging |
| * grammar and lexing problems. |
| private void printToken(int token) { |
| //System.out.print("LOC: " + support.getPosition() + " ~ "); |
| |
| switch (token) { |
| case Tokens.yyErrorCode: System.err.print("yyErrorCode,"); break; |
| case Tokens.kCLASS: System.err.print("kClass,"); break; |
| case Tokens.kMODULE: System.err.print("kModule,"); break; |
| case Tokens.kDEF: System.err.print("kDEF,"); break; |
| case Tokens.kUNDEF: System.err.print("kUNDEF,"); break; |
| case Tokens.kBEGIN: System.err.print("kBEGIN,"); break; |
| case Tokens.kRESCUE: System.err.print("kRESCUE,"); break; |
| case Tokens.kENSURE: System.err.print("kENSURE,"); break; |
| case Tokens.kEND: System.err.print("kEND,"); break; |
| case Tokens.kIF: System.err.print("kIF,"); break; |
| case Tokens.kUNLESS: System.err.print("kUNLESS,"); break; |
| case Tokens.kTHEN: System.err.print("kTHEN,"); break; |
| case Tokens.kELSIF: System.err.print("kELSIF,"); break; |
| case Tokens.kELSE: System.err.print("kELSE,"); break; |
| case Tokens.kCASE: System.err.print("kCASE,"); break; |
| case Tokens.kWHEN: System.err.print("kWHEN,"); break; |
| case Tokens.kWHILE: System.err.print("kWHILE,"); break; |
| case Tokens.kUNTIL: System.err.print("kUNTIL,"); break; |
| case Tokens.kFOR: System.err.print("kFOR,"); break; |
| case Tokens.kBREAK: System.err.print("kBREAK,"); break; |
| case Tokens.kNEXT: System.err.print("kNEXT,"); break; |
| case Tokens.kREDO: System.err.print("kREDO,"); break; |
| case Tokens.kRETRY: System.err.print("kRETRY,"); break; |
| case Tokens.kIN: System.err.print("kIN,"); break; |
| case Tokens.kDO: System.err.print("kDO,"); break; |
| case Tokens.kDO_COND: System.err.print("kDO_COND,"); break; |
| case Tokens.kDO_BLOCK: System.err.print("kDO_BLOCK,"); break; |
| case Tokens.kRETURN: System.err.print("kRETURN,"); break; |
| case Tokens.kYIELD: System.err.print("kYIELD,"); break; |
| case Tokens.kSUPER: System.err.print("kSUPER,"); break; |
| case Tokens.kSELF: System.err.print("kSELF,"); break; |
| case Tokens.kNIL: System.err.print("kNIL,"); break; |
| case Tokens.kTRUE: System.err.print("kTRUE,"); break; |
| case Tokens.kFALSE: System.err.print("kFALSE,"); break; |
| case Tokens.kAND: System.err.print("kAND,"); break; |
| case Tokens.kOR: System.err.print("kOR,"); break; |
| case Tokens.kNOT: System.err.print("kNOT,"); break; |
| case Tokens.kIF_MOD: System.err.print("kIF_MOD,"); break; |
| case Tokens.kUNLESS_MOD: System.err.print("kUNLESS_MOD,"); break; |
| case Tokens.kWHILE_MOD: System.err.print("kWHILE_MOD,"); break; |
| case Tokens.kUNTIL_MOD: System.err.print("kUNTIL_MOD,"); break; |
| case Tokens.kRESCUE_MOD: System.err.print("kRESCUE_MOD,"); break; |
| case Tokens.kALIAS: System.err.print("kALIAS,"); break; |
| case Tokens.kDEFINED: System.err.print("kDEFINED,"); break; |
| case Tokens.klBEGIN: System.err.print("klBEGIN,"); break; |
| case Tokens.klEND: System.err.print("klEND,"); break; |
| case Tokens.k__LINE__: System.err.print("k__LINE__,"); break; |
| case Tokens.k__FILE__: System.err.print("k__FILE__,"); break; |
| case Tokens.tIDENTIFIER: System.err.print("tIDENTIFIER["+ value() + "],"); break; |
| case Tokens.tFID: System.err.print("tFID[" + value() + "],"); break; |
| case Tokens.tGVAR: System.err.print("tGVAR[" + value() + "],"); break; |
| case Tokens.tIVAR: System.err.print("tIVAR[" + value() +"],"); break; |
| case Tokens.tCONSTANT: System.err.print("tCONSTANT["+ value() +"],"); break; |
| case Tokens.tCVAR: System.err.print("tCVAR,"); break; |
| case Tokens.tINTEGER: System.err.print("tINTEGER,"); break; |
| case Tokens.tFLOAT: System.err.print("tFLOAT,"); break; |
| case Tokens.tSTRING_CONTENT: System.err.print("tSTRING_CONTENT[" + yaccValue + "],"); break; |
| case Tokens.tSTRING_BEG: System.err.print("tSTRING_BEG,"); break; |
| case Tokens.tSTRING_END: System.err.print("tSTRING_END,"); break; |
| case Tokens.tSTRING_DBEG: System.err.print("STRING_DBEG,"); break; |
| case Tokens.tSTRING_DVAR: System.err.print("tSTRING_DVAR,"); break; |
| case Tokens.tXSTRING_BEG: System.err.print("tXSTRING_BEG,"); break; |
| case Tokens.tREGEXP_BEG: System.err.print("tREGEXP_BEG,"); break; |
| case Tokens.tREGEXP_END: System.err.print("tREGEXP_END,"); break; |
| case Tokens.tWORDS_BEG: System.err.print("tWORDS_BEG,"); break; |
| case Tokens.tQWORDS_BEG: System.err.print("tQWORDS_BEG,"); break; |
| case Tokens.tBACK_REF: System.err.print("tBACK_REF,"); break; |
| case Tokens.tNTH_REF: System.err.print("tNTH_REF,"); break; |
| case Tokens.tUPLUS: System.err.print("tUPLUS"); break; |
| case Tokens.tUMINUS: System.err.print("tUMINUS,"); break; |
| case Tokens.tPOW: System.err.print("tPOW,"); break; |
| case Tokens.tCMP: System.err.print("tCMP,"); break; |
| case Tokens.tEQ: System.err.print("tEQ,"); break; |
| case Tokens.tEQQ: System.err.print("tEQQ,"); break; |
| case Tokens.tNEQ: System.err.print("tNEQ,"); break; |
| case Tokens.tGEQ: System.err.print("tGEQ,"); break; |
| case Tokens.tLEQ: System.err.print("tLEQ,"); break; |
| case Tokens.tANDOP: System.err.print("tANDOP,"); break; |
| case Tokens.tOROP: System.err.print("tOROP,"); break; |
| case Tokens.tMATCH: System.err.print("tMATCH,"); break; |
| case Tokens.tNMATCH: System.err.print("tNMATCH,"); break; |
| case Tokens.tDOT2: System.err.print("tDOT2,"); break; |
| case Tokens.tDOT3: System.err.print("tDOT3,"); break; |
| case Tokens.tAREF: System.err.print("tAREF,"); break; |
| case Tokens.tASET: System.err.print("tASET,"); break; |
| case Tokens.tLSHFT: System.err.print("tLSHFT,"); break; |
| case Tokens.tRSHFT: System.err.print("tRSHFT,"); break; |
| case Tokens.tCOLON2: System.err.print("tCOLON2,"); break; |
| case Tokens.tCOLON3: System.err.print("tCOLON3,"); break; |
| case Tokens.tOP_ASGN: System.err.print("tOP_ASGN,"); break; |
| case Tokens.tASSOC: System.err.print("tASSOC,"); break; |
| case Tokens.tLPAREN: System.err.print("tLPAREN,"); break; |
| case Tokens.tLPAREN_ARG: System.err.print("tLPAREN_ARG,"); break; |
| case Tokens.tLBRACK: System.err.print("tLBRACK,"); break; |
| case Tokens.tLBRACE: System.err.print("tLBRACE,"); break; |
| case Tokens.tSTAR: System.err.print("tSTAR,"); break; |
| case Tokens.tAMPER: System.err.print("tAMPER,"); break; |
| case Tokens.tSYMBEG: System.err.print("tSYMBEG,"); break; |
| case '\n': System.err.println("NL"); break; |
| default: System.err.print("'" + (int)token + "',"); break; |
| } |
| } |
| |
| // DEBUGGING HELP |
| private int yylex() { |
| int token = yylex2(); |
| |
| printToken(token); |
| |
| return token; |
| } |
| */ |
| |
| /** |
| * Returns the next token. Also sets yyVal is needed. |
| * |
| *@return Description of the Returned Value |
| */ |
| private int yylex() throws IOException { |
| char c; |
| boolean spaceSeen = false; |
| boolean commandState; |
| |
| if (lex_strterm != null) { |
| int tok = lex_strterm.parseString(this, src); |
| if (tok == Tokens.tSTRING_END || tok == Tokens.tREGEXP_END) { |
| lex_strterm = null; |
| lex_state = LexState.EXPR_END; |
| } |
| return tok; |
| } |
| |
| commandState = commandStart; |
| commandStart = false; |
| |
| LexState last_state = lex_state; |
| |
| retry: for(;;) { |
| c = src.read(); |
| switch(c) { |
| case '\004': /* ^D */ |
| case '\032': /* ^Z */ |
| case 0: /* end of script. */ |
| return 0; |
| |
| /* white spaces */ |
| case ' ': case '\t': case '\f': case '\r': |
| case '\13': /* '\v' */ |
| getPosition(); |
| spaceSeen = true; |
| continue retry; |
| case '#': /* it's a comment */ |
| if (readComment(c) == 0) return 0; |
| |
| /* fall through */ |
| case '\n': |
| // Replace a string of newlines with a single one |
| while((c = src.read()) == '\n') { |
| |
| } |
| src.unread( c ); |
| getPosition(); |
| |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT || |
| lex_state == LexState.EXPR_CLASS) { |
| continue retry; |
| } |
| |
| commandStart = true; |
| lex_state = LexState.EXPR_BEG; |
| return '\n'; |
| |
| case '*': |
| if ((c = src.read()) == '*') { |
| if ((c = src.read()) == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("**", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| yaccValue = new Token("**", getPosition()); |
| c = Tokens.tPOW; |
| } else { |
| if (c == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("*", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| if (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c)) { |
| warnings.warning(getPosition(), "`*' interpreted as argument prefix"); |
| c = Tokens.tSTAR; |
| } else if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID) { |
| c = Tokens.tSTAR; |
| } else { |
| c = Tokens.tSTAR2; |
| } |
| yaccValue = new Token("*", getPosition()); |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| return c; |
| |
| case '!': |
| lex_state = LexState.EXPR_BEG; |
| if ((c = src.read()) == '=') { |
| yaccValue = new Token("!=",getPosition()); |
| return Tokens.tNEQ; |
| } |
| if (c == '~') { |
| yaccValue = new Token("!~",getPosition()); |
| return Tokens.tNMATCH; |
| } |
| src.unread(c); |
| yaccValue = new Token("!",getPosition()); |
| return Tokens.tBANG; |
| |
| case '=': |
| // documentation nodes |
| if (src.wasBeginOfLine()) { |
| String equalLabel; |
| if ((equalLabel = isNextNoCase("begin")) != null) { |
| tokenBuffer.setLength(0); |
| tokenBuffer.append(equalLabel); |
| c = src.read(); |
| |
| if (Character.isWhitespace(c)) { |
| // In case last next was the newline. |
| src.unread(c); |
| for (;;) { |
| c = src.read(); |
| tokenBuffer.append(c); |
| |
| // If a line is followed by a blank line put |
| // it back. |
| while (c == '\n') { |
| c = src.read(); |
| tokenBuffer.append(c); |
| } |
| if (c == EOF) { |
| throw new SyntaxException(getPosition(), "embedded document meets end of file"); |
| } |
| if (c != '=') continue; |
| if (src.wasBeginOfLine() && (equalLabel = isNextNoCase("end")) != null) { |
| tokenBuffer.append(equalLabel); |
| tokenBuffer.append(src.readLine()); |
| src.unread('\n'); |
| break; |
| } |
| } |
| |
| parserSupport.getResult().addComment(new CommentNode(getPosition(), tokenBuffer.toString())); |
| continue retry; |
| } |
| src.unread(c); |
| } |
| } |
| |
| if (lex_state == LexState.EXPR_FNAME || lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| |
| c = src.read(); |
| if (c == '=') { |
| c = src.read(); |
| if (c == '=') { |
| yaccValue = new Token("===", getPosition()); |
| return Tokens.tEQQ; |
| } |
| src.unread(c); |
| yaccValue = new Token("==", getPosition()); |
| return Tokens.tEQ; |
| } |
| if (c == '~') { |
| yaccValue = new Token("=~", getPosition()); |
| return Tokens.tMATCH; |
| } else if (c == '>') { |
| yaccValue = new Token("=>", getPosition()); |
| return Tokens.tASSOC; |
| } |
| src.unread(c); |
| yaccValue = new Token("=", getPosition()); |
| return '='; |
| |
| case '<': |
| c = src.read(); |
| if (c == '<' && |
| lex_state != LexState.EXPR_END && |
| lex_state != LexState.EXPR_DOT && |
| lex_state != LexState.EXPR_ENDARG && |
| lex_state != LexState.EXPR_CLASS && |
| (!lex_state.isArgument() || spaceSeen)) { |
| int tok = hereDocumentIdentifier(); |
| if (tok != 0) return tok; |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| if (c == '=') { |
| if ((c = src.read()) == '>') { |
| yaccValue = new Token("<=>", getPosition()); |
| return Tokens.tCMP; |
| } |
| src.unread(c); |
| yaccValue = new Token("<=", getPosition()); |
| return Tokens.tLEQ; |
| } |
| if (c == '<') { |
| if ((c = src.read()) == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("<<", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| yaccValue = new Token("<<", getPosition()); |
| return Tokens.tLSHFT; |
| } |
| yaccValue = new Token("<", getPosition()); |
| src.unread(c); |
| return Tokens.tLT; |
| |
| case '>': |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| |
| if ((c = src.read()) == '=') { |
| yaccValue = new Token(">=", getPosition()); |
| return Tokens.tGEQ; |
| } |
| if (c == '>') { |
| if ((c = src.read()) == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token(">>", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| yaccValue = new Token(">>", getPosition()); |
| return Tokens.tRSHFT; |
| } |
| src.unread(c); |
| yaccValue = new Token(">", getPosition()); |
| return Tokens.tGT; |
| |
| case '"': |
| lex_strterm = new StringTerm(str_dquote, '"', '\0'); |
| yaccValue = new Token("\"", getPosition()); |
| return Tokens.tSTRING_BEG; |
| |
| case '`': |
| yaccValue = new Token("`", getPosition()); |
| if (lex_state == LexState.EXPR_FNAME) { |
| lex_state = LexState.EXPR_END; |
| return Tokens.tBACK_REF2; |
| } |
| if (lex_state == LexState.EXPR_DOT) { |
| if (commandState) { |
| lex_state = LexState.EXPR_CMDARG; |
| } else { |
| lex_state = LexState.EXPR_ARG; |
| } |
| return Tokens.tBACK_REF2; |
| } |
| lex_strterm = new StringTerm(str_xquote, '`', '\0'); |
| return Tokens.tXSTRING_BEG; |
| |
| case '\'': |
| lex_strterm = new StringTerm(str_squote, '\'', '\0'); |
| yaccValue = new Token("'", getPosition()); |
| return Tokens.tSTRING_BEG; |
| |
| case '?': |
| if (lex_state == LexState.EXPR_END || |
| lex_state == LexState.EXPR_ENDARG) { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("?",getPosition()); |
| return '?'; |
| } |
| c = src.read(); |
| if (c == EOF) { |
| throw new SyntaxException(getPosition(), "incomplete character syntax"); |
| } |
| if (Character.isWhitespace(c)){ |
| if (!lex_state.isArgument()){ |
| int c2 = 0; |
| switch (c) { |
| case ' ': |
| c2 = 's'; |
| break; |
| case '\n': |
| c2 = 'n'; |
| break; |
| case '\t': |
| c2 = 't'; |
| break; |
| /* What is \v in C? |
| case '\v': |
| c2 = 'v'; |
| break; |
| */ |
| case '\r': |
| c2 = 'r'; |
| break; |
| case '\f': |
| c2 = 'f'; |
| break; |
| } |
| if (c2 != 0) { |
| warnings.warn(getPosition(), "invalid character syntax; use ?\\" + c2); |
| } |
| } |
| src.unread(c); |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("?", getPosition()); |
| return '?'; |
| /*} else if (ismbchar(c)) { // ruby - we don't support them either? |
| rb_warn("multibyte character literal not supported yet; use ?\\" + c); |
| support.unread(c); |
| lexState = LexState.EXPR_BEG; |
| return '?';*/ |
| } else if ((Character.isLetterOrDigit(c) || c == '_') && |
| !src.peek('\n') && isNext_identchar()) { |
| src.unread(c); |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("?", getPosition()); |
| return '?'; |
| } else if (c == '\\') { |
| c = src.readEscape(); |
| } |
| c &= 0xff; |
| lex_state = LexState.EXPR_END; |
| yaccValue = new FixnumNode(getPosition(), c); |
| return Tokens.tINTEGER; |
| |
| case '&': |
| if ((c = src.read()) == '&') { |
| lex_state = LexState.EXPR_BEG; |
| if ((c = src.read()) == '=') { |
| yaccValue = new Token("&&", getPosition()); |
| lex_state = LexState.EXPR_BEG; |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| yaccValue = new Token("&&", getPosition()); |
| return Tokens.tANDOP; |
| } |
| else if (c == '=') { |
| yaccValue = new Token("&", getPosition()); |
| lex_state = LexState.EXPR_BEG; |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| //tmpPosition is required because getPosition()'s side effects. |
| //if the warning is generated, the getPosition() on line 954 (this line + 18) will create |
| //a wrong position if the "inclusive" flag is not set. |
| ISourcePosition tmpPosition = getPosition(); |
| if (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c)){ |
| warnings.warning(tmpPosition, "`&' interpreted as argument prefix"); |
| c = Tokens.tAMPER; |
| } else if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID) { |
| c = Tokens.tAMPER; |
| } else { |
| c = Tokens.tAMPER2; |
| } |
| |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| yaccValue = new Token("&", tmpPosition); |
| return c; |
| |
| case '|': |
| if ((c = src.read()) == '|') { |
| lex_state = LexState.EXPR_BEG; |
| if ((c = src.read()) == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("||", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| yaccValue = new Token("||", getPosition()); |
| return Tokens.tOROP; |
| } |
| if (c == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("|", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| src.unread(c); |
| yaccValue = new Token("|", getPosition()); |
| return Tokens.tPIPE; |
| |
| case '+': |
| c = src.read(); |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| if (c == '@') { |
| yaccValue = new Token("+@", getPosition()); |
| return Tokens.tUPLUS; |
| } |
| src.unread(c); |
| yaccValue = new Token("+", getPosition()); |
| return Tokens.tPLUS; |
| } |
| if (c == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("+", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID || |
| (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c))) { |
| if (lex_state.isArgument()) arg_ambiguous(); |
| lex_state = LexState.EXPR_BEG; |
| src.unread(c); |
| if (Character.isDigit(c)) { |
| c = '+'; |
| return parseNumber(c); |
| } |
| yaccValue = new Token("+", getPosition()); |
| return Tokens.tUPLUS; |
| } |
| lex_state = LexState.EXPR_BEG; |
| src.unread(c); |
| yaccValue = new Token("+", getPosition()); |
| return Tokens.tPLUS; |
| |
| case '-': |
| c = src.read(); |
| if (lex_state == LexState.EXPR_FNAME || lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| if (c == '@') { |
| yaccValue = new Token("-@", getPosition()); |
| return Tokens.tUMINUS; |
| } |
| src.unread(c); |
| yaccValue = new Token("-", getPosition()); |
| return Tokens.tMINUS; |
| } |
| if (c == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("-", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| if (lex_state == LexState.EXPR_BEG || lex_state == LexState.EXPR_MID || |
| (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c))) { |
| if (lex_state.isArgument()) arg_ambiguous(); |
| lex_state = LexState.EXPR_BEG; |
| src.unread(c); |
| yaccValue = new Token("-", getPosition()); |
| if (Character.isDigit(c)) { |
| return Tokens.tUMINUS_NUM; |
| } |
| return Tokens.tUMINUS; |
| } |
| lex_state = LexState.EXPR_BEG; |
| src.unread(c); |
| yaccValue = new Token("-", getPosition()); |
| return Tokens.tMINUS; |
| |
| case '.': |
| lex_state = LexState.EXPR_BEG; |
| if ((c = src.read()) == '.') { |
| if ((c = src.read()) == '.') { |
| yaccValue = new Token("...", getPosition()); |
| return Tokens.tDOT3; |
| } |
| src.unread(c); |
| yaccValue = new Token("..", getPosition()); |
| return Tokens.tDOT2; |
| } |
| src.unread(c); |
| if (Character.isDigit(c)) { |
| throw new SyntaxException(getPosition(), "no .<digit> floating literal anymore; put 0 before dot"); |
| } |
| lex_state = LexState.EXPR_DOT; |
| yaccValue = new Token(".", getPosition()); |
| return Tokens.tDOT; |
| case '0' : case '1' : case '2' : case '3' : case '4' : |
| case '5' : case '6' : case '7' : case '8' : case '9' : |
| return parseNumber(c); |
| |
| case ')': |
| conditionState.restart(); |
| cmdArgumentState.restart(); |
| lex_state = LexState.EXPR_END; |
| yaccValue = new Token(")", getPosition()); |
| return Tokens.tRPAREN; |
| case ']': |
| conditionState.restart(); |
| cmdArgumentState.restart(); |
| lex_state = LexState.EXPR_END; |
| yaccValue = new Token(")", getPosition()); |
| return Tokens.tRBRACK; |
| case '}': |
| conditionState.restart(); |
| cmdArgumentState.restart(); |
| lex_state = LexState.EXPR_END; |
| yaccValue = new Token("}",getPosition()); |
| return Tokens.tRCURLY; |
| |
| case ':': |
| c = src.read(); |
| if (c == ':') { |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID || |
| lex_state == LexState.EXPR_CLASS || |
| (lex_state.isArgument() && spaceSeen)) { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("::", getPosition()); |
| return Tokens.tCOLON3; |
| } |
| lex_state = LexState.EXPR_DOT; |
| yaccValue = new Token(":",getPosition()); |
| return Tokens.tCOLON2; |
| } |
| if (lex_state == LexState.EXPR_END || |
| lex_state == LexState.EXPR_ENDARG || Character.isWhitespace(c)) { |
| src.unread(c); |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token(":",getPosition()); |
| return ':'; |
| } |
| switch (c) { |
| case '\'': |
| lex_strterm = new StringTerm(str_ssym, c, '\0'); |
| break; |
| case '"': |
| lex_strterm = new StringTerm(str_dsym, c, '\0'); |
| break; |
| default: |
| src.unread(c); |
| break; |
| } |
| lex_state = LexState.EXPR_FNAME; |
| yaccValue = new Token(":", getPosition()); |
| return Tokens.tSYMBEG; |
| |
| case '/': |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID) { |
| lex_strterm = new StringTerm(str_regexp, '/', '\0'); |
| yaccValue = new Token("/",getPosition()); |
| return Tokens.tREGEXP_BEG; |
| } |
| |
| if ((c = src.read()) == '=') { |
| yaccValue = new Token("/", getPosition()); |
| lex_state = LexState.EXPR_BEG; |
| return Tokens.tOP_ASGN; |
| } |
| src.unread(c); |
| if (lex_state.isArgument() && spaceSeen) { |
| if (!Character.isWhitespace(c)) { |
| arg_ambiguous(); |
| lex_strterm = new StringTerm(str_regexp, '/', '\0'); |
| yaccValue = new Token("/",getPosition()); |
| return Tokens.tREGEXP_BEG; |
| } |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| yaccValue = new Token("/", getPosition()); |
| return Tokens.tDIVIDE; |
| |
| case '^': |
| if ((c = src.read()) == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("^", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| src.unread(c); |
| yaccValue = new Token("^", getPosition()); |
| return Tokens.tCARET; |
| |
| case ';': |
| commandStart = true; |
| case ',': |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token(",", getPosition()); |
| return c; |
| |
| case '~': |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| if ((c = src.read()) != '@') { |
| src.unread(c); |
| } |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| yaccValue = new Token("~", getPosition()); |
| return Tokens.tTILDE; |
| case '(': |
| c = Tokens.tLPAREN2; |
| commandStart = true; |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID) { |
| c = Tokens.tLPAREN; |
| } else if (spaceSeen) { |
| if (lex_state == LexState.EXPR_CMDARG) { |
| c = Tokens.tLPAREN_ARG; |
| } else if (lex_state == LexState.EXPR_ARG) { |
| warnings.warn(getPosition(), "don't put space before argument parentheses"); |
| c = Tokens.tLPAREN2; |
| } |
| } |
| conditionState.stop(); |
| cmdArgumentState.stop(); |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("(", getPosition()); |
| return c; |
| |
| case '[': |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| if ((c = src.read()) == ']') { |
| if (src.peek('=')) { |
| c = src.read(); |
| yaccValue = new Token("[]=", getPosition()); |
| return Tokens.tASET; |
| } |
| yaccValue = new Token("[]", getPosition()); |
| return Tokens.tAREF; |
| } |
| src.unread(c); |
| yaccValue = new Token("[", getPosition()); |
| return '['; |
| } else if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID) { |
| c = Tokens.tLBRACK; |
| } else if (lex_state.isArgument() && spaceSeen) { |
| c = Tokens.tLBRACK; |
| } |
| lex_state = LexState.EXPR_BEG; |
| conditionState.stop(); |
| cmdArgumentState.stop(); |
| yaccValue = new Token("[", getPosition()); |
| return c; |
| |
| case '{': |
| c = Tokens.tLCURLY; |
| |
| if (lex_state.isArgument() || lex_state == LexState.EXPR_END) { |
| c = Tokens.tLCURLY; /* block (primary) */ |
| } else if (lex_state == LexState.EXPR_ENDARG) { |
| c = Tokens.tLBRACE_ARG; /* block (expr) */ |
| } else { |
| c = Tokens.tLBRACE; /* hash */ |
| } |
| conditionState.stop(); |
| cmdArgumentState.stop(); |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("{", getPosition()); |
| return c; |
| |
| case '\\': |
| c = src.read(); |
| if (c == '\n') { |
| spaceSeen = true; |
| continue retry; /* skip \\n */ |
| } |
| src.unread(c); |
| yaccValue = new Token("\\", getPosition()); |
| return '\\'; |
| |
| case '%': |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID) { |
| return parseQuote(src.read()); |
| } |
| if ((c = src.read()) == '=') { |
| lex_state = LexState.EXPR_BEG; |
| yaccValue = new Token("%", getPosition()); |
| return Tokens.tOP_ASGN; |
| } |
| if (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c)) { |
| return parseQuote(c); |
| } |
| if (lex_state == LexState.EXPR_FNAME || |
| lex_state == LexState.EXPR_DOT) { |
| lex_state = LexState.EXPR_ARG; |
| } else { |
| lex_state = LexState.EXPR_BEG; |
| } |
| src.unread(c); |
| yaccValue = new Token("%", getPosition()); |
| return Tokens.tPERCENT; |
| |
| case '$': |
| lex_state = LexState.EXPR_END; |
| tokenBuffer.setLength(0); |
| c = src.read(); |
| switch (c) { |
| case '_': /* $_: last read line string */ |
| c = src.read(); |
| if (isIdentifierChar(c)) { |
| tokenBuffer.append('$'); |
| tokenBuffer.append('_'); |
| break; |
| } |
| src.unread(c); |
| c = '_'; |
| /* fall through */ |
| case '~': /* $~: match-data */ |
| case '*': /* $*: argv */ |
| case '$': /* $$: pid */ |
| case '?': /* $?: last status */ |
| case '!': /* $!: error string */ |
| case '@': /* $@: error position */ |
| case '/': /* $/: input record separator */ |
| case '\\': /* $\: output record separator */ |
| case ';': /* $;: field separator */ |
| case ',': /* $,: output field separator */ |
| case '.': /* $.: last read line number */ |
| case '=': /* $=: ignorecase */ |
| case ':': /* $:: load path */ |
| case '<': /* $<: reading filename */ |
| case '>': /* $>: default output handle */ |
| case '\"': /* $": already loaded files */ |
| tokenBuffer.append('$'); |
| tokenBuffer.append(c); |
| yaccValue = new Token(tokenBuffer.toString(), getPosition()); |
| return Tokens.tGVAR; |
| |
| case '-': |
| tokenBuffer.append('$'); |
| tokenBuffer.append(c); |
| c = src.read(); |
| if (isIdentifierChar(c)) { |
| tokenBuffer.append(c); |
| } else { |
| src.unread(c); |
| } |
| yaccValue = new Token(tokenBuffer.toString(), getPosition()); |
| /* xxx shouldn't check if valid option variable */ |
| return Tokens.tGVAR; |
| |
| case '&': /* $&: last match */ |
| case '`': /* $`: string before last match */ |
| case '\'': /* $': string after last match */ |
| case '+': /* $+: string matches last paren. */ |
| yaccValue = new BackRefNode(getPosition(), c); |
| return Tokens.tBACK_REF; |
| |
| case '1': case '2': case '3': |
| case '4': case '5': case '6': |
| case '7': case '8': case '9': |
| tokenBuffer.append('$'); |
| do { |
| tokenBuffer.append(c); |
| c = src.read(); |
| } while (Character.isDigit(c)); |
| src.unread(c); |
| if(last_state == LexState.EXPR_FNAME) { |
| yaccValue = new Token(tokenBuffer.toString(), getPosition()); |
| return Tokens.tGVAR; |
| } else { |
| yaccValue = new NthRefNode(getPosition(), Integer.parseInt(tokenBuffer.substring(1))); |
| return Tokens.tNTH_REF; |
| } |
| default: |
| if (!isIdentifierChar(c)) { |
| src.unread(c); |
| yaccValue = new Token("$", getPosition()); |
| return '$'; |
| } |
| case '0': |
| tokenBuffer.append('$'); |
| } |
| break; |
| |
| case '@': |
| c = src.read(); |
| tokenBuffer.setLength(0); |
| tokenBuffer.append('@'); |
| if (c == '@') { |
| tokenBuffer.append('@'); |
| c = src.read(); |
| } |
| if (Character.isDigit(c)) { |
| if (tokenBuffer.length() == 1) { |
| throw new SyntaxException(getPosition(), "`@" + c + "' is not allowed as an instance variable name"); |
| } |
| throw new SyntaxException(getPosition(), "`@@" + c + "' is not allowed as a class variable name"); |
| } |
| if (!isIdentifierChar(c)) { |
| src.unread(c); |
| yaccValue = new Token("@", getPosition()); |
| return '@'; |
| } |
| break; |
| |
| case '_': |
| if (src.wasBeginOfLine() && src.matchString("_END__\n", false)) { |
| parserSupport.getResult().setEndSeen(true); |
| return 0; |
| } |
| tokenBuffer.setLength(0); |
| break; |
| |
| default: |
| if (!isIdentifierChar(c)) { |
| throw new SyntaxException(getPosition(), "Invalid char `\\" + Integer.parseInt(""+c, 8) + "' in expression"); |
| } |
| |
| tokenBuffer.setLength(0); |
| break; |
| } |
| |
| do { |
| tokenBuffer.append(c); |
| /* no special multibyte character handling is needed in Java |
| * if (ismbchar(c)) { |
| int i, len = mbclen(c)-1; |
| |
| for (i = 0; i < len; i++) { |
| c = src.read(); |
| tokenBuffer.append(c); |
| } |
| }*/ |
| c = src.read(); |
| } while (isIdentifierChar(c)); |
| |
| char peek = src.read(); |
| if ((c == '!' || c == '?') && |
| isIdentifierChar(tokenBuffer.charAt(0)) && peek != '=') { |
| src.unread(peek); |
| tokenBuffer.append(c); |
| } else { |
| src.unread(peek); |
| src.unread(c); |
| } |
| |
| int result = 0; |
| |
| switch (tokenBuffer.charAt(0)) { |
| case '$': |
| lex_state = LexState.EXPR_END; |
| result = Tokens.tGVAR; |
| break; |
| case '@': |
| lex_state = LexState.EXPR_END; |
| if (tokenBuffer.charAt(1) == '@') { |
| result = Tokens.tCVAR; |
| } else { |
| result = Tokens.tIVAR; |
| } |
| break; |
| |
| default: |
| char last = tokenBuffer.charAt(tokenBuffer.length() - 1); |
| if (last == '!' || last == '?') { |
| result = Tokens.tFID; |
| } else { |
| if (lex_state == LexState.EXPR_FNAME) { |
| if ((c = src.read()) == '=') { |
| char c2 = src.read(); |
| |
| if (c2 != '~' && c2 != '>' && |
| (c2 != '=' || (c2 == '\n' && src.peek('>')))) { |
| result = Tokens.tIDENTIFIER; |
| tokenBuffer.append(c); |
| src.unread(c2); |
| } else { |
| src.unread(c2); |
| src.unread(c); |
| } |
| } else { |
| src.unread(c); |
| } |
| } |
| if (result == 0 && Character.isUpperCase(tokenBuffer.charAt(0))) { |
| result = Tokens.tCONSTANT; |
| } else { |
| result = Tokens.tIDENTIFIER; |
| } |
| } |
| |
| if (lex_state != LexState.EXPR_DOT) { |
| /* See if it is a reserved word. */ |
| Keyword keyword = Keyword.getKeyword(tokenBuffer.toString(), tokenBuffer.length()); |
| if (keyword != null) { |
| // enum lex_state |
| LexState state = lex_state; |
| |
| lex_state = keyword.state; |
| if (state.isExprFName()) { |
| yaccValue = new Token(keyword.name, getPosition()); |
| } else { |
| yaccValue = new Token(tokenBuffer.toString(), getPosition()); |
| } |
| if (keyword.id0 == Tokens.kDO) { |
| if (conditionState.isInState()) { |
| return Tokens.kDO_COND; |
| } |
| if (cmdArgumentState.isInState() && state != LexState.EXPR_CMDARG) { |
| return Tokens.kDO_BLOCK; |
| } |
| if (state == LexState.EXPR_ENDARG) { |
| return Tokens.kDO_BLOCK; |
| } |
| return Tokens.kDO; |
| } |
| |
| if (state == LexState.EXPR_BEG) { |
| return keyword.id0; |
| } |
| if (keyword.id0 != keyword.id1) { |
| lex_state = LexState.EXPR_BEG; |
| } |
| return keyword.id1; |
| } |
| } |
| |
| if (lex_state == LexState.EXPR_BEG || |
| lex_state == LexState.EXPR_MID || |
| lex_state == LexState.EXPR_DOT || |
| lex_state == LexState.EXPR_ARG || |
| lex_state == LexState.EXPR_CMDARG) { |
| if (commandState) { |
| lex_state = LexState.EXPR_CMDARG; |
| } else { |
| lex_state = LexState.EXPR_ARG; |
| } |
| } else { |
| lex_state = LexState.EXPR_END; |
| } |
| } |
| |
| String tempVal = tokenBuffer.toString(); |
| |
| // Lame: parsing logic made it into lexer in ruby...So we |
| // are emulating |
| // FIXME: I believe this is much simpler now... |
| StaticScope scope = parserSupport.getCurrentScope(); |
| if (IdUtil.getVarType(tempVal) == IdUtil.LOCAL_VAR && |
| (scope instanceof BlockStaticScope && (scope.isDefined(tempVal) >= 0)) || |
| (scope.getLocalScope().isDefined(tempVal) >= 0)) { |
| lex_state = LexState.EXPR_END; |
| } |
| |
| yaccValue = new Token(tempVal, getPosition()); |
| |
| return result; |
| } |
| } |
| |
| /** |
| * Parse a number from the input stream. |
| * |
| *@param c The first character of the number. |
| *@return A int constant wich represents a token. |
| */ |
| private int parseNumber(char c) throws IOException { |
| lex_state = LexState.EXPR_END; |
| |
| tokenBuffer.setLength(0); |
| |
| if (c == '-') { |
| tokenBuffer.append(c); |
| c = src.read(); |
| } else if (c == '+') { |
| // We don't append '+' since Java number parser gets confused |
| c = src.read(); |
| } |
| |
| char nondigit = '\0'; |
| |
| if (c == '0') { |
| int startLen = tokenBuffer.length(); |
| |
| switch (c = src.read()) { |
| case 'x' : |
| case 'X' : // hexadecimal |
| c = src.read(); |
| if (isHexChar(c)) { |
| for (;; c = src.read()) { |
| if (c == '_') { |
| if (nondigit != '\0') { |
| break; |
| } |
| nondigit = c; |
| } else if (isHexChar(c)) { |
| nondigit = '\0'; |
| tokenBuffer.append(c); |
| } else { |
| break; |
| } |
| } |
| } |
| src.unread(c); |
| |
| if (tokenBuffer.length() == startLen) { |
| throw new SyntaxException(getPosition(), "Hexadecimal number without hex-digits."); |
| } else if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } |
| yaccValue = getInteger(tokenBuffer.toString(), 16); |
| return Tokens.tINTEGER; |
| case 'b' : |
| case 'B' : // binary |
| c = src.read(); |
| if (c == '0' || c == '1') { |
| for (;; c = src.read()) { |
| if (c == '_') { |
| if (nondigit != '\0') { |
| break; |
| } |
| nondigit = c; |
| } else if (c == '0' || c == '1') { |
| nondigit = '\0'; |
| tokenBuffer.append(c); |
| } else { |
| break; |
| } |
| } |
| } |
| src.unread(c); |
| |
| if (tokenBuffer.length() == startLen) { |
| throw new SyntaxException(getPosition(), "Binary number without digits."); |
| } else if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } |
| yaccValue = getInteger(tokenBuffer.toString(), 2); |
| return Tokens.tINTEGER; |
| case 'd' : |
| case 'D' : // decimal |
| c = src.read(); |
| if (Character.isDigit(c)) { |
| for (;; c = src.read()) { |
| if (c == '_') { |
| if (nondigit != '\0') { |
| break; |
| } |
| nondigit = c; |
| } else if (Character.isDigit(c)) { |
| nondigit = '\0'; |
| tokenBuffer.append(c); |
| } else { |
| break; |
| } |
| } |
| } |
| src.unread(c); |
| |
| if (tokenBuffer.length() == startLen) { |
| throw new SyntaxException(getPosition(), "Binary number without digits."); |
| } else if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } |
| yaccValue = getInteger(tokenBuffer.toString(), 2); |
| return Tokens.tINTEGER; |
| case '0' : case '1' : case '2' : case '3' : case '4' : //Octal |
| case '5' : case '6' : case '7' : case '_' : |
| for (;; c = src.read()) { |
| if (c == '_') { |
| if (nondigit != '\0') { |
| break; |
| } |
| nondigit = c; |
| } else if (c >= '0' && c <= '7') { |
| nondigit = '\0'; |
| tokenBuffer.append(c); |
| } else { |
| break; |
| } |
| } |
| if (tokenBuffer.length() > startLen) { |
| src.unread(c); |
| |
| if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } |
| |
| yaccValue = getInteger(tokenBuffer.toString(), 8); |
| return Tokens.tINTEGER; |
| } |
| case '8' : |
| case '9' : |
| throw new SyntaxException(getPosition(), "Illegal octal digit."); |
| case '.' : |
| case 'e' : |
| case 'E' : |
| tokenBuffer.append('0'); |
| break; |
| default : |
| src.unread(c); |
| yaccValue = new FixnumNode(getPosition(), 0); |
| return Tokens.tINTEGER; |
| } |
| } |
| |
| boolean seen_point = false; |
| boolean seen_e = false; |
| |
| for (;; c = src.read()) { |
| switch (c) { |
| case '0' : |
| case '1' : |
| case '2' : |
| case '3' : |
| case '4' : |
| case '5' : |
| case '6' : |
| case '7' : |
| case '8' : |
| case '9' : |
| nondigit = '\0'; |
| tokenBuffer.append(c); |
| break; |
| case '.' : |
| if (nondigit != '\0') { |
| src.unread(c); |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } else if (seen_point || seen_e) { |
| src.unread(c); |
| return getNumberToken(tokenBuffer.toString(), true, nondigit); |
| } else { |
| char c2; |
| if (!Character.isDigit(c2 = src.read())) { |
| src.unread(c2); |
| src.unread('.'); |
| if (c == '_') { |
| // Enebo: c can never be antrhign but '.' |
| // Why did I put this here? |
| } else { |
| yaccValue = getInteger(tokenBuffer.toString(), 10); |
| return Tokens.tINTEGER; |
| } |
| } else { |
| tokenBuffer.append('.'); |
| tokenBuffer.append(c2); |
| seen_point = true; |
| nondigit = '\0'; |
| } |
| } |
| break; |
| case 'e' : |
| case 'E' : |
| if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } else if (seen_e) { |
| src.unread(c); |
| return getNumberToken(tokenBuffer.toString(), true, nondigit); |
| } else { |
| tokenBuffer.append(c); |
| seen_e = true; |
| nondigit = c; |
| c = src.read(); |
| if (c == '-' || c == '+') { |
| tokenBuffer.append(c); |
| nondigit = c; |
| } else { |
| src.unread(c); |
| } |
| } |
| break; |
| case '_' : // '_' in number just ignored |
| if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } |
| nondigit = c; |
| break; |
| default : |
| src.unread(c); |
| return getNumberToken(tokenBuffer.toString(), seen_e || seen_point, nondigit); |
| } |
| } |
| } |
| |
| private int getNumberToken(String number, boolean isFloat, char nondigit) { |
| if (nondigit != '\0') { |
| throw new SyntaxException(getPosition(), "Trailing '_' in number."); |
| } |
| if (isFloat) { |
| double d; |
| try { |
| d = Double.parseDouble(number); |
| } catch (NumberFormatException e) { |
| warnings.warn(getPosition(), "Float " + number + " out of range."); |
| |
| d = number.startsWith("-") ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; |
| } |
| yaccValue = new FloatNode(getPosition(), d); |
| return Tokens.tFLOAT; |
| } |
| yaccValue = getInteger(number, 10); |
| return Tokens.tINTEGER; |
| } |
| } |
