/*=============================================================================#
 # Copyright (c) 2007, 2019 Stephan Wahlbrink and others.
 # 
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
 # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
 # which is available at https://www.apache.org/licenses/LICENSE-2.0.
 # 
 # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
 # 
 # Contributors:
 #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
 #=============================================================================*/

package org.eclipse.statet.r.core.rsource.ast;

import static org.eclipse.statet.ltk.ast.core.AstNode.NA_OFFSET;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS123_SYNTAX_SEQREL_UNEXPECTED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS12_SYNTAX_TOKEN_UNEXPECTED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS12_SYNTAX_TOKEN_UNKNOWN;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_CC_NOT_CLOSED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_CONDITION_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_CONDITION_NOT_CLOSED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_ELEMENTNAME_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_EXPR_AS_BODY_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_EXPR_AS_CONDITION_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_EXPR_BEFORE_OP_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_FCALL_NOT_CLOSED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_FDEF_ARGS_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_FDEF_ARGS_NOT_CLOSED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_IF_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_IN_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_OPERATOR_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_SUBINDEXED_NOT_CLOSED;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS2_SYNTAX_SYMBOL_MISSING;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS3_FDEF;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS3_FOR;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS3_IF;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS3_WHILE;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUSFLAG_ERROR_IN_CHILD;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUSFLAG_REAL_ERROR;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUSFLAG_SUBSEQUENT;
import static org.eclipse.statet.r.core.rsource.IRSourceConstants.STATUS_RUNTIME_ERROR;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.eclipse.statet.jcommons.string.BasicStringFactory;
import org.eclipse.statet.jcommons.string.StringFactory;
import org.eclipse.statet.jcommons.text.core.input.TextParserInput;

import org.eclipse.statet.internal.r.core.RCorePlugin;
import org.eclipse.statet.ltk.ast.core.AstInfo;
import org.eclipse.statet.ltk.ast.core.AstNode;
import org.eclipse.statet.r.core.rlang.RTerminal;
import org.eclipse.statet.r.core.rsource.RLexer;
import org.eclipse.statet.r.core.rsource.ast.RAstNode.Assoc;


/**
 * Scanner to create a R AST.
 */
public final class RScanner {
	
	
	private static final byte LINE_MODE_CONSOLE= 1;
	private static final byte LINE_MODE_BLOCK= 2;
	private static final byte LINE_MODE_EAT= 3;
	
	public static final int LEXER_CONFIG= RLexer.SKIP_WHITESPACE;
	
	
	private static final class ExprContext {
		final RAstNode rootNode;
		final Expression rootExpr;
		RAstNode lastNode;
		Expression openExpr;
		final byte lineMode;
		
		public ExprContext(final RAstNode node, final Expression expr, final byte eatLines) {
			this.rootNode= this.lastNode= node;
			this.rootExpr= this.openExpr= expr;
			this.lineMode= eatLines;
		}
		
		final void update(final RAstNode lastNode, final Expression openExpr) {
			this.lastNode= lastNode;
			if (openExpr == null || openExpr.node != null) {
				this.openExpr= null;
			}
			else {
				this.openExpr= openExpr;
			}
		}
	}
	
	private static final class RoxygenCollector {
		
		private Comment[] lines= new Comment[64];
		private int lineCount;
		private DocuComment current;
		
		void init() {
			this.lineCount= 0;
			this.current= null;
		}
		
		void add(final Comment comment) {
			if (this.current == null) {
				this.current= new DocuComment();
			}
			comment.rParent= this.current;
			
			if (this.lineCount == this.lines.length) {
				this.lines= Arrays.copyOf(this.lines, this.lineCount + 64);
			}
			this.lines[this.lineCount++]= comment;
		}
		
		boolean hasComment() {
			return (this.current != null);
		}
		
		DocuComment finish(final RLexer lexer) {
			final DocuComment comment= new DocuComment();
			final Comment[] lines= Arrays.copyOf(this.lines, this.lineCount);
			comment.lines= lines;
			comment.startOffset= lines[0].startOffset;
			comment.endOffset= lines[this.lineCount - 1].endOffset;
			comment.nextOffset= (lexer != null && lexer.getType() != RTerminal.EOF) ? lexer.getOffset() : NA_OFFSET;
			
			this.lineCount= 0;
			this.current= null;
			return comment;
		}
		
	}
	
	private final static RScannerPostExprVisitor POST_VISITOR= new RScannerPostExprVisitor();
	
	
	private final RLexer lexer;
	private final int level;
	
	private RTerminal nextType;
	private boolean wasLinebreak;
	
	private List<RAstNode> comments;
	private RoxygenCollector roxygen;
	private int commentsLevel;
	
	private final boolean createText;
	private final StringFactory symbolTextFactory;
	
	
	public RScanner(final int level) {
		this(level, (StringFactory) null);
	}
	
	public RScanner(final int level, final StringFactory symbolTextFactory) {
		this(level, new RLexer((level == AstInfo.LEVEL_MINIMAL) ?
						(RLexer.DEFAULT | LEXER_CONFIG | RLexer.ENABLE_QUICK_CHECK) :
						(RLexer.DEFAULT | LEXER_CONFIG) ),
				symbolTextFactory );
	}
	
	public RScanner(final int level, final RLexer lexer, final StringFactory symbolTextFactory) {
		if (lexer == null) {
			throw new NullPointerException("lexer"); //$NON-NLS-1$
		}
		this.symbolTextFactory= (symbolTextFactory != null) ? symbolTextFactory : BasicStringFactory.INSTANCE;
		this.createText= ((level & AstInfo.DEFAULT_LEVEL_MASK) > AstInfo.LEVEL_MINIMAL);
		
		this.level= level;
		this.lexer= lexer;
	}
	
	
	public void setCommentLevel(final int level) {
		this.commentsLevel= level;
		if (level > 0) {
			this.comments= new ArrayList<>();
			if (level > 0x4) {
				this.roxygen= new RoxygenCollector();
			}
		}
	}
	
	public int getAstLevel() {
		return this.level;
	}
	
	public SourceComponent scanSourceUnit(final TextParserInput input) {
		try {
			this.lexer.reset(input);
			init();
			final SourceComponent rootNode= scanSourceUnit((RAstNode) null);
			return rootNode;
		}
		catch (final Exception e) {
			RCorePlugin.logError("Error occured while parsing R code", e);
			final SourceComponent dummy= new SourceComponent();
			dummy.status= STATUS_RUNTIME_ERROR;
			return dummy;
		}
	}
	
	public SourceComponent scanSourceRange(final TextParserInput input, final AstNode parent,
			final boolean expand) {
		final SourceComponent component= scanSourceRange(input, parent);
		if (expand) {
			component.startOffset= input.getStartIndex();
			component.endOffset= input.getStopIndex();
		}
		return component;
	}
	
	public SourceComponent scanSourceRange(final TextParserInput input, final AstNode parent) {
		try {
			this.lexer.reset(input);
			init();
			final SourceComponent rootNode= scanSourceUnit((RAstNode) null);
			rootNode.parent= parent;
			return rootNode;
		}
		catch (final Exception e) {
			RCorePlugin.logError("Error occured while parsing R code", e);
			final SourceComponent dummy= new SourceComponent();
			dummy.status= STATUS_RUNTIME_ERROR;
			if (this.commentsLevel > 0) {
				dummy.comments= Collections.emptyList();
			}
			return dummy;
		}
	}
	
	public RAstNode scanExpr(final TextParserInput input) {
		try {
			this.lexer.reset(input);
			init();
			final SourceComponent rootNode= scanSourceUnit((RAstNode) null);
			if (rootNode.getChildCount() == 1) {
				return rootNode.getChild(0);
			}
		}
		catch (final Exception e) {
			RCorePlugin.logError("Error occured while parsing R code", e);
		}
		return null;
	}
	
	public FDef scanFDef(final TextParserInput input) {
		try {
			this.lexer.reset(input);
			init();
			if (this.nextType == RTerminal.FUNCTION) {
				return scanFDef((ExprContext) null);
			}
		}
		catch (final Exception e) {
			RCorePlugin.logError("Error occured while parsing R code", e);
		}
		return null;
	}
	
	public FCall.Args scanFCallArgs(final TextParserInput input, final boolean expand) {
		try {
			this.lexer.reset(input);
			init();
			final FCall call= new FCall();
			call.endOffset= NA_OFFSET;
			scanInSpecArgs(call.args);
			if (expand) {
				call.args.startOffset= input.getStartIndex();
				call.args.endOffset= input.getIndex();
			}
			return call.args;
		}
		catch (final Exception e) {
			RCorePlugin.logError("Error occured while parsing R code", e);
		}
		return null;
	}
	
	private void init() {
		if (this.roxygen != null) {
			this.roxygen.init();
		}
		this.nextType= RTerminal.LINEBREAK;
		consumeToken();
	}
	
	final SourceComponent scanSourceUnit(final RAstNode parent) {
		final SourceComponent node= new SourceComponent();
		node.rParent= parent;
		scanInExprList(node, true);
//		if (this.nextType == RTerminal.EOF) {
//			this.next.type= null;
//		}
		if (this.commentsLevel > 0) {
			node.comments= Collections.unmodifiableList(this.comments);
		}
		node.updateStartOffset();
		node.updateEndOffset();
		return node;
	}
	
	final void scanInExprList(final ExpressionList node, final boolean script) {
		ITER_TOKEN: while (true) {
			switch (this.nextType) {
			
			case EOF:
				break ITER_TOKEN;
				
			case LINEBREAK:
				consumeToken();
				continue ITER_TOKEN;
			
			default:
				{
					Expression expr= node.appendNewExpr();
					final ExprContext context= new ExprContext(node, expr,
							script ? LINE_MODE_CONSOLE : LINE_MODE_BLOCK );
					scanInExpression(context);
					
					if (expr.node == null) {
						node.expressions.remove(context.rootExpr);
						expr= null;
					}
					else {
						checkExpression(context);
					}
					switch (this.nextType) {
					
					case SEMI:
						if (expr != null) {
							node.setSeparator(this.lexer.getOffset());
							consumeToken();
							continue ITER_TOKEN;
						}
						// else error like comma
						//$FALL-THROUGH$
					case COMMA:
						{
							expr= node.appendNewExpr();
							expr.node= errorFromNext(node);
						}
						continue ITER_TOKEN;
						
					case SUB_INDEXED_CLOSE:
					case BLOCK_CLOSE:
					case GROUP_CLOSE:
						if (script) {
							expr= node.appendNewExpr();
							expr.node= errorFromNext(node);
							continue ITER_TOKEN;
						}
						break ITER_TOKEN;
					}
				}
			}
		}
	}
	
	final int scanInGroup(final RAstNode node, final Expression expr) {
		final ExprContext context= new ExprContext(node, expr, LINE_MODE_EAT);
		scanInExpression(context);
		return checkExpression(context);
	}
	
	final void scanInExpression(final ExprContext context) {
		this.wasLinebreak= false;
		ITER_TOKEN : while(true) {
			
			if (this.wasLinebreak && context.lineMode < LINE_MODE_EAT && context.openExpr == null) {
				break ITER_TOKEN;
			}
			
			switch (this.nextType) {
			
			case LINEBREAK:
				if (context.lineMode < LINE_MODE_EAT && context.openExpr == null) {
					break ITER_TOKEN;
				}
				consumeToken();
				continue ITER_TOKEN;
				
			case SYMBOL:
			case SYMBOL_G:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createSymbol(null));
				continue ITER_TOKEN;
			
			case TRUE:
			case FALSE:
			case NUM_NUM:
			case NUM_INT:
			case NUM_CPLX:
			case NA:
			case NA_REAL:
			case NA_INT:
			case NA_CPLX:
			case NA_CHAR:
			case NAN:
			case INF:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createNumberConst(null));
				continue ITER_TOKEN;
			case NULL:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createNullConst(null));
				continue ITER_TOKEN;
				
			case STRING_D:
			case STRING_S:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createStringConst(null));
				continue ITER_TOKEN;
				
			case ARROW_LEFT_S:
			case ARROW_LEFT_D:
			case ARROW_RIGHT_S:
			case ARROW_RIGHT_D:
			case EQUAL:
			case COLON_EQUAL:
				appendOp(context, createAssignment());
				continue ITER_TOKEN;
				
			case TILDE:
				if (context.openExpr != null) {
					appendNonOp(context, createModel());
				}
				else {
					appendOp(context, createModel());
				}
				continue ITER_TOKEN;
				
			case PLUS:
			case MINUS:
				if (context.openExpr != null) {
					appendNonOp(context, createSign());
				}
				else {
					appendOp(context, createArithmetic());
				}
				continue ITER_TOKEN;
			case MULT:
			case DIV:
				appendOp(context, createArithmetic());
				continue ITER_TOKEN;
			case POWER:
				appendOp(context, createPower());
				continue ITER_TOKEN;
				
			case SEQ:
				appendOp(context, createSeq());
				continue ITER_TOKEN;
				
			case SPECIAL:
				appendOp(context, createSpecial());
				continue ITER_TOKEN;
				
			case REL_LT:
			case REL_LE:
			case REL_EQ:
			case REL_GE:
			case REL_GT:
			case REL_NE:
				appendOp(context, createRelational());
				continue ITER_TOKEN;
				
			case OR:
			case OR_D:
			case AND:
			case AND_D:
				appendOp(context, createLogical());
				continue ITER_TOKEN;
				
			case NOT:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createSign());
				continue ITER_TOKEN;
			
			case IF:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanCIf(context));
				continue ITER_TOKEN;
			case ELSE:
				if (context.rootNode.getNodeType() == NodeType.C_IF) {
					break ITER_TOKEN;
				}
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				
				appendNonOp(context, scanCElse(context));
				continue ITER_TOKEN;
			
			case FOR:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanCForLoop(context));
				continue ITER_TOKEN;
			case REPEAT:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanCRepeatLoop(context));
				continue ITER_TOKEN;
			case WHILE:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanCWhileLoop(context));
				continue ITER_TOKEN;
			
			case BREAK:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createLoopCommand());
				continue ITER_TOKEN;
			case NEXT:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, createLoopCommand());
				continue ITER_TOKEN;
			
			case FUNCTION:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanFDef(context));
				continue ITER_TOKEN;
				
			case GROUP_OPEN:
				if (context.openExpr != null) {
					appendNonOp(context, scanGroup());
				}
				else {
					appendOp(context, scanFCall());
				}
				continue ITER_TOKEN;
				
			case BLOCK_OPEN:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanBlock());
				continue ITER_TOKEN;
			
			case EOF:
				break ITER_TOKEN;
				
			case NS_GET:
			case NS_GET_INT:
				if (this.wasLinebreak && context.openExpr == null) {
					break ITER_TOKEN;
				}
				appendNonOp(context, scanNSGet(context));
				continue ITER_TOKEN;
				
			case SUB_INDEXED_S_OPEN:
			case SUB_INDEXED_D_OPEN:
				appendOp(context, scanSubIndexed(context));
				continue ITER_TOKEN;
				
			case SUB_NAMED_PART:
			case SUB_NAMED_SLOT:
				appendOp(context, scanSubNamed(context));
				continue ITER_TOKEN;
				
			case QUESTIONMARK:
				if (context.openExpr != null) {
					appendNonOp(context, createHelp());
					continue ITER_TOKEN;
				}
				else {
					appendOp(context, createHelp());
					continue ITER_TOKEN;
				}
				
			case UNKNOWN:
			case IN:
				appendNonOp(context, errorFromNext(null));
				continue ITER_TOKEN;
				
			case COMMA:
			case SEMI:
			case SUB_INDEXED_CLOSE:
			case BLOCK_CLOSE:
			case GROUP_CLOSE:
				break ITER_TOKEN;
				
			default:
				throw new IllegalStateException("Unhandled token in expr-scanner: "+this.nextType.name());
			}
		}
	}
	
	final Group scanGroup() {
		final Group node= new Group();
		setupFromSourceToken(node);
		consumeToken();
		scanInGroup(node, node.expr);
		if (this.nextType == RTerminal.GROUP_CLOSE) {
			node.groupCloseOffset= this.lexer.getOffset();
			node.endOffset= node.groupCloseOffset + 1;
			consumeToken();
			return node;
		}
		else {
			node.endOffset= this.lexer.getOffset();
			node.status |= STATUS2_SYNTAX_CC_NOT_CLOSED;
			return node;
		}
	}
	
	final Block scanBlock() {
		final Block node= new Block();
		setupFromSourceToken(node);
		consumeToken();
		scanInExprList(node, false);
		if (this.nextType == RTerminal.BLOCK_CLOSE) {
			node.blockCloseOffset= this.lexer.getOffset();
			node.endOffset= node.blockCloseOffset + 1;
			consumeToken();
			return node;
		}
		else {
			node.endOffset= this.lexer.getOffset();
			node.status |= STATUS2_SYNTAX_CC_NOT_CLOSED;
			return node;
		}
	}
	
	final NSGet scanNSGet(final ExprContext context) {
		final NSGet node;
		switch (this.nextType) {
		case NS_GET:
			node= new NSGet.Std();
			break;
		case NS_GET_INT:
			node= new NSGet.Internal();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		node.operatorOffset= this.lexer.getOffset();
		consumeToken();
		
		// setup ns
		switch (context.lastNode.getNodeType()) {
		case SYMBOL:
		case STRING_CONST:
			{
				node.namespace= (SingleValue) context.lastNode;
				final RAstNode base= context.lastNode.rParent;
				node.namespace.rParent= node;
				final Expression expr= base.getExpr(node.namespace);
				if (expr != null) {
					expr.node= null;
				}
				else {
					throw new IllegalStateException(); // ?
				}
				context.update(base, expr);
				node.startOffset= node.namespace.startOffset;
				break;
			}
		default:
			node.namespace= errorNonExistingSymbol(node, node.startOffset, STATUS2_SYNTAX_ELEMENTNAME_MISSING);
			break;
		}
		
		// element
		switch (this.nextType) {
		case STRING_S:
		case STRING_D:
			node.element= createStringConst(node);
			node.endOffset= node.element.endOffset;
			return node;
		case SYMBOL:
		case SYMBOL_G:
			node.element= createSymbol(node);
			node.endOffset= node.element.endOffset;
			return node;
		default:
			node.element= errorNonExistingSymbol(node, node.endOffset, STATUS2_SYNTAX_ELEMENTNAME_MISSING);
			return node;
		}
	}
	
	final SubNamed scanSubNamed(final ExprContext context) {
		final SubNamed node;
		switch (this.nextType) {
		case SUB_NAMED_PART:
			node= new SubNamed.Named();
			break;
		case SUB_NAMED_SLOT:
			node= new SubNamed.Slot();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		node.operatorOffset= this.lexer.getOffset();
		consumeToken();
		readLines();
		
		switch (this.nextType) {
		case STRING_S:
		case STRING_D:
			node.subname= createStringConst(node);
			node.endOffset= node.subname.endOffset;
			return node;
		case SYMBOL:
		case SYMBOL_G:
			node.subname= createSymbol(node);
			node.endOffset= node.subname.endOffset;
			return node;
		default:
			node.subname= errorNonExistingSymbol(node, node.endOffset, STATUS2_SYNTAX_ELEMENTNAME_MISSING);
			return node;
		}
	}
	
	final SubIndexed scanSubIndexed(final ExprContext context) {
		final SubIndexed node;
		switch (this.nextType) {
		case SUB_INDEXED_S_OPEN:
			node= new SubIndexed.S();
			break;
		case SUB_INDEXED_D_OPEN:
			node= new SubIndexed.D();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		node.openOffset= this.lexer.getOffset();
		consumeToken();
		readLines();
		
		scanInSpecArgs(node.sublist);
		
		if (this.nextType == RTerminal.SUB_INDEXED_CLOSE) {
			node.closeOffset= this.lexer.getOffset();
			consumeToken();
			
			if (node.getNodeType() == NodeType.SUB_INDEXED_D) {
				if (this.nextType == RTerminal.SUB_INDEXED_CLOSE) {
					node.close2Offset= this.lexer.getOffset();
					node.endOffset= node.close2Offset + 1;
					consumeToken();
					return node;
				}
				else {
					node.endOffset= node.closeOffset + 1;
					node.status |= STATUS2_SYNTAX_SUBINDEXED_NOT_CLOSED;
					return node;
				}
			}
			else {
				node.endOffset= node.closeOffset + 1;
				return node;
			}
		}
		else {
			node.endOffset= node.sublist.endOffset;
			node.status |= STATUS2_SYNTAX_SUBINDEXED_NOT_CLOSED;
			return node;
		}
	}
	
	final CIfElse scanCIf(final ExprContext context) {
		final CIfElse node= new CIfElse();
		setupFromSourceToken(node);
		consumeToken();
		int ok= 0;
		readLines();
		
		if (this.nextType == RTerminal.GROUP_OPEN) {
			node.condOpenOffset= this.lexer.getOffset();
			node.endOffset= this.lexer.getOffset() + 1;
			consumeToken();
			readLines();
			
			// condition
			ok+= scanInGroup(node, node.condExpr);
			
			if (this.nextType == RTerminal.GROUP_CLOSE) {
				node.condCloseOffset= this.lexer.getOffset();
				node.endOffset= node.condCloseOffset + 1;
				consumeToken();
				ok= 1;
				readLines();
			}
			else {
				node.endOffset= node.condExpr.node.endOffset;
				node.status |= STATUS2_SYNTAX_CONDITION_NOT_CLOSED;
			}
		}
		else {
			node.status= STATUS2_SYNTAX_CONDITION_MISSING;
			node.condExpr.node= errorNonExistExpression(node, node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_CONDITION_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_IF));
		}
		
		// then
		if (ok > 0 || recoverCCont()) {
			final ExprContext thenContext= new ExprContext(node, node.thenExpr, context.lineMode);
			scanInExpression(thenContext);
			checkExpression(thenContext);
			node.endOffset= node.thenExpr.node.endOffset;
			if (context.lineMode >= LINE_MODE_BLOCK) {
				readLines();
			}
		}
		else {
			node.thenExpr.node= errorNonExistExpression(node, node.condExpr.node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_BODY_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_IF));
		}
		
		// else
		if (this.nextType == RTerminal.ELSE) {
			node.withElse= true;
			node.elseOffset= this.lexer.getOffset();
			consumeToken();
			// else body is added via common expression processing
		}
		
		return node;
	}
	
	final CIfElse scanCElse(final ExprContext context) { // else without if
		final CIfElse node= new CIfElse();
		setupFromSourceToken(node);
		node.status= STATUS2_SYNTAX_IF_MISSING;
		node.condExpr.node= errorNonExistExpression(node, node.startOffset,
				(STATUS2_SYNTAX_EXPR_AS_CONDITION_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_IF));
		node.thenExpr.node= errorNonExistExpression(node, node.startOffset,
				(STATUS2_SYNTAX_EXPR_AS_BODY_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_IF));
		node.elseOffset= this.lexer.getOffset();
		node.withElse= true;
		consumeToken();
		
		return node;
	}
	
	final CForLoop scanCForLoop(final ExprContext context) {
		final CForLoop node= new CForLoop();
		setupFromSourceToken(node);
		consumeToken();
		int ok= 0;
		readLines();
		
		if (this.nextType == RTerminal.GROUP_OPEN) {
			node.condOpenOffset= this.lexer.getOffset();
			consumeToken();
			readLines();
			
			// condition
			switch (this.nextType) {
			case SYMBOL:
			case SYMBOL_G:
				node.varSymbol= createSymbol(node);
				readLines();
				break;
			default:
				node.varSymbol= errorNonExistingSymbol(node, node.condOpenOffset + 1, STATUS2_SYNTAX_SYMBOL_MISSING);
				ok--;
				break;
			}
			
			if (this.nextType == RTerminal.IN) {
				node.inOffset= this.lexer.getOffset();
				node.endOffset= node.inOffset + 2;
				consumeToken();
				readLines();
				
				ok+= scanInGroup(node, node.condExpr);
			}
			else {
				node.endOffset= node.varSymbol.endOffset;
				node.status |= (ok >= 0) ? STATUS2_SYNTAX_IN_MISSING :
						(STATUS2_SYNTAX_IN_MISSING | STATUSFLAG_SUBSEQUENT);
				node.condExpr.node= errorNonExistExpression(node, node.endOffset,
						(STATUS2_SYNTAX_EXPR_AS_CONDITION_MISSING | STATUSFLAG_SUBSEQUENT));
			}
			
			if (this.nextType == RTerminal.GROUP_CLOSE) {
				node.condCloseOffset= this.lexer.getOffset();
				node.endOffset= node.condCloseOffset + 1;
				consumeToken();
				ok= 1;
				readLines();
			}
			else {
				node.endOffset= node.condExpr.node.endOffset;
				if ((node.status & STATUSFLAG_REAL_ERROR) == 0) {
					node.status |= (ok >= 0) ? STATUS2_SYNTAX_CONDITION_NOT_CLOSED :
							(STATUS2_SYNTAX_CONDITION_NOT_CLOSED | STATUSFLAG_SUBSEQUENT);
				}
			}
		}
		else { // missing GROUP_OPEN
			node.status= STATUS2_SYNTAX_CONDITION_MISSING;
			node.varSymbol= errorNonExistingSymbol(node, node.endOffset,
					STATUS2_SYNTAX_SYMBOL_MISSING | STATUSFLAG_SUBSEQUENT);
			node.condExpr.node= errorNonExistExpression(node, node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_CONDITION_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_FOR));
		}
		
		// loop
		if (ok <= 0 && !recoverCCont()) {
			node.loopExpr.node= errorNonExistExpression(node, node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_BODY_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_FOR));
		}
		
		return node;
	}
	
	final CWhileLoop scanCWhileLoop(final ExprContext context) {
		final CWhileLoop node= new CWhileLoop();
		setupFromSourceToken(node);
		consumeToken();
		int ok= 0;
		readLines();
		
		if (this.nextType == RTerminal.GROUP_OPEN) {
			node.condOpenOffset= this.lexer.getOffset();
			node.endOffset= node.condOpenOffset + 1;
			consumeToken();
			readLines();
			
			// condition
			ok+= scanInGroup(node, node.condExpr);
			
			if (this.nextType == RTerminal.GROUP_CLOSE) {
				node.condCloseOffset= this.lexer.getOffset();
				node.endOffset= node.condCloseOffset + 1;
				consumeToken();
				ok= 1;
				readLines();
			}
			else {
				node.endOffset= node.condExpr.node.endOffset;
				node.status= (ok >= 0) ? STATUS2_SYNTAX_CONDITION_NOT_CLOSED :
						(STATUS2_SYNTAX_CONDITION_NOT_CLOSED | STATUSFLAG_SUBSEQUENT);
			}
		}
		else {
			node.status= STATUS2_SYNTAX_CONDITION_MISSING;
			node.condExpr.node= errorNonExistExpression(node, node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_CONDITION_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_WHILE));
		}
		
		// loop
		if (ok <= 0 && !recoverCCont()) {
			node.loopExpr.node= errorNonExistExpression(node, node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_BODY_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_WHILE));
		}
		
		return node;
	}
	
	final CRepeatLoop scanCRepeatLoop(final ExprContext context) {
		final CRepeatLoop node= new CRepeatLoop();
		setupFromSourceToken(node);
		consumeToken();
		
		return node;
	}
	
	final FDef scanFDef(final ExprContext context) {
		final FDef node= new FDef();
		setupFromSourceToken(node);
		consumeToken();
		int ok= 0;
		readLines();
		
		if (this.nextType == RTerminal.GROUP_OPEN) {
			node.argsOpenOffset= this.lexer.getOffset();
			node.endOffset= node.argsOpenOffset + 1;
			consumeToken();
			readLines();
			
			// args
			scanInFDefArgs(node.args);
			
			if (this.nextType == RTerminal.GROUP_CLOSE) {
				node.argsCloseOffset= this.lexer.getOffset();
				node.endOffset= node.argsCloseOffset + 1;
				consumeToken();
				ok= 1;
				readLines();
			}
			else {
				node.endOffset= node.args.endOffset;
				node.status |= STATUS2_SYNTAX_FDEF_ARGS_NOT_CLOSED;
			}
		}
		else {
			node.args.startOffset= node.args.endOffset= node.endOffset;
			node.status= STATUS2_SYNTAX_FDEF_ARGS_MISSING;
		}
		
		// body
		if (ok <= 0 && !recoverCCont()) {
			node.expr.node= errorNonExistExpression(node, node.endOffset,
					(STATUS2_SYNTAX_EXPR_AS_BODY_MISSING | STATUSFLAG_SUBSEQUENT | STATUS3_FDEF));
		}
		
		return node;
	}
	
	final FCall scanFCall() {
		final FCall node= new FCall();
		
		setupFromSourceToken(node);
		node.argsOpenOffset= this.lexer.getOffset();
		consumeToken();
		readLines();
		
		scanInSpecArgs(node.args);
		
		if (this.nextType == RTerminal.GROUP_CLOSE) {
			node.argsCloseOffset= this.lexer.getOffset();
			node.endOffset= node.argsCloseOffset + 1;
			consumeToken();
		}
		else {
			node.endOffset= node.args.endOffset;
			node.status |= STATUS2_SYNTAX_FCALL_NOT_CLOSED;
		}
		
		return node;
	}
	
	final void scanInFDefArgs(final FDef.Args args) {
		args.startOffset= args.endOffset= args.rParent.endOffset;
		ITER_ARGS : while (true) {
			final FDef.Arg arg= new FDef.Arg(args);
			switch(this.nextType) {
			case SYMBOL:
			case SYMBOL_G:
				arg.argName= createSymbol(arg);
				arg.startOffset= arg.argName.startOffset;
				arg.endOffset= arg.argName.endOffset;
				readLines();
				break;
			case EQUAL:
			case COMMA:
				arg.startOffset= arg.endOffset= this.lexer.getOffset();
				break;
			default:
				if (args.specs.isEmpty()) {
					return;
				}
				arg.startOffset= arg.endOffset= args.endOffset;
				break;
			}
			
			if (arg.argName == null) {
				arg.argName= errorNonExistingSymbol(arg, arg.endOffset, STATUS2_SYNTAX_SYMBOL_MISSING);
			}
			
			if (this.nextType == RTerminal.EQUAL) {
				arg.endOffset= this.lexer.getOffset() + 1;
				consumeToken();
				
				final Expression expr= arg.addDefault();
				scanInGroup(arg, expr);
				arg.endOffset= arg.defaultExpr.node.endOffset;
			}
			
			args.specs.add(arg);
			args.status= POST_VISITOR.checkTerminal(arg);
			if (this.nextType == RTerminal.COMMA) {
				args.endOffset= this.lexer.getOffset() + 1;
				consumeToken();
				readLines();
				continue ITER_ARGS;
			}
			else {
				args.startOffset= args.specs.get(0).startOffset;
				args.endOffset= arg.endOffset;
				return;
			}
		}
	}
	
	final void scanInSpecArgs(final FCall.Args args) {
		args.startOffset= args.endOffset= args.rParent.endOffset;
		ITER_ARGS : while (true) {
			final FCall.Arg arg= new FCall.Arg(args);
			arg.startOffset= this.lexer.getOffset();
			switch(this.nextType) {
			case SYMBOL:
				arg.argName= createSymbol(arg);
				readLines();
				break;
			case STRING_S:
			case STRING_D:
				arg.argName= createStringConst(arg);
				readLines();
				break;
			case NULL:
				arg.argName= createNullConst(arg);
				readLines();
				break;
			case EQUAL:
				arg.argName= errorNonExistingSymbol(arg, this.lexer.getOffset(), STATUS2_SYNTAX_ELEMENTNAME_MISSING);
				break;
			default:
				break;
			}
			if (arg.argName != null) {
				if (this.nextType == RTerminal.EQUAL) {
					arg.equalsOffset= this.lexer.getOffset();
					arg.endOffset= arg.equalsOffset + 1;
					consumeToken();
					
					final ExprContext valueContext= new ExprContext(arg, arg.valueExpr, LINE_MODE_EAT);
					scanInExpression(valueContext);
					if (arg.valueExpr.node != null) { // empty items are allowed
						checkExpression(valueContext);
						arg.endOffset= arg.valueExpr.node.endOffset;
					}
				}
				else {
					// argName -> valueExpr
					arg.valueExpr.node= arg.argName;
					arg.argName= null;
					
					final ExprContext valueContext= new ExprContext(arg, arg.valueExpr, LINE_MODE_EAT);
					valueContext.update(arg.valueExpr.node, null);
					scanInExpression(valueContext);
					checkExpression(valueContext);
					arg.endOffset= arg.valueExpr.node.endOffset;
				}
			}
			else {
				final ExprContext valueContext= new ExprContext(arg, arg.valueExpr, LINE_MODE_EAT);
				scanInExpression(valueContext);
				if (arg.valueExpr.node != null) { // empty items are allowed
					checkExpression(valueContext);
					arg.endOffset= arg.valueExpr.node.endOffset;
				}
				else {
					arg.startOffset= arg.endOffset= args.endOffset;
				}
			}
			
			if (this.nextType == RTerminal.COMMA) {
				args.specs.add(arg);
				args.status= POST_VISITOR.checkTerminal(arg);
				args.sepList.add(this.lexer.getOffset());
				args.endOffset= this.lexer.getOffset() + 1;
				consumeToken();
				readLines();
				continue ITER_ARGS;
			}
			// last arg before )
			if (args.specs.isEmpty() && !arg.hasChildren()) {
				return;
			}
			args.specs.add(arg);
			args.status= POST_VISITOR.checkTerminal(arg);
			args.startOffset= args.specs.get(0).startOffset;
			args.endOffset= arg.endOffset;
			return;
		}
	}
	
	final void scanInSpecArgs(final SubIndexed.Args args) {
		args.startOffset= args.endOffset= args.rParent.endOffset;
		ITER_ARGS : while (true) {
			final SubIndexed.Arg arg= new SubIndexed.Arg(args);
			arg.startOffset= this.lexer.getOffset();
			switch(this.nextType) {
			case SYMBOL:
				arg.argName= createSymbol(arg);
				readLines();
				break;
			case STRING_S:
			case STRING_D:
				arg.argName= createStringConst(arg);
				readLines();
				break;
			case NULL:
				arg.argName= createNullConst(arg);
				readLines();
				break;
			case EQUAL:
				arg.argName= errorNonExistingSymbol(arg, this.lexer.getOffset(), STATUS2_SYNTAX_ELEMENTNAME_MISSING);
				break;
			default:
				break;
			}
			if (arg.argName != null) {
				if (this.nextType == RTerminal.EQUAL) {
					arg.equalsOffset= this.lexer.getOffset();
					arg.endOffset= arg.equalsOffset + 1;
					consumeToken();
					
					final ExprContext valueContext= new ExprContext(arg, arg.valueExpr, LINE_MODE_EAT);
					scanInExpression(valueContext);
					if (arg.valueExpr.node != null) { // empty items are allowed
						checkExpression(valueContext);
						arg.endOffset= arg.valueExpr.node.endOffset;
					}
				}
				else {
					// argName -> valueExpr
					arg.valueExpr.node= arg.argName;
					arg.argName= null;
					
					final ExprContext valueContext= new ExprContext(arg, arg.valueExpr, LINE_MODE_EAT);
					valueContext.update(arg.valueExpr.node, null);
					scanInExpression(valueContext);
					checkExpression(valueContext);
					arg.endOffset= arg.valueExpr.node.endOffset;
				}
			}
			else {
				final ExprContext valueContext= new ExprContext(arg, arg.valueExpr, LINE_MODE_EAT);
				scanInExpression(valueContext);
				if (arg.valueExpr.node != null) { // empty items are allowed
					checkExpression(valueContext);
					arg.endOffset= arg.valueExpr.node.endOffset;
				}
				else {
					arg.startOffset= arg.endOffset= args.endOffset;
				}
			}
			
			if (this.nextType == RTerminal.COMMA) {
				args.specs.add(arg);
				args.status= POST_VISITOR.checkTerminal(arg);
				args.endOffset= this.lexer.getOffset() + 1;
				consumeToken();
				readLines();
				continue ITER_ARGS;
			}
			// last arg before )
			if (args.specs.isEmpty() && !arg.hasChildren()) {
				return;
			}
			args.specs.add(arg);
			args.status= POST_VISITOR.checkTerminal(arg);
			args.startOffset= args.specs.get(0).startOffset;
			args.endOffset= arg.endOffset;
			return;
		}
	}
	
	final boolean recoverCCont() {
		return !this.wasLinebreak
			&& (this.nextType == RTerminal.SYMBOL || this.nextType == RTerminal.SYMBOL_G || this.nextType == RTerminal.BLOCK_OPEN);
	}
	
	final void appendNonOp(final ExprContext context, final RAstNode newNode) {
		if (context.openExpr != null) {
			newNode.rParent= context.lastNode;
			context.openExpr.node= newNode;
		}
		else {
			// setup missing op
			final Dummy.Operator error= new Dummy.Operator(STATUS2_SYNTAX_OPERATOR_MISSING);
			error.rParent= context.rootNode;
			error.leftExpr.node= context.rootExpr.node;
			error.startOffset= error.endOffset= newNode.startOffset;
			context.rootExpr.node= error;
			// append news
			newNode.rParent= error;
			error.rightExpr.node= newNode;
			context.rootExpr.node= error;
		}
		context.update(newNode, newNode.getRightExpr());
		return;
	}
	
	final void appendOp(final ExprContext context, final RAstNode newNode) {
		if (context.openExpr != null) {
			context.openExpr.node= errorNonExistExpression(context.lastNode, newNode.startOffset, STATUS2_SYNTAX_EXPR_BEFORE_OP_MISSING);
			context.update(context.openExpr.node, null);
		}
		
		final int newP= newNode.getNodeType().opPrec;
		RAstNode left= context.lastNode;
		RAstNode cand= context.lastNode;
		
		ITER_CAND : while (cand != null && cand != context.rootNode) {
			final NodeType candType= cand.getNodeType();
			if (candType.opPrec == newP) {
				switch (candType.opAssoc) {
				case Assoc.NOSTD:
					left= cand;
					if ((newNode.status & STATUSFLAG_REAL_ERROR) == 0) {
						newNode.status= STATUS123_SYNTAX_SEQREL_UNEXPECTED;
					}
					break ITER_CAND;
				case Assoc.LEFTSTD:
					left= cand;
					break ITER_CAND;
				case Assoc.RIGHTSTD:
				default:
					break ITER_CAND;
				}
			}
			if (candType.opPrec > newP) {
				break ITER_CAND;
			}
			left= cand;
			cand= cand.rParent;
		}
		
		final RAstNode baseNode= left.rParent;
		if (baseNode == null) {
			throw new IllegalStateException(); // DEBUG
		}
		final Expression baseExpr= baseNode.getExpr(left);
		newNode.getLeftExpr().node= left;
		left.rParent= newNode;
		baseExpr.node= newNode;
		newNode.rParent= baseNode;
		
		context.update(newNode, newNode.getRightExpr());
		return;
	}
	
	Dummy.Terminal errorNonExistExpression(final RAstNode parent, final int stopHint, final int status) {
		final Dummy.Terminal error= new Dummy.Terminal(status);
		error.rParent= parent;
		error.startOffset= error.endOffset= (stopHint != NA_OFFSET) ? stopHint : parent.endOffset;
		error.text= ""; //$NON-NLS-1$
		parent.status |= STATUSFLAG_ERROR_IN_CHILD;
		return error;
	}
	
	Dummy.Terminal errorFromNext(final RAstNode parent) {
		final Dummy.Terminal error= new Dummy.Terminal((this.nextType == RTerminal.UNKNOWN) ?
				STATUS12_SYNTAX_TOKEN_UNKNOWN : STATUS12_SYNTAX_TOKEN_UNEXPECTED);
		error.rParent= parent;
		error.startOffset= this.lexer.getOffset();
		error.endOffset= this.lexer.getOffset() + this.lexer.getLength();
		if (this.createText) {
			error.text= this.lexer.getText();
		}
		consumeToken();
		if (parent != null) {
			parent.status |= STATUSFLAG_ERROR_IN_CHILD;
		}
		return error;
	}
	
	Symbol errorNonExistingSymbol(final RAstNode parent, final int offset, final int status) {
		final Symbol error= new Symbol.Std();
		error.rParent= parent;
		error.startOffset= error.endOffset= offset;
		error.text= ""; //$NON-NLS-1$
		error.status= status;
		parent.status |= STATUSFLAG_ERROR_IN_CHILD;
		return error;
	}
	
	protected Symbol createSymbol(final RAstNode parent) {
		final Symbol symbol;
		switch (this.nextType) {
		case SYMBOL_G:
			symbol= new Symbol.G();
			break;
		case SYMBOL:
			symbol= new Symbol.Std();
			break;
		default:
			throw new IllegalStateException();
		}
		symbol.rParent= parent;
		setupFromSourceToken(symbol);
		if (parent != null) {
			parent.status |= POST_VISITOR.checkTerminal(symbol);
		}
		consumeToken();
		return symbol;
	}
	
	protected NumberConst createNumberConst(final RAstNode parent) {
		final NumberConst num= new NumberConst(this.nextType);
		num.rParent= parent;
		setupFromSourceToken(num);
		consumeToken();
		return num;
	}
	
	protected NullConst createNullConst(final RAstNode parent) {
		final NullConst num= new NullConst();
		num.rParent= parent;
		setupFromSourceToken(num);
		consumeToken();
		return num;
	}
	
	protected StringConst createStringConst(final RAstNode parent) {
		final StringConst str;
		switch (this.nextType) {
		case STRING_D:
			str= new StringConst.D();
			break;
		case STRING_S:
			str= new StringConst.S();
			break;
		default:
			throw new IllegalStateException();
		}
		str.rParent= parent;
		setupFromSourceToken(str);
		consumeToken();
		return str;
	}
	
	protected Assignment createAssignment() {
		Assignment node;
		switch (this.nextType) {
		case ARROW_LEFT_S:
			node= new Assignment.LeftS();
			break;
		case ARROW_LEFT_D:
			node= new Assignment.LeftD();
			break;
		case ARROW_RIGHT_S:
			node= new Assignment.RightS();
			break;
		case ARROW_RIGHT_D:
			node= new Assignment.RightD();
			break;
		case EQUAL:
			node= new Assignment.LeftE();
			break;
		case COLON_EQUAL:
			node= new Assignment.LeftC();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Model createModel() {
		final Model node= new Model();
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected CLoopCommand createLoopCommand() {
		final CLoopCommand node;
		switch (this.nextType) {
		case NEXT:
			node= new CLoopCommand.Next();
			break;
		case BREAK:
			node= new CLoopCommand.Break();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Sign createSign() {
		final Sign node;
		switch (this.nextType) {
		case PLUS:
			node= new Sign.PlusSign();
			break;
		case MINUS:
			node= new Sign.MinusSign();
			break;
		case NOT:
			node= new Sign.Not();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Arithmetic createArithmetic() {
		final Arithmetic node;
		switch (this.nextType) {
		case PLUS:
			node= new Arithmetic.Plus();
			break;
		case MINUS:
			node= new Arithmetic.Minus();
			break;
		case MULT:
			node= new Arithmetic.Mult();
			break;
		case DIV:
			node= new Arithmetic.Div();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Power createPower() {
		final Power node= new Power();
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Seq createSeq() {
		final Seq node= new Seq();
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Special createSpecial() {
		final Special node= new Special();
		setupFromSourceToken(node);
		if (this.createText) {
			node.qualifier= this.lexer.getText(this.symbolTextFactory);
		}
		consumeToken();
		return node;
	}
	
	protected Relational createRelational() {
		final Relational node;
		switch (this.nextType) {
		case REL_LT:
			node= new Relational.LT();
			break;
		case REL_LE:
			node= new Relational.LE();
			break;
		case REL_EQ:
			node= new Relational.EQ();
			break;
		case REL_GE:
			node= new Relational.GE();
			break;
		case REL_GT:
			node= new Relational.GT();
			break;
		case REL_NE:
			node= new Relational.NE();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Logical createLogical() {
		final Logical node;
		switch (this.nextType) {
		case AND:
			node= new Logical.And();
			break;
		case AND_D:
			node= new Logical.AndD();
			break;
		case OR:
			node= new Logical.Or();
			break;
		case OR_D:
			node= new Logical.OrD();
			break;
		default:
			throw new IllegalStateException();
		}
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	protected Help createHelp() {
		final Help node= new Help();
		setupFromSourceToken(node);
		consumeToken();
		return node;
	}
	
	private final void setupFromSourceToken(final RAstNode node) {
		node.startOffset= this.lexer.getOffset();
		node.endOffset= this.lexer.getOffset() + this.lexer.getLength();
		node.status= this.lexer.getFlags();
	}
	
	private final void setupFromSourceToken(final Symbol node) {
		node.startOffset= this.lexer.getOffset();
		node.endOffset= this.lexer.getOffset() + this.lexer.getLength();
		if (this.createText) {
			node.text= this.lexer.getText(this.symbolTextFactory);
			if (this.lexer.getStatusDetail() != null) {
				node.addAttachment(this.lexer.getStatusDetail());
			}
		}
		node.status= this.lexer.getFlags();
	}
	
	private final void setupFromSourceToken(final SingleValue node) {
		node.startOffset= this.lexer.getOffset();
		node.endOffset= this.lexer.getOffset() + this.lexer.getLength();
		if (this.createText) {
			node.text= this.lexer.getText();
			if (this.lexer.getStatusDetail() != null) {
				node.addAttachment(this.lexer.getStatusDetail());
			}
		}
		node.status= this.lexer.getFlags();
	}
	
	private final int checkExpression(final ExprContext context) {
		int state= 0;
		if (context.openExpr != null && context.openExpr.node == null) {
			context.openExpr.node= errorNonExistExpression(context.lastNode, context.lastNode.endOffset,
					context.lastNode.getMissingExprStatus(context.openExpr));
			state= -1;
		}
		context.rootNode.status |= POST_VISITOR.check(context.rootExpr.node);
		return state;
	}
	
	private final void readLines() {
		while (this.nextType == RTerminal.LINEBREAK) {
			consumeToken();
		}
	}
	
	private final void consumeToken() {
		this.wasLinebreak= (this.nextType == RTerminal.LINEBREAK);
		this.nextType= this.lexer.next();
		switch (this.nextType) {
		case COMMENT:
		case ROXYGEN_COMMENT:
			if (this.commentsLevel > 0x4) {
				consumeCommentWithRoxygen();
			}
			else {
				consumeComment();
			}
			return;
		default:
			return;
		}
	}
	
	
	private void consumeCommentWithRoxygen() {
		while (true) {
			final Comment comment;
			switch (this.nextType) {
			case COMMENT:
				if (this.roxygen.hasComment()) {
					this.comments.add(this.roxygen.finish(this.lexer));
				}
				comment= new Comment.CommonLine();
				setupFromSourceToken(comment);
				this.comments.add(comment);
				
				this.nextType= this.lexer.next();
				continue;
				
			case ROXYGEN_COMMENT:
				comment= new Comment.RoxygenLine();
				setupFromSourceToken(comment);
				this.roxygen.add(comment);
				
				this.nextType= this.lexer.next();
				continue;
				
			case LINEBREAK:
				this.nextType= this.lexer.next();
				if (this.nextType == RTerminal.LINEBREAK && this.roxygen.hasComment()) {
					this.comments.add(this.roxygen.finish(null));
				}
				continue;
				
			default:
				if (this.roxygen.hasComment()) {
					this.comments.add(this.roxygen.finish(this.lexer));
				}
				
				this.wasLinebreak= true;
				return;
			}
		}
	}
	
	private void consumeComment() {
		while (true) {
			switch (this.nextType) {
			case COMMENT:
			case ROXYGEN_COMMENT:
				if (this.commentsLevel > 0) {
					final Comment comment= (this.nextType == RTerminal.ROXYGEN_COMMENT) ?
							new Comment.RoxygenLine() :
							new Comment.CommonLine();
					setupFromSourceToken(comment);
					this.comments.add(comment);
				} // no break
				
				this.nextType= this.lexer.next();
				continue;
				
			case LINEBREAK:
				this.nextType= this.lexer.next();
				continue;
				
			default:
				this.wasLinebreak= true;
				return;
			}
		}
	}
	
}