core/plugins/org.eclipse.dltk.ui/src/org/eclipse/dltk/ui/text/heredoc/HereDocEnabledPartitionScanner.java

package org.eclipse.dltk.ui.text.heredoc;

import java.util.ArrayList;
import java.util.List;

import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.rules.ICharacterScanner;
import org.eclipse.jface.text.rules.IPredicateRule;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.RuleBasedPartitionScanner;

/**
 * A slightly modified version of a <code>RuleBasedPartitionScanner</code> that
 * knows how to properly scan for heredoc partitions.
 *
 * <p>
 * There is no need to use this partition scanner if heredoc is not supported by
 * the underlying dynamic language.
 * </p>
 *
 * <p>
 * If you do use this partitioner, you <b>must</b> also use the
 * <code>HereDocEnabledPartitioner</code> partitioner as it knows how to
 * properly manage heredoc partitions.
 * </p>
 *
 * @see HereDocPartitionRule
 * @see HereDocEnabledPartitioner
 * @see HereDocEnabledPresentationReconciler
 */
public class HereDocEnabledPartitionScanner extends RuleBasedPartitionScanner {
	private List<TokenContainer> buffer = new ArrayList<>();

	private HereDocPartitionRule hereDocRule;

	/**
	 * Creates a new heredoc partition scanner
	 * 
	 * @param rules
	 *            list of predicate rules that should be used to create document
	 *            partitions
	 * @param hereDocRule
	 *            heredoc partitioning rule
	 */
	public HereDocEnabledPartitionScanner(List<IPredicateRule> rules,
			HereDocPartitionRule hereDocRule) {
		this.hereDocRule = hereDocRule;

		IPredicateRule[] result = new IPredicateRule[rules.size()];
		setPredicateRules(rules.toArray(result));
	}

	@Override
	public IToken nextToken() {
		if (HereDocUtils.isHereDocContent(fContentType)) {
			return handleHereDoc();
		}

		if (!buffer.isEmpty()) {
			return buffer.remove(0).getToken();
		}

		return getNextToken(false);
	}

	@Override
	public void setPartialRange(IDocument document, int offset, int length,
			String contentType, int partitionOffset) {
		/*
		 * clear out any tokens that may be sitting in the buffer before the
		 * 'nextToken()' is requested. this method is only called if the
		 * document changes, which means the 'manual' rule should be passed all
		 * the information it needs to finish it's evaluation piggy-backed off
		 * the passed content type.
		 */
		buffer.clear();
		super.setPartialRange(document, offset, length, contentType,
				partitionOffset);
	}

	private IToken evalPossibleHereDoc(boolean inScan) {
		// our token is going to start wherever the scanner is
		fTokenOffset = fOffset;

		IToken token = hereDocRule.evaluate(this);

		if (token.isUndefined() || inScan) {
			return token;
		}

		buffer.add(new TokenContainer(token));

		int c;
		while ((c = read()) != ICharacterScanner.EOF) {
			unread();

			IToken next = getNextToken(true);
			buffer.add(new TokenContainer(next));

			if (c == '\n') {
				break;
			}
		}

		if (c != ICharacterScanner.EOF) {
			consumeHereDoc();
		}

		return buffer.remove(0).getToken();
	}

	private void consumeHereDoc() {
		// need to work w/ a copy otherwise we get concurrent modification
		// exception
		for (TokenContainer container : new ArrayList<>(buffer)) {
			if (HereDocUtils.isHereDocContent(container.getContentType())) {
				fTokenOffset = fOffset;
				fColumn = UNDEFINED;

				IToken body = hereDocRule.evaluate(this,
						container.getContentType());
				buffer.add(new TokenContainer(body));
			}
		}
	}

	private IToken getNextToken(boolean inScan) {
		IToken token = evalPossibleHereDoc(inScan);
		if (token.isUndefined()) {
			token = super.nextToken();
		}

		return token;
	}

	private IToken handleHereDoc() {
		// reset to partition start so we get all the characters
		fTokenOffset = fPartitionOffset;

		IToken token = hereDocRule.evaluate(this, fContentType);

		// we found our rule, reset the contentType just like our parent would
		fContentType = null;
		return token;
	}

	private class TokenContainer {
		private int offset;
		private IToken token;
		private int tokenOffset;

		TokenContainer(IToken token) {
			this.offset = fOffset;
			this.tokenOffset = fTokenOffset;

			this.token = token;
		}

		String getContentType() {
			return (String) token.getData();
		}

		IToken getToken() {
			fOffset = offset;
			fTokenOffset = tokenOffset;

			return token;
		}
	}
}