[424552] Improve the performance of the markup validator
diff --git a/bundles/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLLineTokenizer.jflex b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLLineTokenizer.jflex
index 0785f72..4bf824d 100644
--- a/bundles/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLLineTokenizer.jflex
+++ b/bundles/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLLineTokenizer.jflex
@@ -9,7 +9,7 @@
* IBM Corporation - initial API and implementation
*******************************************************************************/
-package org.eclipse.wst.xml.core.internal.tasks;
+package org.eclipse.wst.xml.core.internal.parser;
import java.io.CharArrayReader;
import java.io.IOException;
@@ -134,6 +134,10 @@
public final int getOffset() {
return fOffset + yychar;
}
+/* user method */
+public final int getLine() {
+ return yyline;
+}
private final boolean isBlockMarker() {
return isBlockMarker(fCurrentTagName);
}
@@ -485,6 +489,7 @@
// do nothing, this is the downstream parser's job
%eof}
+%public
%class XMLLineTokenizer
%implements BlockTokenizer, DOMRegionContext
%function primGetNextToken
diff --git a/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/StreamingMarkupValidator.java b/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/StreamingMarkupValidator.java
new file mode 100644
index 0000000..bd8513b
--- /dev/null
+++ b/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/StreamingMarkupValidator.java
@@ -0,0 +1,741 @@
+/*******************************************************************************
+ * Copyright (c) 2013 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * IBM Corporation - initial API and implementation
+ *
+ *******************************************************************************/
+package org.eclipse.wst.xml.core.internal.validation;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Stack;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.resources.IResourceProxy;
+import org.eclipse.core.resources.IResourceProxyVisitor;
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.NullProgressMonitor;
+import org.eclipse.core.runtime.Path;
+import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.Preferences;
+import org.eclipse.core.runtime.content.IContentDescription;
+import org.eclipse.core.runtime.content.IContentType;
+import org.eclipse.osgi.util.NLS;
+import org.eclipse.wst.sse.core.StructuredModelManager;
+import org.eclipse.wst.sse.core.internal.document.DocumentReader;
+import org.eclipse.wst.sse.core.internal.provisional.IStructuredModel;
+import org.eclipse.wst.sse.core.internal.provisional.text.ITextRegion;
+import org.eclipse.wst.validation.AbstractValidator;
+import org.eclipse.wst.validation.ValidationResult;
+import org.eclipse.wst.validation.ValidationState;
+import org.eclipse.wst.validation.internal.core.Message;
+import org.eclipse.wst.validation.internal.core.ValidationException;
+import org.eclipse.wst.validation.internal.operations.IWorkbenchContext;
+import org.eclipse.wst.validation.internal.operations.LocalizedMessage;
+import org.eclipse.wst.validation.internal.provisional.core.IMessage;
+import org.eclipse.wst.validation.internal.provisional.core.IReporter;
+import org.eclipse.wst.validation.internal.provisional.core.IValidationContext;
+import org.eclipse.wst.validation.internal.provisional.core.IValidator;
+import org.eclipse.wst.xml.core.internal.Logger;
+import org.eclipse.wst.xml.core.internal.XMLCoreMessages;
+import org.eclipse.wst.xml.core.internal.XMLCorePlugin;
+import org.eclipse.wst.xml.core.internal.parser.XMLLineTokenizer;
+import org.eclipse.wst.xml.core.internal.preferences.XMLCorePreferenceNames;
+import org.eclipse.wst.xml.core.internal.provisional.document.IDOMNode;
+import org.eclipse.wst.xml.core.internal.regions.DOMRegionContext;
+import org.w3c.dom.Node;
+
+/**
+ * A markup validator that avoids loading the model into memory. Problems are detected while
+ * tokenizing the file.
+ *
+ */
+public class StreamingMarkupValidator extends AbstractValidator implements IValidator {
+
+ private static final String ANNOTATIONMSG = AnnotationMsg.class.getName();
+
+ /** The error threshold - sometimes, after you get so many errors, it's not worth seeing the others */
+ private static final int ERROR_THRESHOLD = 25;
+
+ /** The existing model, if available. */
+ private IStructuredModel model;
+
+ /** The validation reporter */
+ private IReporter fReporter;
+
+ /**
+ * A token from the tokenizer
+ */
+ private static class Token {
+ String type;
+ int offset;
+ int length;
+ int line;
+ String text;
+
+ public Token(String type, String text, int offset, int length, int line) {
+ this.type = type;
+ this.text = text;
+ this.offset = offset;
+ this.length = length;
+ this.line = line;
+ }
+ }
+
+ /** A stack used for finding missing start- and end-tag pairs */
+ private Stack tagStack;
+
+ /** Only count so many tag errors */
+ private int tagErrorCount = 0;
+
+ /** The xml content type */
+ private IContentType xmlContentType;
+
+ public void getAnnotationMsg(IReporter reporter, int problemId, LocalizedMessage message, Object attributeValueText, int len){
+ AnnotationMsg annotation = new AnnotationMsg(problemId, attributeValueText,len);
+ message.setAttribute(ANNOTATIONMSG, annotation);
+ reporter.addMessage(this, message);
+ }
+
+ /* (non-Javadoc)
+ * @see org.eclipse.wst.validation.internal.provisional.core.IValidator#cleanup(org.eclipse.wst.validation.internal.provisional.core.IReporter)
+ */
+ public void cleanup(IReporter reporter) {
+ if (tagStack != null) {
+ tagStack.clear();
+ tagStack = null;
+ }
+ xmlContentType = null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.eclipse.wst.validation.internal.provisional.core.IValidator#validate(org.eclipse.wst.validation.internal.provisional.core.IValidationContext, org.eclipse.wst.validation.internal.provisional.core.IReporter)
+ */
+ public void validate(IValidationContext helper, IReporter reporter) throws ValidationException {
+ final String[] uris = helper.getURIs();
+ IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
+ if (uris.length > 0) {
+ IFile currentFile = null;
+
+ for (int i = 0; i < uris.length && !reporter.isCancelled(); i++) {
+ // might be called with just project path?
+ IPath path = new Path(uris[i]);
+ if (path.segmentCount() > 1) {
+ currentFile = root.getFile(path);
+ if (shouldValidate(currentFile, true)) {
+ validateFile(currentFile, reporter);
+ }
+ }
+ else if (uris.length == 1) {
+ validateProject(helper, reporter);
+ }
+ }
+ }
+ else
+ validateProject(helper, reporter);
+ }
+
+ /* (non-Javadoc)
+ * @see org.eclipse.wst.validation.AbstractValidator#validate(org.eclipse.core.resources.IResource, int, org.eclipse.wst.validation.ValidationState, org.eclipse.core.runtime.IProgressMonitor)
+ */
+ public ValidationResult validate(IResource resource, int kind, ValidationState state, IProgressMonitor monitor) {
+ if (resource.getType() != IResource.FILE)
+ return null;
+ ValidationResult result = new ValidationResult();
+ fReporter = result.getReporter(monitor);
+ validateFile((IFile) resource, fReporter);
+ return result;
+ }
+
+ /**
+ * Convenience method for validating a resource and getting back the reporter
+ * @param resource The resource to be validated.
+ * @param kind The way the resource changed. It uses the same values as the kind parameter in IResourceDelta.
+ * @param state A way to pass arbitrary, validator specific, data from one invocation of a validator to the next, during the validation phase. At the end of the validation phase, this object will be cleared, thereby allowing any of this state information to be garbaged collected.
+ * @return the validator's reporter
+ */
+ public IReporter validate(IResource resource, int kind, ValidationState state) {
+ validate(resource, kind, state, new NullProgressMonitor());
+ return fReporter;
+ }
+
+ /**
+ * Gets the line number for a token
+ * @param token the token to find the line of
+ * @return the line in the document where the token can be found
+ */
+ private int getLine(Token token) {
+ return token.line + 1;
+ }
+
+ /**
+ * Checks that a start tag is immediately followed by a tag name
+ * @param token the token to check
+ * @param previousRegion the previous region
+ * @param reporter the reporter
+ */
+ private void checkForSpaceBeforeName(Token token, List previousRegion, IReporter reporter) {
+ if (previousRegion != null && previousRegion.size() == 1) {
+ // Check that the start tag's name comes right after the <
+ Token first = (Token) previousRegion.get(0);
+ if (DOMRegionContext.XML_TAG_OPEN.equals(first.type) && token.text.trim().length() == 0) {
+ final String messageText = XMLCoreMessages.ReconcileStepForMarkup_2;
+ final int length = token.length;
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.WHITESPACE_BEFORE_TAGNAME) , messageText);
+ message.setOffset(token.offset);
+ message.setLength(length);
+ message.setLineNo(getLine(token));
+ getAnnotationMsg(reporter, ProblemIDsXML.SpacesBeforeTagName, message, null, length);
+ }
+ }
+ }
+
+ /**
+ * Check that when a start- or end-tag has been opened it is properly closed
+ * @param previousRegion the previous region
+ * @param reporter the reporter
+ */
+ private void checkForTagClose(List previousRegion, IReporter reporter) {
+ if (previousRegion != null && previousRegion.size() > 0) {
+ final Token first = (Token) previousRegion.get(0);
+ // If the previous region was a start- or end-tag, look for the tag close
+ if (first.type == DOMRegionContext.XML_TAG_OPEN || first.type == DOMRegionContext.XML_END_TAG_OPEN) {
+ final int length = previousRegion.size();
+ boolean isClosed = false;
+ int textLength = first.length;
+ for (int i = 1; i < length; i++) {
+ Token t = (Token) previousRegion.get(i);
+ // Valid tag closings, EMPTY_TAG_CLOSE only works for a start tag, though
+ if ((t.type == DOMRegionContext.XML_EMPTY_TAG_CLOSE && first.type == DOMRegionContext.XML_TAG_OPEN) || t.type == DOMRegionContext.XML_TAG_CLOSE) {
+ isClosed = true;
+ break;
+ }
+ else if (t.type == DOMRegionContext.XML_TAG_NAME) {
+ textLength += t.length;
+ }
+ }
+ if (!isClosed) {
+ String messageText = XMLCoreMessages.ReconcileStepForMarkup_6;
+
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.MISSING_CLOSING_BRACKET) , messageText);
+ message.setOffset(first.offset);
+ message.setLength(textLength);
+ message.setLineNo(getLine(first));
+ getAnnotationMsg(reporter, ProblemIDsXML.MissingClosingBracket, message, null, textLength);
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks that there is no content before the XML declaration
+ * @param previousRegion the region prior to the processing instruction
+ * @param reporter the reporter
+ */
+ private void checkContentBeforeProcessingInstruction(List previousRegion, IReporter reporter) {
+ if (previousRegion != null && previousRegion.size() > 0) {
+ Token first = (Token) previousRegion.get(0);
+ if (first.type == DOMRegionContext.XML_CONTENT && first.offset == 0) {
+ // XML declaration only allowed at the start of the document
+ String messageText = XMLCoreMessages.ReconcileStepForMarkup_5;
+
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.WHITESPACE_AT_START) , messageText);
+ message.setOffset(first.offset);
+ message.setLength(first.length);
+ message.setLineNo(first.line + 1);
+ getAnnotationMsg(reporter, ProblemIDsXML.SpacesBeforePI, message, null, first.length);
+ }
+ }
+ }
+
+ /**
+ * Checks that the processing instruction name doesn't contain a namespace
+ * @param region the processing instruction region
+ * @param reporter the reporter
+ */
+ private void checkNamespacesInProcessingInstruction(List region, IReporter reporter) {
+ final int regionLength = region.size();
+ for (int i = 0; i < regionLength; i++) {
+ Token t = (Token) region.get(i);
+ if (t.type == DOMRegionContext.XML_TAG_NAME) {
+ int index = t.text.indexOf(":"); //$NON-NLS-1$
+ if (index != -1) {
+ String messageText = XMLCoreMessages.ReconcileStepForMarkup_4;
+ int start = t.offset + index;
+ int length = t.text.trim().length() - index;
+
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.NAMESPACE_IN_PI_TARGET) , messageText);
+ message.setOffset(start);
+ message.setLength(length);
+ message.setLineNo(t.line + 1);
+ getAnnotationMsg(reporter, ProblemIDsXML.NamespaceInPI, message, null, length);
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Check that a tag has a name (<> is invalid)
+ * @param token The xml tag close token
+ * @param region the tag region
+ * @param reporter the reporter
+ */
+ private void checkEmptyTag(List region, IReporter reporter) {
+ if (region.size() == 2) {
+ // Check that the tag is not empty
+ Token first = (Token)region.get(0);
+ if (first.type == DOMRegionContext.XML_TAG_OPEN) {
+ String messageText = XMLCoreMessages.ReconcileStepForMarkup_3;
+ final int length = first.length + ((Token)region.get(1)).length;
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.MISSING_TAG_NAME) , messageText);
+ message.setOffset(first.offset);
+ message.setLength(length);
+ message.setLineNo(first.line + 1);
+
+ getAnnotationMsg(reporter, ProblemIDsXML.EmptyTag, message, null, length);
+ }
+ }
+ }
+
+ /**
+ * Checks the end-tag region for attributes. There should be no attributes in the end tag
+ * @param first the first token in the region
+ * @param region the end-tag region
+ * @param reporter the reporter
+ */
+ private void checkAttributsInEndTag(Token first, List region, IReporter reporter) {
+ int errors = 0;
+ int start = first.offset, end = first.offset;
+ final int regionLength = region.size();
+
+ // Start at one, since we know the first token is an tag-open
+ for (int i = 1; (i < regionLength) && (errors < ERROR_THRESHOLD); i++) {
+ Token t = (Token) region.get(i);
+ if ((t.type == DOMRegionContext.XML_TAG_ATTRIBUTE_NAME) || (t.type == DOMRegionContext.XML_TAG_ATTRIBUTE_EQUALS) || (t.type == DOMRegionContext.XML_TAG_ATTRIBUTE_VALUE)) {
+ if (start == first.offset) {
+ start = t.offset;
+ }
+ end = t.offset + t.length;
+ errors++;
+ }
+ }
+
+ // create one error for all attributes in the end tag
+ if (errors > 0) {
+ // Position p = new Position(start, end - start);
+ String messageText = XMLCoreMessages.End_tag_has_attributes;
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.END_TAG_WITH_ATTRIBUTES), messageText);
+ message.setOffset(start);
+ message.setLength(end - start);
+ message.setLineNo(first.line + 1);
+
+ getAnnotationMsg(reporter, ProblemIDsXML.AttrsInEndTag, message, null, end-start);
+
+ }
+ }
+
+ /**
+ * Checks that all the attribute in the start-tag have values and that those values are properly quoted
+ * @param region the start-tag region
+ * @param reporter the reporter
+ */
+ private void checkAttributes(List region, IReporter reporter) {
+ int attrState = 0;
+ int errorCount = 0;
+ final int regionLength = region.size();
+
+ // Start at one, since we know the first token is an tag-open
+ for (int i = 1; i < regionLength && errorCount < ERROR_THRESHOLD; i++) {
+ Token t = (Token) region.get(i);
+ if (t.type == DOMRegionContext.XML_TAG_ATTRIBUTE_NAME || t.type == DOMRegionContext.XML_TAG_CLOSE || t.type == DOMRegionContext.XML_EMPTY_TAG_CLOSE) {
+ // dangling name and '='
+ if ((attrState == 2) && (i >= 2)) {
+ // create annotation
+ Token nameRegion = (Token) region.get(i - 2);
+ Object[] args = {nameRegion.text};
+ String messageText = NLS.bind(XMLCoreMessages.Attribute__is_missing_a_value, args);
+
+ int start = nameRegion.offset;
+ int end = start + nameRegion.length;
+
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.ATTRIBUTE_HAS_NO_VALUE) , messageText);
+ message.setOffset(start);
+ message.setLength(nameRegion.length);
+ message.setLineNo(nameRegion.line + 1);
+
+ // quick fix info
+ Token equalsRegion = (Token) region.get(i - 2 + 1);
+ int insertOffset = (equalsRegion.offset + equalsRegion.length) - end;
+ Object[] additionalFixInfo = {nameRegion.text, new Integer(insertOffset)};
+
+ getAnnotationMsg(reporter, ProblemIDsXML.MissingAttrValue, message, additionalFixInfo, nameRegion.text.length());
+ errorCount++;
+ }
+ // name but no '=' (XML only)
+ else if ((attrState == 1) && (i >= 1)) {
+ // create annotation
+ Token nameToken = (Token) region.get(i - 1);
+ Object[] args = {nameToken.text};
+ String messageText = NLS.bind(XMLCoreMessages.Attribute__has_no_value, args);
+ int start = nameToken.offset;
+ int textLength = nameToken.text.trim().length();
+ int lineNo = nameToken.line;
+
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.ATTRIBUTE_HAS_NO_VALUE), messageText);
+ message.setOffset(start);
+ message.setLength(textLength);
+ message.setLineNo(lineNo + 1);
+
+ getAnnotationMsg(reporter, ProblemIDsXML.NoAttrValue, message, nameToken.text, textLength);
+ errorCount++;
+ }
+ attrState = 1;
+ }
+ else if (t.type == DOMRegionContext.XML_TAG_ATTRIBUTE_EQUALS) {
+ attrState = 2;
+ }
+ else if (t.type == DOMRegionContext.XML_TAG_ATTRIBUTE_VALUE) {
+ attrState = 0;
+
+ // Check that there are quotes around the attribute value and that they match
+ final String trimmed = t.text.trim();
+ if (trimmed.length() > 0) {
+ final char q1 = trimmed.charAt(0), q2 = trimmed.charAt(trimmed.length() - 1);
+ if ((q1 == '\'' || q1 == '"') && (q1 != q2 || trimmed.length() == 1)) {
+ // missing closing quote
+ String message = XMLCoreMessages.ReconcileStepForMarkup_0;
+ addAttributeError(message, t.text, t.offset, t.length, t.line, ProblemIDsXML.Unclassified, reporter, getPluginPreference().getInt(XMLCorePreferenceNames.MISSING_CLOSING_QUOTE));
+ errorCount++;
+ }
+ else if (q1 != '\'' && q1 != '"') {
+ // missing both
+ String message = XMLCoreMessages.ReconcileStepForMarkup_1;
+ addAttributeError(message, t.text, t.offset, t.length, t.line, ProblemIDsXML.AttrValueNotQuoted, reporter, getPluginPreference().getInt(XMLCorePreferenceNames.MISSING_CLOSING_QUOTE));
+ errorCount++;
+ }
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Creates a missing tag error for the token
+ * @param token the token that's missing its pair tag
+ * @param isStartTag is the token a start tag
+ * @param reporter the reporter
+ */
+ private void createMissingTagError(Token token, boolean isStartTag, IReporter reporter) {
+ Object[] args = {token.text};
+ String messageText = NLS.bind(isStartTag ? XMLCoreMessages.Missing_end_tag_ : XMLCoreMessages.Missing_start_tag_, args);
+
+ LocalizedMessage message = new LocalizedMessage(getPluginPreference().getInt(XMLCorePreferenceNames.MISSING_END_TAG) , messageText);
+ message.setOffset(token.offset);
+ message.setLength(token.length);
+ message.setLineNo(getLine(token));
+
+ Object fixInfo = isStartTag ? (Object) getStartEndFixInfo(token.text, token) : token.text;
+ getAnnotationMsg(reporter, isStartTag ? ProblemIDsXML.MissingEndTag : ProblemIDsXML.MissingStartTag, message, fixInfo, token.length);
+
+ if (++tagErrorCount > ERROR_THRESHOLD) {
+ tagStack.clear();
+ tagStack = null;
+ }
+ }
+
+ private Object[] getStartEndFixInfo(String tagName, Token token) {
+ Object[] additionalInfo = null;
+ if (model != null) {
+ IDOMNode xmlNode = (IDOMNode) model.getIndexedRegion(token.offset);
+
+ if (xmlNode != null) {
+ // quick fix info
+ String tagClose = "/>"; //$NON-NLS-1$
+ int tagCloseOffset = xmlNode.getFirstStructuredDocumentRegion().getEndOffset();
+ ITextRegion last = xmlNode.getFirstStructuredDocumentRegion().getLastRegion();
+ if ((last != null) && (last.getType() == DOMRegionContext.XML_TAG_CLOSE)) {
+ tagClose = "/"; //$NON-NLS-1$
+ tagCloseOffset--;
+ }
+ IDOMNode firstChild = (IDOMNode) xmlNode.getFirstChild();
+ while ((firstChild != null) && (firstChild.getNodeType() == Node.TEXT_NODE)) {
+ firstChild = (IDOMNode) firstChild.getNextSibling();
+ }
+ int endOffset = xmlNode.getEndOffset();
+ int firstChildStartOffset = firstChild == null ? endOffset : firstChild.getStartOffset();
+ additionalInfo = new Object[] {tagName, tagClose, new Integer(tagCloseOffset), new Integer(xmlNode.getFirstStructuredDocumentRegion().getEndOffset()), // startTagEndOffset
+ new Integer(firstChildStartOffset), // firstChildStartOffset
+ new Integer(endOffset)}; // endOffset
+ }
+ }
+
+ return additionalInfo != null ? additionalInfo : new Object[] {};
+ }
+
+ private void checkTokens(XMLLineTokenizer tokenizer, IReporter reporter) throws IOException {
+ List previousRegion = null;
+ String type = null;
+ List region = null;
+ boolean isClosed = true;
+ while ((type = getNextToken(tokenizer)) != null) {
+
+ Token token = new Token(type, tokenizer.yytext(), tokenizer.getOffset(), tokenizer.yylength(), tokenizer.getLine());
+ isClosed = false;
+ if ((type == DOMRegionContext.XML_CONTENT) || (type == DOMRegionContext.XML_CHAR_REFERENCE) || (type == DOMRegionContext.XML_ENTITY_REFERENCE) || (type == DOMRegionContext.XML_PI_OPEN) || (type == DOMRegionContext.XML_TAG_OPEN) || (type == DOMRegionContext.XML_END_TAG_OPEN) || (type == DOMRegionContext.XML_COMMENT_OPEN) || (type == DOMRegionContext.XML_CDATA_OPEN) || (type == DOMRegionContext.XML_DECLARATION_OPEN)) {
+ // Validate the previous
+ // Create a new Region
+ previousRegion = region;
+ region = new ArrayList(0);
+ region.add(token);
+ if (type == DOMRegionContext.XML_PI_OPEN) {
+ checkContentBeforeProcessingInstruction(previousRegion, reporter);
+ }
+ else if (type == DOMRegionContext.XML_CONTENT) {
+ checkForSpaceBeforeName(token, previousRegion, reporter);
+ }
+ checkForTagClose(previousRegion, reporter);
+
+ }
+ else if ((type == DOMRegionContext.XML_TAG_NAME) || (type == DOMRegionContext.XML_TAG_ATTRIBUTE_NAME) || (type == DOMRegionContext.XML_TAG_ATTRIBUTE_EQUALS) || (type == DOMRegionContext.XML_TAG_ATTRIBUTE_VALUE) || (type == DOMRegionContext.XML_COMMENT_TEXT) || (type == DOMRegionContext.XML_PI_CONTENT) || (type == DOMRegionContext.XML_DOCTYPE_INTERNAL_SUBSET)) {
+ region.add(token);
+ }
+ else if ((type == DOMRegionContext.XML_PI_CLOSE) || (type == DOMRegionContext.XML_TAG_CLOSE) || (type == DOMRegionContext.XML_EMPTY_TAG_CLOSE) || (type == DOMRegionContext.XML_COMMENT_CLOSE) || (type == DOMRegionContext.XML_DECLARATION_CLOSE) || (type == DOMRegionContext.XML_CDATA_CLOSE)) {
+ region.add(token);
+ if (type == DOMRegionContext.XML_PI_CLOSE) {
+ checkNamespacesInProcessingInstruction(region, reporter);
+ }
+ else if (type == DOMRegionContext.XML_TAG_CLOSE || type == DOMRegionContext.XML_EMPTY_TAG_CLOSE) {
+ checkEmptyTag(region, reporter);
+ final int regionLength = region.size();
+ if (regionLength > 0) {
+ Token first = (Token) region.get(0);
+ if (first.type == DOMRegionContext.XML_END_TAG_OPEN) {
+ checkAttributsInEndTag(first, region, reporter);
+ if (first.type == DOMRegionContext.XML_END_TAG_OPEN && tagStack != null) {
+ if (regionLength > 1) {
+ Token name = (Token) region.get(1);
+ if (tagStack.isEmpty()) {
+ // We have an end tag without a start tag
+ createMissingTagError(name, false, reporter);
+ }
+ else {
+ if (!((Token) tagStack.peek()).text.equals(name.text)) {
+ boolean wasFound = false;
+ final int stackSize = tagStack.size();
+ for (int i = stackSize - 1; i >= 0; i--) {
+ Token pointer = (Token) tagStack.get(i);
+ if (pointer.text.equals(name.text)) {
+ wasFound = true;
+ Token top = null;
+ // Found the opening tag - everything in between that was unclosed should be flagged
+ while (!tagStack.isEmpty() && !(top = (Token) tagStack.pop()).text.equals(pointer.text)) {
+ createMissingTagError(top, true, reporter);
+ }
+ break;
+ }
+ }
+ if (!wasFound) {
+ // End tag doesn't have a matching start
+ createMissingTagError(name, false, reporter);
+ }
+ }
+ else {
+ // We've got a match
+ tagStack.pop();
+ }
+ }
+ }
+ }
+ }
+ else if (first.type == DOMRegionContext.XML_TAG_OPEN) {
+ checkAttributes(region, reporter);
+ if (type == DOMRegionContext.XML_TAG_CLOSE && tagStack != null && regionLength > 1) {
+ Token name = (Token) region.get(1);
+ if (name.type == DOMRegionContext.XML_TAG_NAME) {
+ tagStack.push(name);
+ }
+ }
+ }
+ }
+ }
+ isClosed = true;
+ }
+ }
+
+ if (!isClosed && region != null) {
+ // Check some things about the last region, just in case it wasn't properly closed
+ final int regionLength = region.size();
+ if (regionLength > 0) {
+ final Token first = (Token) region.get(0);
+ if (first.type == DOMRegionContext.XML_PI_OPEN) {
+ checkNamespacesInProcessingInstruction(region, reporter);
+ }
+ if (first.type == DOMRegionContext.XML_TAG_OPEN) {
+ checkForTagClose(region, reporter);
+ }
+ }
+ }
+
+ if (tagStack != null) {
+ while (!tagStack.isEmpty()) {
+ createMissingTagError((Token) tagStack.pop(), true, reporter);
+ }
+ }
+ }
+
+ /**
+ * Gets the next token from the tokenizer.
+ * @param tokenizer the XML tokenizer for the file being validated
+ * @return the next token type from the tokenizer, or null if it's at the end of the file
+ */
+ private String getNextToken(XMLLineTokenizer tokenizer) {
+ String token = null;
+ try {
+ if (!tokenizer.isEOF()) {
+ token = tokenizer.primGetNextToken();
+ }
+ }
+ catch (IOException e) {
+ }
+ return token;
+ }
+
+ /**
+ * Validates the given file. It will stream the contents of the file without creating a model for the file; it will only
+ * use existing
+ * @param file the file to validate
+ * @param reporter the reporter
+ */
+ private void validateFile(IFile file, IReporter reporter) {
+ Message message = new LocalizedMessage(IMessage.LOW_SEVERITY, file.getFullPath().toString().substring(1));
+ reporter.displaySubtask(StreamingMarkupValidator.this, message);
+
+ XMLLineTokenizer tokenizer = null;
+ try {
+ tagStack = new Stack();
+ model = StructuredModelManager.getModelManager().getExistingModelForRead(file);
+ try {
+ if (model == null) {
+ tokenizer = new XMLLineTokenizer(new BufferedReader(new InputStreamReader(file.getContents(true), getCharset(file))));
+ }
+ else {
+ tokenizer = new XMLLineTokenizer(new BufferedReader(new DocumentReader(model.getStructuredDocument())));
+ }
+ checkTokens(tokenizer, reporter);
+ }
+ finally {
+ if (model != null) {
+ model.releaseFromRead();
+ model = null;
+ }
+ }
+ } catch (UnsupportedEncodingException e) {
+ } catch (CoreException e) {
+ } catch (IOException e) {
+ }
+ }
+
+ private void validateProject(IValidationContext helper, final IReporter reporter) {
+ // if uris[] length 0 -> validate() gets called for each project
+ if (helper instanceof IWorkbenchContext) {
+ IProject project = ((IWorkbenchContext) helper).getProject();
+ IResourceProxyVisitor visitor = new IResourceProxyVisitor() {
+ public boolean visit(IResourceProxy proxy) throws CoreException {
+ if (shouldValidate(proxy)) {
+ validateFile((IFile) proxy.requestResource(), reporter);
+ }
+ return true;
+ }
+ };
+ try {
+ // collect all jsp files for the project
+ project.accept(visitor, IResource.DEPTH_INFINITE);
+ }
+ catch (CoreException e) {
+ Logger.logException(e);
+ }
+ }
+ }
+
+ private String getCharset(IFile file) {
+ if (file != null && file.isAccessible()) {
+ try {
+ return file.getCharset(true);
+ } catch (CoreException e) {
+ }
+ }
+ return ResourcesPlugin.getEncoding();
+ }
+
+ private Preferences getPluginPreference(){
+ return XMLCorePlugin.getDefault().getPluginPreferences();
+ }
+
+ /**
+ * Creates an error related to an attribute
+ */
+ private void addAttributeError(String messageText, String attributeValueText, int start, int length, int line, int problemId, IReporter reporter, int messageSeverity) {
+ LocalizedMessage message = new LocalizedMessage(messageSeverity, messageText);
+ message.setOffset(start);
+ message.setLength(length);
+ message.setLineNo(line + 1);
+ getAnnotationMsg(reporter, problemId, message, attributeValueText,length);
+ }
+
+ private boolean shouldValidate(IResourceProxy proxy) {
+ if(proxy.getType() == IResource.FILE) {
+ String name = proxy.getName();
+ if(name.toLowerCase(Locale.US).endsWith(".xml")) { //$NON-NLS-1$
+ return true;
+ }
+ }
+ return shouldValidate(proxy.requestResource(), false);
+ }
+
+ private boolean shouldValidate(IResource file, boolean checkExtension) {
+ if (file == null || !file.exists() || file.getType() != IResource.FILE)
+ return false;
+ if (checkExtension) {
+ String extension = file.getFileExtension();
+ if (extension != null && "xml".endsWith(extension.toLowerCase(Locale.US))) //$NON-NLS-1$
+ return true;
+ }
+
+ IContentDescription contentDescription = null;
+ try {
+ contentDescription = ((IFile) file).getContentDescription();
+ if (contentDescription != null) {
+ IContentType contentType = contentDescription.getContentType();
+ return contentDescription != null && contentType.isKindOf(getXMLContentType());
+ }
+ }
+ catch (CoreException e) {
+ Logger.logException(e);
+ }
+ return false;
+ }
+
+ private IContentType getXMLContentType() {
+ if (xmlContentType == null) {
+ xmlContentType = Platform.getContentTypeManager().getContentType("org.eclipse.core.runtime.xml"); //$NON-NLS-1$
+ }
+ return xmlContentType;
+ }
+}
diff --git a/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/XMLValidator.java b/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/XMLValidator.java
index f94d22b..22f3d19 100644
--- a/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/XMLValidator.java
+++ b/bundles/org.eclipse.wst.xml.core/src-validation/org/eclipse/wst/xml/core/internal/validation/XMLValidator.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2001, 2012 IBM Corporation and others.
+ * Copyright (c) 2001, 2013 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -97,7 +97,7 @@
private static final String FILE_NOT_FOUND_KEY = "FILE_NOT_FOUND"; //$NON-NLS-1$
- private MarkupValidator val = new MarkupValidator();
+ private StreamingMarkupValidator val = new StreamingMarkupValidator();
private final String ANNOTATIONMSG = AnnotationMsg.class.getName();
@@ -511,7 +511,7 @@
}
IReporter reporter = null;
if (resource != null){
- reporter = val.validate(resource, 0, new ValOperation().getState()) ;
+ reporter = val.validate(resource, 0, new ValOperation().getState());
}
return reporter;
}
diff --git a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLLineTokenizer.java b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/parser/XMLLineTokenizer.java
similarity index 98%
rename from bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLLineTokenizer.java
rename to bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/parser/XMLLineTokenizer.java
index 02eb4e7..0e147a1 100644
--- a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLLineTokenizer.java
+++ b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/parser/XMLLineTokenizer.java
@@ -1,42 +1,42 @@
-/* The following code was generated by JFlex 1.2.2 on 12/7/13 8:44 PM */
+/* The following code was generated by JFlex 1.2.2 on 12/20/13 11:48 AM */
-/*******************************************************************************
- * Copyright (c) 2004, 2013 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * IBM Corporation - initial API and implementation
- *******************************************************************************/
-
-package org.eclipse.wst.xml.core.internal.tasks;
-
-import java.io.CharArrayReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.eclipse.wst.sse.core.internal.ltk.parser.BlockMarker;
-import org.eclipse.wst.sse.core.internal.ltk.parser.BlockTokenizer;
-import org.eclipse.wst.sse.core.internal.provisional.text.ITextRegion;
-import org.eclipse.wst.sse.core.internal.util.Debug;
-import org.eclipse.wst.sse.core.utils.StringUtils;
-import org.eclipse.wst.xml.core.internal.Logger;
-import org.eclipse.wst.xml.core.internal.parser.IntStack;
-import org.eclipse.wst.xml.core.internal.parser.regions.XMLParserRegionFactory;
-import org.eclipse.wst.xml.core.internal.regions.DOMRegionContext;
-
+/*******************************************************************************
+ * Copyright (c) 2004, 2013 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * IBM Corporation - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.wst.xml.core.internal.parser;
+
+import java.io.CharArrayReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.eclipse.wst.sse.core.internal.ltk.parser.BlockMarker;
+import org.eclipse.wst.sse.core.internal.ltk.parser.BlockTokenizer;
+import org.eclipse.wst.sse.core.internal.provisional.text.ITextRegion;
+import org.eclipse.wst.sse.core.internal.util.Debug;
+import org.eclipse.wst.sse.core.utils.StringUtils;
+import org.eclipse.wst.xml.core.internal.Logger;
+import org.eclipse.wst.xml.core.internal.parser.IntStack;
+import org.eclipse.wst.xml.core.internal.parser.regions.XMLParserRegionFactory;
+import org.eclipse.wst.xml.core.internal.regions.DOMRegionContext;
+
/**
* This class is a scanner generated by
* <a href="http://www.informatik.tu-muenchen.de/~kleing/jflex/">JFlex</a> 1.2.2
- * on 12/7/13 8:44 PM from the specification file
- * <tt>file:/Users/nitin/git/webtools.sourceediting/bundles/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLLineTokenizer.jflex</tt>
+ * on 12/20/13 11:48 AM from the specification file
+ * <tt>file:/Users/dev/git/wtp/luna/webtools.sourceediting/bundles/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLLineTokenizer.jflex</tt>
*/
-class XMLLineTokenizer implements BlockTokenizer, DOMRegionContext {
+public class XMLLineTokenizer implements BlockTokenizer, DOMRegionContext {
/** this character denotes the end of file */
final public static int YYEOF = -1;
@@ -488,455 +488,459 @@
private boolean yy_eof_done;
/* user code: */
- private int fTokenCount = 0;
-
- // required holders for white-space compacting
- private boolean fShouldLoadBuffered = false;
- private String fBufferedContext = null;
- private int fBufferedStart = 1;
- private int fBufferedLength = 0;
- private String f_context = null;
-
- // state stack for handling embedded regions
- private IntStack fStateStack = new IntStack();
-
- private String context = null;
- private int start = 0;
- private int textLength = 0;
- private int length = 0;
-
- // offset for tracking position specific block tags
- private int fOffset = 0;
-
- // the name of the current tag being opened
- private String fCurrentTagName = null;
-
- // the list of tag name BlockMarkers
- private List fBlockMarkers = new ArrayList();
-
- // required to not seek text blocks on an end tag
- private boolean fIsBlockingEnabled = false;
- private boolean fIsCaseSensitiveBlocking = true;
-
- private XMLParserRegionFactory fRegionFactory = new XMLParserRegionFactory();
-/**
- * user method
- */
-public final void addBlockMarker(BlockMarker marker) {
- if(containsTagName(marker.getTagName()))
- return;
- fBlockMarkers.add(marker);
-}
-/**
- * user method
- */
-public final void removeBlockMarker(BlockMarker marker) {
- fBlockMarkers.remove(marker);
-}
-/**
- * user method
- */
-public final void removeBlockMarker(String tagname) {
- if (fBlockMarkers != null) {
- Iterator blocks = fBlockMarkers.iterator();
- while (blocks.hasNext()) {
- if (((BlockMarker) blocks.next()).getTagName().equals(tagname))
- blocks.remove();
- }
- }
-}
-/* user method */
-public final boolean isCaseSensitiveBlocking() {
- return fIsCaseSensitiveBlocking;
-}
-/* user method */
-public final void setCaseSensitiveBlocking(boolean newValue) {
- fIsCaseSensitiveBlocking = newValue;
-}
-/* user method */
-public boolean getBlockMarkerCaseSensitivity() {
- return getBlockMarkerCaseSensitivity(fCurrentTagName);
-}
-/* user method */
-public boolean getBlockMarkerCaseSensitivity(String name) {
- Iterator iterator = fBlockMarkers.iterator();
- while(iterator.hasNext()) {
- BlockMarker marker = (BlockMarker)iterator.next();
- boolean casesensitive = marker.isCaseSensitive();
- if(casesensitive && marker.getTagName().equals(name))
- return casesensitive;
- else if(!casesensitive && marker.getTagName().equalsIgnoreCase(name))
- return casesensitive;
- }
- return true;
-}
-/* user method */
-public String getBlockMarkerContext() {
- return getBlockMarkerContext(fCurrentTagName);
-}
-/* user method */
-public String getBlockMarkerContext(String name) {
- Iterator iterator = fBlockMarkers.iterator();
- while(iterator.hasNext()) {
- BlockMarker marker = (BlockMarker)iterator.next();
- if(marker.getTagName().equals(name))
- return marker.getContext();
- }
- return BLOCK_TEXT;
-}
-/* user method */
-public List getBlockMarkers() {
- return fBlockMarkers;
-}
-/* user method */
-public final int getOffset() {
- return fOffset + yychar;
-}
-private final boolean isBlockMarker() {
- return isBlockMarker(fCurrentTagName);
-}
-private final boolean isBlockMarker(String tagName) {
- if (!fIsBlockingEnabled)
- return false;
- return containsTagName(tagName);
-}
-/**
- * user method
- */
-public final void beginBlockTagScan(String newTagName) {
- beginBlockMarkerScan(newTagName, BLOCK_TEXT);
-}
-/**
- * user method
- *
- * Special tokenizer setup. Allows tokenization to be initiated at the
- * start of a text block within a "newTagName" tag.
- *
- * Example:
- * Tokenizer toker = new Tokenizer();
- * toker.setCaseSensitiveBlocking(false);
- * toker.reset(new java.io.StringReader("afiuhqwkejhtasihgalkwhtq</scripter></scr></script>asgdasga"));
- * toker.beginBlockMarkerScan("script", BLOCK_TEXT);
- * toker.getRegions();
- *
- * Returns:
- * BLOCK_TEXT: 0-40
- * XML_END_TAG_OPEN: 41-42
- * XML_TAG_NAME: 43-48
- * XML_TAG_CLOSE: 49-49
- * XML_CONTENT: 50-57
- *
- */
-public final void beginBlockMarkerScan(String newTagName, String blockcontext) {
- yybegin(ST_BLOCK_TAG_SCAN);
- fCurrentTagName = newTagName;
-}
-/**
- * Method doScan.
- *
- * Returns a context region for all of the text from the current position upto the end of input or
- * to right *before* the first occurence of searchString
- *
- * @param searchString - target string to search for ex.: "-->", "</tagname"
- * @param requireTailSeparator - whether the target must be immediately followed by whitespace or '>'
- * @param context - the context of the scanned region if non-zero length
- * @param exitState - the state to go to if the region was of non-zero length
- * @param abortState - the state to go to if the searchString was found immediately
- * @return String - the context found: the desired context on a non-zero length match, the abortContext on immediate success
- * @throws IOException
- */
-private final String doScan(String searchString, boolean requireTailSeparator, String searchContext, int exitState, int immediateFallbackState) throws IOException {
- boolean stillSearching = true;
- // Disable further block (probably)
- fIsBlockingEnabled = false;
- int searchStringLength = searchString.length();
- int n = 0;
- char lastCheckChar;
- int i;
- boolean same = false;
- while (stillSearching) {
- n = 0;
- // Ensure that enough data from the input exists to compare against the search String.
- n = yy_advance();
- while(n != YYEOF && yy_currentPos < searchStringLength)
- n = yy_advance();
- // If the input was too short or we've exhausted the input, stop immediately.
- if (n == YYEOF) {
- stillSearching = false;
- }
- else {
- same = true;
- // Ensure that we've not encountered a complete block (<%%>) that was *shorter* than the closeTagString and
- // thus found twice at current-targetLength [since the first scan would have come out this far anyway].
- // Check the characters in the target versus the last targetLength characters read from the buffer
- // and see if it matches
-
- // safety check for array accesses (yy_currentPos is the *last* character we can check against)
- if(yy_currentPos >= searchStringLength && yy_currentPos <= yy_buffer.length) {
- for(i = 0; i < searchStringLength; i++) {
- if(same && fIsCaseSensitiveBlocking)
- same = yy_buffer[i + yy_currentPos - searchStringLength] == searchString.charAt(i);
- else if(same && !fIsCaseSensitiveBlocking)
- same = Character.toLowerCase(yy_buffer[i + yy_currentPos - searchStringLength]) == Character.toLowerCase(searchString.charAt(i));
- }
- }
- // safety check failed; no match is possible right now
- else {
- same = false;
- }
- if (same && requireTailSeparator && yy_currentPos < yy_buffer.length) {
- // Additional check for close tags to ensure that targetString="</script" doesn't match
- // "</scriptS"
- lastCheckChar = yy_buffer[yy_currentPos];
- // Succeed on "</script>" and "</script "
- if(lastCheckChar == '>' || Character.isWhitespace(lastCheckChar))
- stillSearching = false;
- }
- else {
- stillSearching = !same || (yy_currentPos < yy_startRead + searchStringLength);
- }
- }
- }
- if (n != YYEOF || same) {
- // We've stopped short of the end or definitely found a match
- yy_markedPos = yy_currentPos - searchStringLength;
- yy_currentPos = yy_markedPos + 1;
- // If the searchString occurs at the very beginning of what would have
- // been a Block, resume scanning normally immediately
- if (yy_markedPos == yy_startRead) {
- yybegin(immediateFallbackState);
- return primGetNextToken();
- }
- }
- else {
- // We ran through the rest of the input
- yy_markedPos = yy_currentPos;
- yy_currentPos++;
- }
- yybegin(exitState);
- // If the ending occurs at the very beginning of what would have
- // been a Block, resume scanning normally immediately
- if(yy_markedPos == yy_startRead)
- return primGetNextToken();
- return searchContext;
-}
-/**
- * user method
- *
- * A generic lookahead-like operation
- */
-private final String doBlockScan(String target, String targetContext, int immediateFallbackState) throws IOException {
- return doScan(target, false, targetContext, immediateFallbackState, immediateFallbackState);
-}
-/**
- * user method
- * does a lookahead for the current tag name
- */
-private final String doBlockTagScan() throws IOException {
- fIsCaseSensitiveBlocking = getBlockMarkerCaseSensitivity();
- return doScan("</" + fCurrentTagName, true, getBlockMarkerContext(fCurrentTagName), YYINITIAL, YYINITIAL);
-}
-
-/**
- * user method
- *
- * Converts the raw context String returned by the primGetNextToken()
- * method into a full ITextRegion by pulling in values for the
- * current offset within the scanning text.
- *
- * Returns null when EOF is encountered and attaches intermittently
- * discovered whitespace onto the end of useful regions.
- *
- * Note that this algorithm caches the token following the one being returned
- * so that whitespace can be collapsed.
- */
-public final ITextRegion getNextToken() throws IOException {
- // load the starting non-whitespace token (assume that it is so)
- if (fShouldLoadBuffered) {
- context = fBufferedContext;
- start = fBufferedStart;
- textLength = length = fBufferedLength;
- fShouldLoadBuffered = false;
- }
- else {
- context = primGetNextToken();
- if (context == XML_TAG_NAME) {
- if(containsTagName(yy_buffer, yy_startRead, yy_markedPos-yy_startRead))
- fCurrentTagName = yytext();
- else
- fCurrentTagName = null;
- }
- else if (context == XML_TAG_OPEN) {
- fIsBlockingEnabled = true;
- }
- else if (context == XML_END_TAG_OPEN) {
- fIsBlockingEnabled = false;
- }
- start = yychar;
- textLength = length = yylength();
- if (yy_atEOF) {
- fTokenCount++;
- return null;
- }
- }
- // store the next token
- f_context = primGetNextToken();
- if (f_context == XML_TAG_NAME) {
- if(containsTagName(yy_buffer, yy_startRead, yy_markedPos-yy_startRead))
- fCurrentTagName = yytext();
- else
- fCurrentTagName = null;
- }
- else if (f_context == XML_TAG_OPEN) {
- fIsBlockingEnabled = true;
- }
- else if (f_context == XML_END_TAG_OPEN) {
- fIsBlockingEnabled = false;
- }
- fBufferedContext = f_context;
- fBufferedStart = yychar;
- fBufferedLength = yylength();
- fShouldLoadBuffered = true;
- if (fBufferedContext == WHITE_SPACE) {
- fShouldLoadBuffered = false;
- length += fBufferedLength;
- }
- if (context == null) {
- // EOF
- if (Debug.debugTokenizer) {
- System.out.println(getClass().getName() + " discovered " + fTokenCount + " tokens."); //$NON-NLS-2$//$NON-NLS-1$
- }
- return null;
- }
- fTokenCount++;
- return fRegionFactory.createToken(context, start, textLength, length, null, fCurrentTagName);
-}
-/* user method */
-public XMLLineTokenizer(){
- super();
-}
-/* user method */
-public XMLLineTokenizer(char[] charArray){
- this(new CharArrayReader(charArray));
-}
-/* user method */
-public void reset(char[] charArray) {
- reset(new CharArrayReader(charArray), 0);
-}
-/* user method */
-public void reset(char[] charArray, int newOffset) {
- reset(new CharArrayReader(charArray), newOffset);
-}
-/* user method */
-public void reset(java.io.InputStream in) {
- reset(new java.io.InputStreamReader(in), 0);
-}
-/* user method */
-public void reset(java.io.InputStream in, int newOffset) {
- reset(new java.io.InputStreamReader(in), newOffset);
-}
-/* user method */
-public void reset(java.io.Reader in) {
- reset(in, 0);
-}
-/**
- * user method *
- *
- * Reset internal counters and vars to "newly created" values, in the hopes
- * that resetting a pre-existing tokenizer is faster than creating a new one.
- *
- * This method contains code blocks that were essentially duplicated from the
- * <em>generated</em> output of this specification before this method was
- * added. Those code blocks were under the above copyright.
- */
-public void reset(java.io.Reader in, int newOffset) {
- if (Debug.debugTokenizer) {
- System.out.println("resetting tokenizer");//$NON-NLS-1$
- }
- fOffset = newOffset;
-
- /* the input device */
- yy_reader = in;
-
- /* the current state of the DFA */
- yy_state = 0;
-
- /* the current lexical state */
- yy_lexical_state = YYINITIAL;
-
- /* this buffer contains the current text to be matched and is
- the source of the yytext() string */
- java.util.Arrays.fill(yy_buffer, (char)0);
-
- /* the textposition at the last accepting state */
- yy_markedPos = 0;
-
- /* the textposition at the last state to be included in yytext */
- //yy_pushbackPos = 0;
-
- /* the current text position in the buffer */
- yy_currentPos = 0;
-
- /* startRead marks the beginning of the yytext() string in the buffer */
- yy_startRead = 0;
-
- /**
- * endRead marks the last character in the buffer, that has been read
- * from input
- */
- yy_endRead = 0;
-
- /* number of newlines encountered up to the start of the matched text */
- yyline = 0;
-
- /* the number of characters up to the start of the matched text */
- yychar = 0;
-
- /* yy_atEOF == true <=> the scanner has returned a value for EOF */
- yy_atEOF = false;
-
- /* denotes if the user-EOF-code has already been executed */
- yy_eof_done = false;
-
-
- /* user vars: */
- fTokenCount = 0;
-
- fShouldLoadBuffered = false;
- fBufferedContext = null;
- fBufferedStart = 1;
- fBufferedLength = 0;
- fStateStack = new IntStack();
-
- context = null;
- start = 0;
- textLength = 0;
- length = 0;
-}
-
- /**
- * user method
- *
- */
- public BlockTokenizer newInstance() {
- XMLLineTokenizer newInstance = new XMLLineTokenizer();
- // global tagmarkers can be shared; they have no state and
- // are never destroyed (e.g. 'release')
- for(int i = 0; i < fBlockMarkers.size(); i++) {
- BlockMarker blockMarker = (BlockMarker) fBlockMarkers.get(i);
- if(blockMarker.isGlobal())
- newInstance.addBlockMarker(blockMarker);
- }
- return newInstance;
- }
-/* user method */
-private final String scanXMLCommentText() throws IOException {
- // Scan for '-->' and return the text up to that point as
- // XML_COMMENT_TEXT unless the string occurs IMMEDIATELY, in which
- // case change to the ST_XML_COMMENT_END state and return the next
- // context as usual.
- return doScan("-->", false, XML_COMMENT_TEXT, ST_XML_COMMENT_END, ST_XML_COMMENT_END);
-}
+ private int fTokenCount = 0;
+
+ // required holders for white-space compacting
+ private boolean fShouldLoadBuffered = false;
+ private String fBufferedContext = null;
+ private int fBufferedStart = 1;
+ private int fBufferedLength = 0;
+ private String f_context = null;
+
+ // state stack for handling embedded regions
+ private IntStack fStateStack = new IntStack();
+
+ private String context = null;
+ private int start = 0;
+ private int textLength = 0;
+ private int length = 0;
+
+ // offset for tracking position specific block tags
+ private int fOffset = 0;
+
+ // the name of the current tag being opened
+ private String fCurrentTagName = null;
+
+ // the list of tag name BlockMarkers
+ private List fBlockMarkers = new ArrayList();
+
+ // required to not seek text blocks on an end tag
+ private boolean fIsBlockingEnabled = false;
+ private boolean fIsCaseSensitiveBlocking = true;
+
+ private XMLParserRegionFactory fRegionFactory = new XMLParserRegionFactory();
+/**
+ * user method
+ */
+public final void addBlockMarker(BlockMarker marker) {
+ if(containsTagName(marker.getTagName()))
+ return;
+ fBlockMarkers.add(marker);
+}
+/**
+ * user method
+ */
+public final void removeBlockMarker(BlockMarker marker) {
+ fBlockMarkers.remove(marker);
+}
+/**
+ * user method
+ */
+public final void removeBlockMarker(String tagname) {
+ if (fBlockMarkers != null) {
+ Iterator blocks = fBlockMarkers.iterator();
+ while (blocks.hasNext()) {
+ if (((BlockMarker) blocks.next()).getTagName().equals(tagname))
+ blocks.remove();
+ }
+ }
+}
+/* user method */
+public final boolean isCaseSensitiveBlocking() {
+ return fIsCaseSensitiveBlocking;
+}
+/* user method */
+public final void setCaseSensitiveBlocking(boolean newValue) {
+ fIsCaseSensitiveBlocking = newValue;
+}
+/* user method */
+public boolean getBlockMarkerCaseSensitivity() {
+ return getBlockMarkerCaseSensitivity(fCurrentTagName);
+}
+/* user method */
+public boolean getBlockMarkerCaseSensitivity(String name) {
+ Iterator iterator = fBlockMarkers.iterator();
+ while(iterator.hasNext()) {
+ BlockMarker marker = (BlockMarker)iterator.next();
+ boolean casesensitive = marker.isCaseSensitive();
+ if(casesensitive && marker.getTagName().equals(name))
+ return casesensitive;
+ else if(!casesensitive && marker.getTagName().equalsIgnoreCase(name))
+ return casesensitive;
+ }
+ return true;
+}
+/* user method */
+public String getBlockMarkerContext() {
+ return getBlockMarkerContext(fCurrentTagName);
+}
+/* user method */
+public String getBlockMarkerContext(String name) {
+ Iterator iterator = fBlockMarkers.iterator();
+ while(iterator.hasNext()) {
+ BlockMarker marker = (BlockMarker)iterator.next();
+ if(marker.getTagName().equals(name))
+ return marker.getContext();
+ }
+ return BLOCK_TEXT;
+}
+/* user method */
+public List getBlockMarkers() {
+ return fBlockMarkers;
+}
+/* user method */
+public final int getOffset() {
+ return fOffset + yychar;
+}
+/* user method */
+public final int getLine() {
+ return yyline;
+}
+private final boolean isBlockMarker() {
+ return isBlockMarker(fCurrentTagName);
+}
+private final boolean isBlockMarker(String tagName) {
+ if (!fIsBlockingEnabled)
+ return false;
+ return containsTagName(tagName);
+}
+/**
+ * user method
+ */
+public final void beginBlockTagScan(String newTagName) {
+ beginBlockMarkerScan(newTagName, BLOCK_TEXT);
+}
+/**
+ * user method
+ *
+ * Special tokenizer setup. Allows tokenization to be initiated at the
+ * start of a text block within a "newTagName" tag.
+ *
+ * Example:
+ * Tokenizer toker = new Tokenizer();
+ * toker.setCaseSensitiveBlocking(false);
+ * toker.reset(new java.io.StringReader("afiuhqwkejhtasihgalkwhtq</scripter></scr></script>asgdasga"));
+ * toker.beginBlockMarkerScan("script", BLOCK_TEXT);
+ * toker.getRegions();
+ *
+ * Returns:
+ * BLOCK_TEXT: 0-40
+ * XML_END_TAG_OPEN: 41-42
+ * XML_TAG_NAME: 43-48
+ * XML_TAG_CLOSE: 49-49
+ * XML_CONTENT: 50-57
+ *
+ */
+public final void beginBlockMarkerScan(String newTagName, String blockcontext) {
+ yybegin(ST_BLOCK_TAG_SCAN);
+ fCurrentTagName = newTagName;
+}
+/**
+ * Method doScan.
+ *
+ * Returns a context region for all of the text from the current position upto the end of input or
+ * to right *before* the first occurence of searchString
+ *
+ * @param searchString - target string to search for ex.: "-->", "</tagname"
+ * @param requireTailSeparator - whether the target must be immediately followed by whitespace or '>'
+ * @param context - the context of the scanned region if non-zero length
+ * @param exitState - the state to go to if the region was of non-zero length
+ * @param abortState - the state to go to if the searchString was found immediately
+ * @return String - the context found: the desired context on a non-zero length match, the abortContext on immediate success
+ * @throws IOException
+ */
+private final String doScan(String searchString, boolean requireTailSeparator, String searchContext, int exitState, int immediateFallbackState) throws IOException {
+ boolean stillSearching = true;
+ // Disable further block (probably)
+ fIsBlockingEnabled = false;
+ int searchStringLength = searchString.length();
+ int n = 0;
+ char lastCheckChar;
+ int i;
+ boolean same = false;
+ while (stillSearching) {
+ n = 0;
+ // Ensure that enough data from the input exists to compare against the search String.
+ n = yy_advance();
+ while(n != YYEOF && yy_currentPos < searchStringLength)
+ n = yy_advance();
+ // If the input was too short or we've exhausted the input, stop immediately.
+ if (n == YYEOF) {
+ stillSearching = false;
+ }
+ else {
+ same = true;
+ // Ensure that we've not encountered a complete block (<%%>) that was *shorter* than the closeTagString and
+ // thus found twice at current-targetLength [since the first scan would have come out this far anyway].
+ // Check the characters in the target versus the last targetLength characters read from the buffer
+ // and see if it matches
+
+ // safety check for array accesses (yy_currentPos is the *last* character we can check against)
+ if(yy_currentPos >= searchStringLength && yy_currentPos <= yy_buffer.length) {
+ for(i = 0; i < searchStringLength; i++) {
+ if(same && fIsCaseSensitiveBlocking)
+ same = yy_buffer[i + yy_currentPos - searchStringLength] == searchString.charAt(i);
+ else if(same && !fIsCaseSensitiveBlocking)
+ same = Character.toLowerCase(yy_buffer[i + yy_currentPos - searchStringLength]) == Character.toLowerCase(searchString.charAt(i));
+ }
+ }
+ // safety check failed; no match is possible right now
+ else {
+ same = false;
+ }
+ if (same && requireTailSeparator && yy_currentPos < yy_buffer.length) {
+ // Additional check for close tags to ensure that targetString="</script" doesn't match
+ // "</scriptS"
+ lastCheckChar = yy_buffer[yy_currentPos];
+ // Succeed on "</script>" and "</script "
+ if(lastCheckChar == '>' || Character.isWhitespace(lastCheckChar))
+ stillSearching = false;
+ }
+ else {
+ stillSearching = !same || (yy_currentPos < yy_startRead + searchStringLength);
+ }
+ }
+ }
+ if (n != YYEOF || same) {
+ // We've stopped short of the end or definitely found a match
+ yy_markedPos = yy_currentPos - searchStringLength;
+ yy_currentPos = yy_markedPos + 1;
+ // If the searchString occurs at the very beginning of what would have
+ // been a Block, resume scanning normally immediately
+ if (yy_markedPos == yy_startRead) {
+ yybegin(immediateFallbackState);
+ return primGetNextToken();
+ }
+ }
+ else {
+ // We ran through the rest of the input
+ yy_markedPos = yy_currentPos;
+ yy_currentPos++;
+ }
+ yybegin(exitState);
+ // If the ending occurs at the very beginning of what would have
+ // been a Block, resume scanning normally immediately
+ if(yy_markedPos == yy_startRead)
+ return primGetNextToken();
+ return searchContext;
+}
+/**
+ * user method
+ *
+ * A generic lookahead-like operation
+ */
+private final String doBlockScan(String target, String targetContext, int immediateFallbackState) throws IOException {
+ return doScan(target, false, targetContext, immediateFallbackState, immediateFallbackState);
+}
+/**
+ * user method
+ * does a lookahead for the current tag name
+ */
+private final String doBlockTagScan() throws IOException {
+ fIsCaseSensitiveBlocking = getBlockMarkerCaseSensitivity();
+ return doScan("</" + fCurrentTagName, true, getBlockMarkerContext(fCurrentTagName), YYINITIAL, YYINITIAL);
+}
+
+/**
+ * user method
+ *
+ * Converts the raw context String returned by the primGetNextToken()
+ * method into a full ITextRegion by pulling in values for the
+ * current offset within the scanning text.
+ *
+ * Returns null when EOF is encountered and attaches intermittently
+ * discovered whitespace onto the end of useful regions.
+ *
+ * Note that this algorithm caches the token following the one being returned
+ * so that whitespace can be collapsed.
+ */
+public final ITextRegion getNextToken() throws IOException {
+ // load the starting non-whitespace token (assume that it is so)
+ if (fShouldLoadBuffered) {
+ context = fBufferedContext;
+ start = fBufferedStart;
+ textLength = length = fBufferedLength;
+ fShouldLoadBuffered = false;
+ }
+ else {
+ context = primGetNextToken();
+ if (context == XML_TAG_NAME) {
+ if(containsTagName(yy_buffer, yy_startRead, yy_markedPos-yy_startRead))
+ fCurrentTagName = yytext();
+ else
+ fCurrentTagName = null;
+ }
+ else if (context == XML_TAG_OPEN) {
+ fIsBlockingEnabled = true;
+ }
+ else if (context == XML_END_TAG_OPEN) {
+ fIsBlockingEnabled = false;
+ }
+ start = yychar;
+ textLength = length = yylength();
+ if (yy_atEOF) {
+ fTokenCount++;
+ return null;
+ }
+ }
+ // store the next token
+ f_context = primGetNextToken();
+ if (f_context == XML_TAG_NAME) {
+ if(containsTagName(yy_buffer, yy_startRead, yy_markedPos-yy_startRead))
+ fCurrentTagName = yytext();
+ else
+ fCurrentTagName = null;
+ }
+ else if (f_context == XML_TAG_OPEN) {
+ fIsBlockingEnabled = true;
+ }
+ else if (f_context == XML_END_TAG_OPEN) {
+ fIsBlockingEnabled = false;
+ }
+ fBufferedContext = f_context;
+ fBufferedStart = yychar;
+ fBufferedLength = yylength();
+ fShouldLoadBuffered = true;
+ if (fBufferedContext == WHITE_SPACE) {
+ fShouldLoadBuffered = false;
+ length += fBufferedLength;
+ }
+ if (context == null) {
+ // EOF
+ if (Debug.debugTokenizer) {
+ System.out.println(getClass().getName() + " discovered " + fTokenCount + " tokens."); //$NON-NLS-2$//$NON-NLS-1$
+ }
+ return null;
+ }
+ fTokenCount++;
+ return fRegionFactory.createToken(context, start, textLength, length, null, fCurrentTagName);
+}
+/* user method */
+public XMLLineTokenizer(){
+ super();
+}
+/* user method */
+public XMLLineTokenizer(char[] charArray){
+ this(new CharArrayReader(charArray));
+}
+/* user method */
+public void reset(char[] charArray) {
+ reset(new CharArrayReader(charArray), 0);
+}
+/* user method */
+public void reset(char[] charArray, int newOffset) {
+ reset(new CharArrayReader(charArray), newOffset);
+}
+/* user method */
+public void reset(java.io.InputStream in) {
+ reset(new java.io.InputStreamReader(in), 0);
+}
+/* user method */
+public void reset(java.io.InputStream in, int newOffset) {
+ reset(new java.io.InputStreamReader(in), newOffset);
+}
+/* user method */
+public void reset(java.io.Reader in) {
+ reset(in, 0);
+}
+/**
+ * user method *
+ *
+ * Reset internal counters and vars to "newly created" values, in the hopes
+ * that resetting a pre-existing tokenizer is faster than creating a new one.
+ *
+ * This method contains code blocks that were essentially duplicated from the
+ * <em>generated</em> output of this specification before this method was
+ * added. Those code blocks were under the above copyright.
+ */
+public void reset(java.io.Reader in, int newOffset) {
+ if (Debug.debugTokenizer) {
+ System.out.println("resetting tokenizer");//$NON-NLS-1$
+ }
+ fOffset = newOffset;
+
+ /* the input device */
+ yy_reader = in;
+
+ /* the current state of the DFA */
+ yy_state = 0;
+
+ /* the current lexical state */
+ yy_lexical_state = YYINITIAL;
+
+ /* this buffer contains the current text to be matched and is
+ the source of the yytext() string */
+ java.util.Arrays.fill(yy_buffer, (char)0);
+
+ /* the textposition at the last accepting state */
+ yy_markedPos = 0;
+
+ /* the textposition at the last state to be included in yytext */
+ //yy_pushbackPos = 0;
+
+ /* the current text position in the buffer */
+ yy_currentPos = 0;
+
+ /* startRead marks the beginning of the yytext() string in the buffer */
+ yy_startRead = 0;
+
+ /**
+ * endRead marks the last character in the buffer, that has been read
+ * from input
+ */
+ yy_endRead = 0;
+
+ /* number of newlines encountered up to the start of the matched text */
+ yyline = 0;
+
+ /* the number of characters up to the start of the matched text */
+ yychar = 0;
+
+ /* yy_atEOF == true <=> the scanner has returned a value for EOF */
+ yy_atEOF = false;
+
+ /* denotes if the user-EOF-code has already been executed */
+ yy_eof_done = false;
+
+
+ /* user vars: */
+ fTokenCount = 0;
+
+ fShouldLoadBuffered = false;
+ fBufferedContext = null;
+ fBufferedStart = 1;
+ fBufferedLength = 0;
+ fStateStack = new IntStack();
+
+ context = null;
+ start = 0;
+ textLength = 0;
+ length = 0;
+}
+
+ /**
+ * user method
+ *
+ */
+ public BlockTokenizer newInstance() {
+ XMLLineTokenizer newInstance = new XMLLineTokenizer();
+ // global tagmarkers can be shared; they have no state and
+ // are never destroyed (e.g. 'release')
+ for(int i = 0; i < fBlockMarkers.size(); i++) {
+ BlockMarker blockMarker = (BlockMarker) fBlockMarkers.get(i);
+ if(blockMarker.isGlobal())
+ newInstance.addBlockMarker(blockMarker);
+ }
+ return newInstance;
+ }
+/* user method */
+private final String scanXMLCommentText() throws IOException {
+ // Scan for '-->' and return the text up to that point as
+ // XML_COMMENT_TEXT unless the string occurs IMMEDIATELY, in which
+ // case change to the ST_XML_COMMENT_END state and return the next
+ // context as usual.
+ return doScan("-->", false, XML_COMMENT_TEXT, ST_XML_COMMENT_END, ST_XML_COMMENT_END);
+}
/**
@@ -945,7 +949,7 @@
*
* @param in the java.io.Reader to read input from.
*/
- XMLLineTokenizer(java.io.Reader in) {
+ public XMLLineTokenizer(java.io.Reader in) {
this.yy_reader = in;
}
@@ -955,7 +959,7 @@
*
* @param in the java.io.Inputstream to read input from.
*/
- XMLLineTokenizer(java.io.InputStream in) {
+ public XMLLineTokenizer(java.io.InputStream in) {
this(new java.io.InputStreamReader(in));
}
@@ -1214,7 +1218,7 @@
private void yy_do_eof() {
if (!yy_eof_done) {
yy_eof_done = true;
- // do nothing, this is the downstream parser's job
+ // do nothing, this is the downstream parser's job
}
}
@@ -1293,146 +1297,146 @@
switch (yy_action) {
case 268:
- {
- if(Debug.debugTokenizer)
- dump("XSL processing instruction target");//$NON-NLS-1$
- yybegin(ST_XML_PI_ATTRIBUTE_NAME);
- return XML_TAG_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("XSL processing instruction target");//$NON-NLS-1$
+ yybegin(ST_XML_PI_ATTRIBUTE_NAME);
+ return XML_TAG_NAME;
}
case 270: break;
case 260:
case 262:
case 263:
- {
- if(Debug.debugTokenizer)
- dump("\nCDATA start");//$NON-NLS-1$
- fStateStack.push(yystate());
- yybegin(ST_CDATA_TEXT);
- return XML_CDATA_OPEN;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nCDATA start");//$NON-NLS-1$
+ fStateStack.push(yystate());
+ yybegin(ST_CDATA_TEXT);
+ return XML_CDATA_OPEN;
}
case 271: break;
case 253:
- {
- if(Debug.debugTokenizer)
- dump("element");//$NON-NLS-1$
- yybegin(ST_XML_ELEMENT_DECLARATION);
- return XML_ELEMENT_DECLARATION;
+ {
+ if(Debug.debugTokenizer)
+ dump("element");//$NON-NLS-1$
+ yybegin(ST_XML_ELEMENT_DECLARATION);
+ return XML_ELEMENT_DECLARATION;
}
case 272: break;
case 252:
- {
- if(Debug.debugTokenizer)
- dump("attlist");//$NON-NLS-1$
- yybegin(ST_XML_ATTLIST_DECLARATION);
- return XML_ATTLIST_DECLARATION;
+ {
+ if(Debug.debugTokenizer)
+ dump("attlist");//$NON-NLS-1$
+ yybegin(ST_XML_ATTLIST_DECLARATION);
+ return XML_ATTLIST_DECLARATION;
}
case 273: break;
case 251:
- {
- if(Debug.debugTokenizer)
- dump("doctype");//$NON-NLS-1$
- yybegin(ST_XML_DOCTYPE_DECLARATION);
- return XML_DOCTYPE_DECLARATION;
+ {
+ if(Debug.debugTokenizer)
+ dump("doctype");//$NON-NLS-1$
+ yybegin(ST_XML_DOCTYPE_DECLARATION);
+ return XML_DOCTYPE_DECLARATION;
}
case 274: break;
case 246:
- {
- if(Debug.debugTokenizer)
- dump("doctype external id");//$NON-NLS-1$
- yybegin(ST_XML_DOCTYPE_ID_PUBLIC);
- return XML_DOCTYPE_EXTERNAL_ID_PUBLIC;
+ {
+ if(Debug.debugTokenizer)
+ dump("doctype external id");//$NON-NLS-1$
+ yybegin(ST_XML_DOCTYPE_ID_PUBLIC);
+ return XML_DOCTYPE_EXTERNAL_ID_PUBLIC;
}
case 275: break;
case 245:
- {
- if(Debug.debugTokenizer)
- dump("doctype external id");//$NON-NLS-1$
- yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
- return XML_DOCTYPE_EXTERNAL_ID_SYSTEM;
+ {
+ if(Debug.debugTokenizer)
+ dump("doctype external id");//$NON-NLS-1$
+ yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
+ return XML_DOCTYPE_EXTERNAL_ID_SYSTEM;
}
case 276: break;
case 216:
- {
- if(Debug.debugTokenizer)
- dump("\nCharRef");//$NON-NLS-1$
- return XML_CHAR_REFERENCE;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nCharRef");//$NON-NLS-1$
+ return XML_CHAR_REFERENCE;
}
case 277: break;
case 212:
- {
- if(Debug.debugTokenizer)
- dump("\ncomment start");//$NON-NLS-1$
- yybegin(ST_XML_COMMENT);
- return XML_COMMENT_OPEN;
+ {
+ if(Debug.debugTokenizer)
+ dump("\ncomment start");//$NON-NLS-1$
+ yybegin(ST_XML_COMMENT);
+ return XML_COMMENT_OPEN;
}
case 278: break;
case 194:
case 195:
- {
- if(Debug.debugTokenizer)
- dump("XML processing instruction target");//$NON-NLS-1$
- yybegin(ST_XML_PI_ATTRIBUTE_NAME);
- return XML_TAG_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("XML processing instruction target");//$NON-NLS-1$
+ yybegin(ST_XML_PI_ATTRIBUTE_NAME);
+ return XML_TAG_NAME;
}
case 279: break;
case 193:
- {
- if(Debug.debugTokenizer)
- dump("comment end");//$NON-NLS-1$
- yybegin(YYINITIAL);
- return XML_COMMENT_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("comment end");//$NON-NLS-1$
+ yybegin(YYINITIAL);
+ return XML_COMMENT_CLOSE;
}
case 280: break;
case 192:
- {
- if(Debug.debugTokenizer)
- dump("CDATA end");//$NON-NLS-1$
- yybegin(fStateStack.pop());
- return XML_CDATA_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("CDATA end");//$NON-NLS-1$
+ yybegin(fStateStack.pop());
+ return XML_CDATA_CLOSE;
}
case 281: break;
case 191:
- {
- if(Debug.debugTokenizer)
- dump("\nPEReference");//$NON-NLS-1$
- return XML_PE_REFERENCE;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nPEReference");//$NON-NLS-1$
+ return XML_PE_REFERENCE;
}
case 282: break;
case 188:
- {
- if(Debug.debugTokenizer)
- dump("\nEntityRef");//$NON-NLS-1$
- return XML_ENTITY_REFERENCE;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nEntityRef");//$NON-NLS-1$
+ return XML_ENTITY_REFERENCE;
}
case 283: break;
case 139:
case 153:
case 161:
- {
- return XML_DOCTYPE_INTERNAL_SUBSET;
+ {
+ return XML_DOCTYPE_INTERNAL_SUBSET;
}
case 284: break;
case 127:
- {
- yybegin(YYINITIAL);
- if(Debug.debugTokenizer)
- dump("empty tag close");//$NON-NLS-1$
- return XML_EMPTY_TAG_CLOSE;
+ {
+ yybegin(YYINITIAL);
+ if(Debug.debugTokenizer)
+ dump("empty tag close");//$NON-NLS-1$
+ return XML_EMPTY_TAG_CLOSE;
}
case 285: break;
case 120:
- {
- if(Debug.debugTokenizer)
- dump("XML processing instruction end");//$NON-NLS-1$
- yybegin(YYINITIAL);
- return XML_PI_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("XML processing instruction end");//$NON-NLS-1$
+ yybegin(YYINITIAL);
+ return XML_PI_CLOSE;
}
case 286: break;
case 119:
- {
- // ended with nothing inside
- yybegin(YYINITIAL);
- return XML_PI_CLOSE;
+ {
+ // ended with nothing inside
+ yybegin(YYINITIAL);
+ return XML_PI_CLOSE;
}
case 287: break;
case 52:
@@ -1440,41 +1444,41 @@
case 55:
case 56:
case 124:
- {
- if(Debug.debugTokenizer)
- dump("XML processing instruction attribute value");//$NON-NLS-1$
- yybegin(ST_XML_PI_ATTRIBUTE_NAME);
- return XML_TAG_ATTRIBUTE_VALUE;
+ {
+ if(Debug.debugTokenizer)
+ dump("XML processing instruction attribute value");//$NON-NLS-1$
+ yybegin(ST_XML_PI_ATTRIBUTE_NAME);
+ return XML_TAG_ATTRIBUTE_VALUE;
}
case 288: break;
case 51:
- {
- if(Debug.debugTokenizer)
- dump("XML processing instruction '='");//$NON-NLS-1$
- yybegin(ST_XML_PI_ATTRIBUTE_VALUE);
- return XML_TAG_ATTRIBUTE_EQUALS;
+ {
+ if(Debug.debugTokenizer)
+ dump("XML processing instruction '='");//$NON-NLS-1$
+ yybegin(ST_XML_PI_ATTRIBUTE_VALUE);
+ return XML_TAG_ATTRIBUTE_EQUALS;
}
case 289: break;
case 50:
- {
- if(Debug.debugTokenizer)
- dump("XML processing instruction attribute name");//$NON-NLS-1$
- yybegin(ST_XML_PI_EQUALS);
- return XML_TAG_ATTRIBUTE_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("XML processing instruction attribute name");//$NON-NLS-1$
+ yybegin(ST_XML_PI_EQUALS);
+ return XML_TAG_ATTRIBUTE_NAME;
}
case 290: break;
case 46:
case 47:
case 48:
- {
- // block scan until close is found
- return doScan("?>", false, XML_PI_CONTENT, ST_XML_PI_TAG_CLOSE, ST_XML_PI_TAG_CLOSE);
+ {
+ // block scan until close is found
+ return doScan("?>", false, XML_PI_CONTENT, ST_XML_PI_TAG_CLOSE, ST_XML_PI_TAG_CLOSE);
}
case 291: break;
case 45:
- {
- yybegin(ST_PI_CONTENT);
- return WHITE_SPACE;
+ {
+ yybegin(ST_PI_CONTENT);
+ return WHITE_SPACE;
}
case 292: break;
case 42:
@@ -1492,37 +1496,37 @@
case 265:
case 266:
case 267:
- {
- if(Debug.debugTokenizer)
- dump("processing instruction target");//$NON-NLS-1$
- yybegin(ST_PI_WS);
- return XML_TAG_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("processing instruction target");//$NON-NLS-1$
+ yybegin(ST_PI_WS);
+ return XML_TAG_NAME;
}
case 293: break;
case 37:
case 38:
- {
- if(Debug.debugTokenizer)
- dump("comment content");//$NON-NLS-1$
- return scanXMLCommentText();
+ {
+ if(Debug.debugTokenizer)
+ dump("comment content");//$NON-NLS-1$
+ return scanXMLCommentText();
}
case 294: break;
case 36:
- {
- if(Debug.debugTokenizer)
- dump("LINE FEED");//$NON-NLS-1$
- return WHITE_SPACE;
+ {
+ if(Debug.debugTokenizer)
+ dump("LINE FEED");//$NON-NLS-1$
+ return WHITE_SPACE;
}
case 295: break;
case 31:
case 32:
- {
- if(Debug.debugTokenizer)
- dump("CDATA text");//$NON-NLS-1$
- String blockContext = doBlockScan("]]>", XML_CDATA_TEXT, ST_CDATA_END);//$NON-NLS-1$
- if(blockContext == XML_CDATA_TEXT)
- yybegin(ST_CDATA_END);
- return blockContext;
+ {
+ if(Debug.debugTokenizer)
+ dump("CDATA text");//$NON-NLS-1$
+ String blockContext = doBlockScan("]]>", XML_CDATA_TEXT, ST_CDATA_END);//$NON-NLS-1$
+ if(blockContext == XML_CDATA_TEXT)
+ yybegin(ST_CDATA_END);
+ return blockContext;
}
case 296: break;
case 0:
@@ -1535,10 +1539,10 @@
case 215:
case 230:
case 231:
- {
- if(Debug.debugTokenizer)
- dump("\nXML content");//$NON-NLS-1$
- return XML_CONTENT;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nXML content");//$NON-NLS-1$
+ return XML_CONTENT;
}
case 297: break;
case 5:
@@ -1557,19 +1561,19 @@
case 22:
case 24:
case 41:
- {
- if(Debug.debugTokenizer)
- dump("white space");//$NON-NLS-1$
- return WHITE_SPACE;
+ {
+ if(Debug.debugTokenizer)
+ dump("white space");//$NON-NLS-1$
+ return WHITE_SPACE;
}
case 298: break;
case 12:
case 58:
- {
- if(Debug.debugTokenizer)
- dump("inappropriate tag name");//$NON-NLS-1$
- yybegin(YYINITIAL);
- return XML_CONTENT;
+ {
+ if(Debug.debugTokenizer)
+ dump("inappropriate tag name");//$NON-NLS-1$
+ yybegin(YYINITIAL);
+ return XML_CONTENT;
}
case 299: break;
case 23:
@@ -1582,10 +1586,10 @@
case 247:
case 254:
case 258:
- {
- if(Debug.debugTokenizer)
- dump("elementdecl contentspec");//$NON-NLS-1$
- return XML_ELEMENT_DECL_CONTENT;
+ {
+ if(Debug.debugTokenizer)
+ dump("elementdecl contentspec");//$NON-NLS-1$
+ return XML_ELEMENT_DECL_CONTENT;
}
case 300: break;
case 25:
@@ -1598,20 +1602,20 @@
case 248:
case 255:
case 259:
- {
- if(Debug.debugTokenizer)
- dump("attlist contentspec");//$NON-NLS-1$
- return XML_ATTLIST_DECL_CONTENT;
+ {
+ if(Debug.debugTokenizer)
+ dump("attlist contentspec");//$NON-NLS-1$
+ return XML_ATTLIST_DECL_CONTENT;
}
case 301: break;
case 28:
case 59:
case 70:
- {
- if(Debug.debugTokenizer)
- dump("\nstart tag open");//$NON-NLS-1$
- yybegin(ST_XML_TAG_NAME);
- return XML_TAG_OPEN;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nstart tag open");//$NON-NLS-1$
+ yybegin(ST_XML_TAG_NAME);
+ return XML_TAG_OPEN;
}
case 302: break;
case 29:
@@ -1635,87 +1639,87 @@
case 85:
case 90:
case 97:
- {
- if (Debug.debugTokenizer)
- System.out.println("!!!unexpected!!!: \"" + yytext() + "\":" + //$NON-NLS-2$//$NON-NLS-1$
- yychar + "-" + (yychar + yylength()));//$NON-NLS-1$
- return UNDEFINED;
+ {
+ if (Debug.debugTokenizer)
+ System.out.println("!!!unexpected!!!: \"" + yytext() + "\":" + //$NON-NLS-2$//$NON-NLS-1$
+ yychar + "-" + (yychar + yylength()));//$NON-NLS-1$
+ return UNDEFINED;
}
case 303: break;
case 30:
case 112:
- {
- if(Debug.debugTokenizer)
- dump("non-reference %");//$NON-NLS-1$
- return XML_CONTENT;
+ {
+ if(Debug.debugTokenizer)
+ dump("non-reference %");//$NON-NLS-1$
+ return XML_CONTENT;
}
case 304: break;
case 60:
- {
- if(Debug.debugTokenizer)
- dump("tag close");//$NON-NLS-1$
- if(isBlockMarker()) {
- yybegin(ST_BLOCK_TAG_SCAN);
- }
- else
- yybegin(YYINITIAL);
- return XML_TAG_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("tag close");//$NON-NLS-1$
+ if(isBlockMarker()) {
+ yybegin(ST_BLOCK_TAG_SCAN);
+ }
+ else
+ yybegin(YYINITIAL);
+ return XML_TAG_CLOSE;
}
case 305: break;
case 62:
case 63:
- {
- if(Debug.debugTokenizer)
- dump("tag name");//$NON-NLS-1$
- yybegin(ST_XML_ATTRIBUTE_NAME);
- return XML_TAG_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("tag name");//$NON-NLS-1$
+ yybegin(ST_XML_ATTRIBUTE_NAME);
+ return XML_TAG_NAME;
}
case 306: break;
case 64:
- {
- if(Debug.debugTokenizer)
- dump("attr name");//$NON-NLS-1$
- yybegin(ST_XML_EQUALS);
- return XML_TAG_ATTRIBUTE_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("attr name");//$NON-NLS-1$
+ yybegin(ST_XML_EQUALS);
+ return XML_TAG_ATTRIBUTE_NAME;
}
case 307: break;
case 65:
- {
- if(Debug.debugTokenizer)
- dump("equals");//$NON-NLS-1$
- yybegin(ST_XML_ATTRIBUTE_VALUE);
- return XML_TAG_ATTRIBUTE_EQUALS;
+ {
+ if(Debug.debugTokenizer)
+ dump("equals");//$NON-NLS-1$
+ yybegin(ST_XML_ATTRIBUTE_VALUE);
+ return XML_TAG_ATTRIBUTE_EQUALS;
}
case 308: break;
case 66:
case 68:
case 69:
case 131:
- {
- if(Debug.debugTokenizer)
- dump("attr value");//$NON-NLS-1$
- yybegin(ST_XML_ATTRIBUTE_NAME);
- return XML_TAG_ATTRIBUTE_VALUE;
+ {
+ if(Debug.debugTokenizer)
+ dump("attr value");//$NON-NLS-1$
+ yybegin(ST_XML_ATTRIBUTE_NAME);
+ return XML_TAG_ATTRIBUTE_VALUE;
}
case 309: break;
case 71:
- {
- if(Debug.debugTokenizer)
- dump("declaration end");//$NON-NLS-1$
- if (Debug.debugTokenizer) {
- if(fStateStack.peek()!=YYINITIAL)
- System.out.println("end embedded region");//$NON-NLS-1$
- }
- yybegin(fStateStack.pop());
- return XML_DECLARATION_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("declaration end");//$NON-NLS-1$
+ if (Debug.debugTokenizer) {
+ if(fStateStack.peek()!=YYINITIAL)
+ System.out.println("end embedded region");//$NON-NLS-1$
+ }
+ yybegin(fStateStack.pop());
+ return XML_DECLARATION_CLOSE;
}
case 310: break;
case 76:
- {
- if(Debug.debugTokenizer)
- dump("doctype type");//$NON-NLS-1$
- yybegin(ST_XML_DOCTYPE_EXTERNAL_ID);
- return XML_DOCTYPE_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("doctype type");//$NON-NLS-1$
+ yybegin(ST_XML_DOCTYPE_EXTERNAL_ID);
+ return XML_DOCTYPE_NAME;
}
case 311: break;
case 79:
@@ -1727,11 +1731,11 @@
case 149:
case 150:
case 202:
- {
- if(Debug.debugTokenizer)
- dump("doctype public reference");//$NON-NLS-1$
- yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
- return XML_DOCTYPE_EXTERNAL_ID_PUBREF;
+ {
+ if(Debug.debugTokenizer)
+ dump("doctype public reference");//$NON-NLS-1$
+ yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
+ return XML_DOCTYPE_EXTERNAL_ID_PUBREF;
}
case 312: break;
case 84:
@@ -1739,11 +1743,11 @@
case 87:
case 88:
case 157:
- {
- if(Debug.debugTokenizer)
- dump("doctype system reference");//$NON-NLS-1$
- yybegin(ST_XML_DECLARATION_CLOSE);
- return XML_DOCTYPE_EXTERNAL_ID_SYSREF;
+ {
+ if(Debug.debugTokenizer)
+ dump("doctype system reference");//$NON-NLS-1$
+ yybegin(ST_XML_DECLARATION_CLOSE);
+ return XML_DOCTYPE_EXTERNAL_ID_SYSREF;
}
case 313: break;
case 89:
@@ -1754,23 +1758,23 @@
case 169:
case 170:
case 205:
- {
- if(Debug.debugTokenizer)
- dump("elementdecl name");//$NON-NLS-1$
- yybegin(ST_XML_ELEMENT_DECLARATION_CONTENT);
- return XML_ELEMENT_DECL_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("elementdecl name");//$NON-NLS-1$
+ yybegin(ST_XML_ELEMENT_DECLARATION_CONTENT);
+ return XML_ELEMENT_DECL_NAME;
}
case 314: break;
case 95:
- {
- if(Debug.debugTokenizer)
- dump("elementdecl close");//$NON-NLS-1$
- if (Debug.debugTokenizer) {
- if(fStateStack.peek()!=YYINITIAL)
- System.out.println("end embedded region");//$NON-NLS-1$
- }
- yybegin(fStateStack.pop());
- return XML_DECLARATION_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("elementdecl close");//$NON-NLS-1$
+ if (Debug.debugTokenizer) {
+ if(fStateStack.peek()!=YYINITIAL)
+ System.out.println("end embedded region");//$NON-NLS-1$
+ }
+ yybegin(fStateStack.pop());
+ return XML_DECLARATION_CLOSE;
}
case 315: break;
case 96:
@@ -1781,62 +1785,62 @@
case 180:
case 181:
case 209:
- {
- if(Debug.debugTokenizer)
- dump("attlist name");//$NON-NLS-1$
- yybegin(ST_XML_ATTLIST_DECLARATION_CONTENT);
- return XML_ATTLIST_DECL_NAME;
+ {
+ if(Debug.debugTokenizer)
+ dump("attlist name");//$NON-NLS-1$
+ yybegin(ST_XML_ATTLIST_DECLARATION_CONTENT);
+ return XML_ATTLIST_DECL_NAME;
}
case 316: break;
case 102:
- {
- if(Debug.debugTokenizer)
- dump("attlist close");//$NON-NLS-1$
- if (Debug.debugTokenizer) {
- if(fStateStack.peek()!=YYINITIAL)
- System.out.println("end embedded region");//$NON-NLS-1$
- }
- yybegin(fStateStack.pop());
- return XML_DECLARATION_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("attlist close");//$NON-NLS-1$
+ if (Debug.debugTokenizer) {
+ if(fStateStack.peek()!=YYINITIAL)
+ System.out.println("end embedded region");//$NON-NLS-1$
+ }
+ yybegin(fStateStack.pop());
+ return XML_DECLARATION_CLOSE;
}
case 317: break;
case 105:
- {
- if(Debug.debugTokenizer)
- dump("\nend tag open");//$NON-NLS-1$
- yybegin(ST_XML_TAG_NAME);
- return XML_END_TAG_OPEN;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nend tag open");//$NON-NLS-1$
+ yybegin(ST_XML_TAG_NAME);
+ return XML_END_TAG_OPEN;
}
case 318: break;
case 106:
- {
- if(Debug.debugTokenizer)
- dump("\nprocessing instruction start");//$NON-NLS-1$
- yybegin(ST_PI);
- return XML_PI_OPEN;
+ {
+ if(Debug.debugTokenizer)
+ dump("\nprocessing instruction start");//$NON-NLS-1$
+ yybegin(ST_PI);
+ return XML_PI_OPEN;
}
case 319: break;
case 107:
- {
- fStateStack.push(yystate());
- if(Debug.debugTokenizer)
- dump("\ndeclaration start");//$NON-NLS-1$
- yybegin(ST_XML_DECLARATION);
- return XML_DECLARATION_OPEN;
+ {
+ fStateStack.push(yystate());
+ if(Debug.debugTokenizer)
+ dump("\ndeclaration start");//$NON-NLS-1$
+ yybegin(ST_XML_DECLARATION);
+ return XML_DECLARATION_OPEN;
}
case 320: break;
case 116:
- {
- if(Debug.debugTokenizer)
- dump("processing instruction end");//$NON-NLS-1$
- yybegin(YYINITIAL);
- return XML_PI_CLOSE;
+ {
+ if(Debug.debugTokenizer)
+ dump("processing instruction end");//$NON-NLS-1$
+ yybegin(YYINITIAL);
+ return XML_PI_CLOSE;
}
case 321: break;
case 103:
case 104:
- {
- return doBlockTagScan();
+ {
+ return doBlockTagScan();
}
case 322: break;
default:
diff --git a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLStreamingFileTaskScanner.java b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLStreamingFileTaskScanner.java
index eea16f7..296d6a4 100644
--- a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLStreamingFileTaskScanner.java
+++ b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/tasks/XMLStreamingFileTaskScanner.java
@@ -37,6 +37,7 @@
import org.eclipse.wst.sse.core.internal.provisional.tasks.IFileTaskScanner;
import org.eclipse.wst.sse.core.internal.provisional.tasks.TaskTag;
import org.eclipse.wst.xml.core.internal.Logger;
+import org.eclipse.wst.xml.core.internal.parser.XMLLineTokenizer;
import org.eclipse.wst.xml.core.internal.regions.DOMRegionContext;
/**
@@ -128,7 +129,7 @@
String markerDescription = lineComment.substring(tagIndex);
int markerOffset = getOffset() + line.getOffset() + tagIndex;
int markerLength = line.getLength() - tagIndex;
- fNewMarkerAttributeMaps.add(createInitialMarkerAttributes(markerDescription, lineNumber + yyline, markerOffset, markerLength, taskTags[i].getPriority()));
+ fNewMarkerAttributeMaps.add(createInitialMarkerAttributes(markerDescription, lineNumber + getLine(), markerOffset, markerLength, taskTags[i].getPriority()));
}
}
}
