fixes for unit tests and content type
diff --git a/bundles/org.eclipse.wst.xml.core/plugin.xml b/bundles/org.eclipse.wst.xml.core/plugin.xml index 408e69d..89b2a7a 100644 --- a/bundles/org.eclipse.wst.xml.core/plugin.xml +++ b/bundles/org.eclipse.wst.xml.core/plugin.xml
@@ -65,7 +65,7 @@ contentTypeId="org.eclipse.core.runtime.xml" class="org.eclipse.wst.sse.core.internal.filebuffers.BasicStructuredDocumentFactory"/> </extension> - + <extension point="org.eclipse.team.core.fileTypes"> <fileTypes @@ -109,12 +109,11 @@ point="org.eclipse.core.runtime.contentTypes"> <content-type file-extensions="xml,xsl,xslt,dadx,wsdl,nst,xmi,xsd,exsd" - priority="low" + priority="high" name="%XML_Content_Type_Extension_Element.name" id="xmlsource" base-type="org.eclipse.core.runtime.xml" default-charset="UTF-8" - alias-for="org.eclipse.core.runtime.xml" > <describer class="org.eclipse.wst.xml.core.internal.contenttype.ContentDescriberForXML"/> </content-type>
diff --git a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/ContentDescriberForXML.java b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/ContentDescriberForXML.java index fdba1fb..082ef41 100644 --- a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/ContentDescriberForXML.java +++ b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/ContentDescriberForXML.java
@@ -27,6 +27,13 @@ public final class ContentDescriberForXML implements ITextContentDescriber { private final static QualifiedName[] SUPPORTED_OPTIONS = {IContentDescription.CHARSET, IContentDescription.BYTE_ORDER_MARK, IContentDescriptionExtended.DETECTED_CHARSET, IContentDescriptionExtended.UNSUPPORTED_CHARSET, IContentDescriptionExtended.APPROPRIATE_DEFAULT}; + /** + * <code>restrictedMode</code> is used just for testing/experiments. + * + * If in restrictedMode, our "custom" contentType is seen as valid only in cases + * that the platform's standard one does not cover. + */ + private boolean restrictedMode = false; private IResourceCharsetDetector getDetector() { return new XMLResourceEncodingDetector(); @@ -41,7 +48,7 @@ public int describe(InputStream contents, IContentDescription description) throws IOException { int result = IContentDescriber.INDETERMINATE; - calculateSupportedOptions(contents, description); + result = calculateSupportedOptions(result, contents, description); return result; } @@ -55,7 +62,7 @@ public int describe(Reader contents, IContentDescription description) throws IOException { int result = IContentDescriber.INDETERMINATE; - calculateSupportedOptions(contents, description); + result = calculateSupportedOptions(result, contents, description); return result; } @@ -70,12 +77,14 @@ return SUPPORTED_OPTIONS; } - private void calculateSupportedOptions(InputStream contents, IContentDescription description) throws IOException { + private int calculateSupportedOptions(int result, InputStream contents, IContentDescription description) throws IOException { + int returnResult = result; if (isRelevent(description)) { IResourceCharsetDetector detector = getDetector(); detector.set(contents); - handleCalculations(description, detector); + returnResult = handleCalculations(result, description, detector); } + return returnResult; } /** @@ -83,12 +92,14 @@ * @param description * @throws IOException */ - private void calculateSupportedOptions(Reader contents, IContentDescription description) throws IOException { + private int calculateSupportedOptions(int result, Reader contents, IContentDescription description) throws IOException { + int returnResult = result; if (isRelevent(description)) { IResourceCharsetDetector detector = getDetector(); detector.set(contents); - handleCalculations(description, detector); + returnResult = handleCalculations(result, description, detector); } + return returnResult; } private void handleDetectedSpecialCase(IContentDescription description, Object detectedCharset, Object javaCharset) { @@ -118,7 +129,7 @@ private boolean isRelevent(IContentDescription description) { boolean result = false; if (description == null) - result = false; + result = true; else if (description.isRequested(IContentDescription.BYTE_ORDER_MARK)) result = true; else if (description.isRequested(IContentDescription.CHARSET)) @@ -140,66 +151,87 @@ * @param detector * @throws IOException */ - private void handleCalculations(IContentDescription description, IResourceCharsetDetector detector) throws IOException { - // note: if we're asked for one, we set them all. I need to be sure if - // called - // mulitiple times (one for each, say) that we don't waste time - // processing same - // content again. + private int handleCalculations(int result, IContentDescription description, IResourceCharsetDetector detector) throws IOException { + int returnResult = result; EncodingMemento encodingMemento = detector.getEncodingMemento(); - // TODO: I need to verify to see if this BOM work is always done - // by text type. - Object detectedByteOrderMark = encodingMemento.getUnicodeBOM(); - if (detectedByteOrderMark != null) { - Object existingByteOrderMark = description.getProperty(IContentDescription.BYTE_ORDER_MARK); - // not sure why would ever be different, so if is different, may - // need to "push" up into base. - if (!detectedByteOrderMark.equals(existingByteOrderMark)) - description.setProperty(IContentDescription.BYTE_ORDER_MARK, detectedByteOrderMark); - } + if (description != null) { + // TODO: I need to verify to see if this BOM work is always done + // by text type. + Object detectedByteOrderMark = encodingMemento.getUnicodeBOM(); + if (detectedByteOrderMark != null) { + Object existingByteOrderMark = description.getProperty(IContentDescription.BYTE_ORDER_MARK); + // not sure why would ever be different, so if is different, + // may + // need to "push" up into base. + if (!detectedByteOrderMark.equals(existingByteOrderMark)) + description.setProperty(IContentDescription.BYTE_ORDER_MARK, detectedByteOrderMark); + } - if (!encodingMemento.isValid()) { - // note: after setting here, its the mere presence of - // IContentDescriptionExtended.UNSUPPORTED_CHARSET - // in the resource's description that can be used to determine if - // invalid - // in those cases, the "detected" property contains an - // "appropriate default" to use. - description.setProperty(IContentDescriptionExtended.UNSUPPORTED_CHARSET, encodingMemento.getInvalidEncoding()); - description.setProperty(IContentDescriptionExtended.APPROPRIATE_DEFAULT, encodingMemento.getAppropriateDefault()); - } + if (!encodingMemento.isValid()) { + // note: after setting here, its the mere presence of + // IContentDescriptionExtended.UNSUPPORTED_CHARSET + // in the resource's description that can be used to determine + // if invalid in those cases, the "detected" property contains + // an "appropriate default" to use. + description.setProperty(IContentDescriptionExtended.UNSUPPORTED_CHARSET, encodingMemento.getInvalidEncoding()); + description.setProperty(IContentDescriptionExtended.APPROPRIATE_DEFAULT, encodingMemento.getAppropriateDefault()); + } - Object detectedCharset = encodingMemento.getDetectedCharsetName(); - Object javaCharset = encodingMemento.getJavaCharsetName(); + Object detectedCharset = encodingMemento.getDetectedCharsetName(); + Object javaCharset = encodingMemento.getJavaCharsetName(); - // we always include detected, if its different than java - handleDetectedSpecialCase(description, detectedCharset, javaCharset); + // we always include detected, if its different than java + handleDetectedSpecialCase(description, detectedCharset, javaCharset); - if (javaCharset != null) { - Object existingCharset = description.getProperty(IContentDescription.CHARSET); - if (javaCharset.equals(existingCharset)) { - handleDetectedSpecialCase(description, detectedCharset, javaCharset); - } else { - // we may need to add what we found, but only need to add - // if different from default.the - Object defaultCharset = getDetector().getSpecDefaultEncoding(); - if (defaultCharset != null) { - if (!defaultCharset.equals(javaCharset)) { + if (javaCharset != null) { + Object existingCharset = description.getProperty(IContentDescription.CHARSET); + if (javaCharset.equals(existingCharset)) { + handleDetectedSpecialCase(description, detectedCharset, javaCharset); + } + else { + // we may need to add what we found, but only need to add + // if different from the default + Object defaultCharset = getDetector().getSpecDefaultEncoding(); + if (defaultCharset != null) { + if (!defaultCharset.equals(javaCharset)) { + description.setProperty(IContentDescription.CHARSET, javaCharset); + } + } + else { + // assuming if there is no spec default, we always + // need to add. + // TODO: this is probably a dead branch in current + // code, should re-examine for removal. description.setProperty(IContentDescription.CHARSET, javaCharset); } - } else { - // assuming if there is no spec default, we always need to - // add, I'm assuming - description.setProperty(IContentDescription.CHARSET, javaCharset); } } } - // avoid adding anything if not absolutly needed, since always - // "cached" per session - // description.setProperty(IContentDescriptionExtended.ENCODING_MEMENTO, - // encodingMemento); + returnResult = determineValidity(detector, returnResult); + return returnResult; + } + + private int determineValidity(IResourceCharsetDetector detector, int returnResult) { + // we always expect XMLResourceEncodingDetector, but just to make safe + // cast. + if (detector instanceof XMLResourceEncodingDetector) { + XMLResourceEncodingDetector xmlResourceDetector = (XMLResourceEncodingDetector) detector; + if (xmlResourceDetector.isDeclDetected()) { + if (restrictedMode ) { + // if there is no initial whitespace, then platform's + // default one will do. + if (xmlResourceDetector.hasInitialWhiteSpace()) { + returnResult = IContentDescriber.VALID; + } + } + else { + returnResult = IContentDescriber.VALID; + } + } + } + return returnResult; } }
diff --git a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/XMLResourceEncodingDetector.java b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/XMLResourceEncodingDetector.java index c010026..fdefa02 100644 --- a/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/XMLResourceEncodingDetector.java +++ b/bundles/org.eclipse.wst.xml.core/src/org/eclipse/wst/xml/core/internal/contenttype/XMLResourceEncodingDetector.java
@@ -20,6 +20,8 @@ public class XMLResourceEncodingDetector extends AbstractResourceEncodingDetector implements IResourceCharsetDetector { private XMLHeadTokenizer fTokenizer; + private boolean fDeclDetected = false; + private boolean fInitialWhiteSpace = false; private boolean canHandleAsUnicodeStream(String tokenType) { boolean canHandleAsUnicodeStream = false; @@ -34,7 +36,8 @@ canHandleAsUnicodeStream = true; String enc = "UTF-16BE"; //$NON-NLS-1$ createEncodingMemento(enc, EncodingMemento.DETECTED_STANDARD_UNICODE_BYTES); - } else if (tokenType == EncodingParserConstants.UTF16LE) { + } + else if (tokenType == EncodingParserConstants.UTF16LE) { canHandleAsUnicodeStream = true; String enc = "UTF-16"; //$NON-NLS-1$ createEncodingMemento(enc, EncodingMemento.DETECTED_STANDARD_UNICODE_BYTES); @@ -54,9 +57,9 @@ private XMLHeadTokenizer getTokenizer() { // TODO: need to work on 'reset' in tokenizer, so new instance isn't // always needed - //if (fTokenizer == null) { + // if (fTokenizer == null) { fTokenizer = new XMLHeadTokenizer(); - //} + // } return fTokenizer; } @@ -75,9 +78,24 @@ do { token = tokenizer.getNextToken(); tokenType = token.getType(); + + // handle xml content type detection + if (tokenType == XMLHeadTokenizerConstants.XMLDeclStart) { + fDeclDetected = true; + String declText = token.getText(); + if (declText.startsWith("<?")) { + fInitialWhiteSpace = false; + } + else { + fInitialWhiteSpace = true; + } + } + + // handle encoding detection if (canHandleAsUnicodeStream(tokenType)) { // side effect of canHandle is to create appropriate memento - } else { + } + else { if (tokenType == XMLHeadTokenizerConstants.XMLDelEncoding) { if (tokenizer.hasMoreTokens()) { token = tokenizer.getNextToken(); @@ -87,13 +105,21 @@ if (enc != null && enc.length() > 0) { createEncodingMemento(enc, EncodingMemento.FOUND_ENCODING_IN_CONTENT); } - } } } } - } while (tokenizer.hasMoreTokens()); + } + while (tokenizer.hasMoreTokens()); } + public boolean isDeclDetected() { + return fDeclDetected; + } + + public boolean hasInitialWhiteSpace() { + return fInitialWhiteSpace; + } + }