Bug 530723 - Upgrade lucene indexer to 7.1
Change-Id: I5fe6a33d0ca9f55ebd300fbff9dfc72117fe57f0
Signed-off-by: Dawid Pakuła <zulus@w3des.net>
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/META-INF/MANIFEST.MF b/core/plugins/org.eclipse.dltk.core.index.lucene/META-INF/MANIFEST.MF
index 94ea1e3..a8f96ac 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/META-INF/MANIFEST.MF
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/META-INF/MANIFEST.MF
@@ -9,9 +9,9 @@
Require-Bundle: org.eclipse.core.runtime,
org.eclipse.core.resources,
org.eclipse.dltk.core,
- org.apache.lucene.core;bundle-version="[6.1.0,7.0.0)",
- org.apache.lucene.misc;bundle-version="[6.1.0,7.0.0)",
- org.apache.lucene.analyzers-common;bundle-version="[6.1.0,7.0.0)"
+ org.apache.lucene.core;bundle-version="[7.1.0,7.2.0)",
+ org.apache.lucene.misc;bundle-version="[7.1.0,7.2.0)",
+ org.apache.lucene.analyzers-common;bundle-version="[7.1.0,7.2.0)"
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
Bundle-ActivationPolicy: lazy
Export-Package: org.eclipse.dltk.core.index.lucene
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/BitFlagsQuery.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/BitFlagsQuery.java
index 6df98d7..3de57ac 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/BitFlagsQuery.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/BitFlagsQuery.java
@@ -12,22 +12,16 @@
import java.io.IOException;
import java.text.MessageFormat;
-import java.util.Set;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.Term;
import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.DocValuesDocIdSet;
-import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.Bits;
import org.eclipse.dltk.ast.Modifiers;
/**
@@ -79,84 +73,39 @@
}
@Override
- public Weight createWeight(IndexSearcher searcher, boolean needsScores)
- throws IOException {
- return new Weight(this) {
- @Override
- public void extractTerms(Set<Term> terms) {
- // Ignore
- }
-
- @Override
- public void normalize(float norm, float topLevelBoost) {
- // Ignore
- }
-
- @Override
- public float getValueForNormalization() throws IOException {
- return 0;
- }
-
- @Override
- public Explanation explain(LeafReaderContext context, int doc)
- throws IOException {
- final Scorer scorer = scorer(context);
- final boolean match = (scorer != null
- && scorer.iterator().advance(doc) == doc);
- if (match) {
- assert scorer.score() == 0;
- return Explanation.match(0, "Match on id" + doc); //$NON-NLS-1$
- } else {
- return Explanation.match(0, "No match on id" + doc); //$NON-NLS-1$
- }
- }
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores,
+ float boost) throws IOException {
+ return new ConstantScoreWeight(this, 10) {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
- final DocIdSet set = getDocIdSet(context,
- context.reader().getLiveDocs());
- if (set == null) {
+
+ NumericDocValues fields = context.reader()
+ .getNumericDocValues(IndexFields.NDV_FLAGS);
+ if (fields == null) {
return null;
}
- final DocIdSetIterator iterator = set.iterator();
- if (iterator == null) {
- return null;
- }
- return new ConstantScoreScorer(this, 0, iterator);
- }
- };
+ TwoPhaseIterator iterator = new TwoPhaseIterator(fields) {
- }
+ @Override
+ public boolean matches() throws IOException {
+ long flags = fields.longValue();
+ if (fTrueFlags != 0 && (flags & fTrueFlags) == 0) {
+ return false;
+ }
+ if (fFalseFlags != 0 && (flags & fFalseFlags) != 0) {
+ return false;
+ }
+ return true;
+ }
- /**
- * Finds and returns matching doc ID set.
- *
- * @param context
- * @param acceptDocs
- * @return matching doc ID set
- * @throws IOException
- */
- protected DocIdSet getDocIdSet(final LeafReaderContext context,
- Bits acceptDocs) throws IOException {
- final NumericDocValues numDocValues = DocValues
- .getNumeric(context.reader(), IndexFields.NDV_FLAGS);
- return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
- @Override
- protected boolean matchDoc(int doc) {
- long flags = numDocValues.get(doc);
- if (fTrueFlags != 0) {
- if ((flags & fTrueFlags) == 0) {
- return false;
+ @Override
+ public float matchCost() {
+ return 2;
}
- }
- if (fFalseFlags != 0) {
- if ((flags & fFalseFlags) != 0) {
- return false;
- }
- }
- return true;
+ };
+ return new ConstantScoreScorer(this, 10, iterator);
}
};
}
-
}
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/DocumentFactory.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/DocumentFactory.java
index f63691d..5db6b1b 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/DocumentFactory.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/DocumentFactory.java
@@ -10,7 +10,23 @@
*******************************************************************************/
package org.eclipse.dltk.internal.core.index.lucene;
-import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.*;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.BDV_DOC;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.BDV_ELEMENT_NAME;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.BDV_METADATA;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.BDV_PARENT;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.BDV_PATH;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.BDV_QUALIFIER;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.F_CC_NAME;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.F_ELEMENT_NAME_LC;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.F_PARENT;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.F_PATH;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.F_QUALIFIER;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.NDV_FLAGS;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.NDV_LENGTH;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.NDV_NAME_LENGTH;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.NDV_NAME_OFFSET;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.NDV_OFFSET;
+import static org.eclipse.dltk.internal.core.index.lucene.IndexFields.NDV_TIMESTAMP;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
@@ -140,7 +156,8 @@
}
}
camelCaseName = camelCaseNameBuf.length() > 0
- ? camelCaseNameBuf.toString() : null;
+ ? camelCaseNameBuf.toString()
+ : null;
addStringEntry(doc, F_CC_NAME, camelCaseName, false);
}
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneIndexer.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneIndexer.java
index c73b608..badda0b 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneIndexer.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneIndexer.java
@@ -14,9 +14,8 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.util.Set;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -50,9 +49,6 @@
private static final class TimestampsCollector implements Collector {
- private static final Set<String> fFields = Collections
- .singleton(IndexFields.F_PATH);
-
private final Map<String, Long> fResult;
public TimestampsCollector(Map<String, Long> result) {
@@ -68,19 +64,22 @@
public LeafCollector getLeafCollector(LeafReaderContext context)
throws IOException {
final LeafReader reader = context.reader();
- final NumericDocValues timestampField = context.reader()
+ final NumericDocValues timestampField = reader
.getNumericDocValues(IndexFields.NDV_TIMESTAMP);
+ final BinaryDocValues pathField = reader
+ .getBinaryDocValues(IndexFields.F_PATH);
return new LeafCollector() {
@Override
public void setScorer(Scorer scorer) throws IOException {
- // ignore
}
@Override
public void collect(int docId) throws IOException {
- Document document = reader.document(docId, fFields);
- fResult.put(document.get(IndexFields.F_PATH),
- timestampField.get(docId));
+ if (timestampField.advanceExact(docId)
+ && pathField.advanceExact(docId)) {
+ fResult.put(pathField.binaryValue().utf8ToString(),
+ timestampField.longValue());
+ }
}
};
}
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java
index 58187b5..8c843a9 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java
@@ -130,7 +130,7 @@
@Override
public boolean needsScores() {
- return true;
+ return false;
}
@Override
@@ -152,9 +152,9 @@
}
}
return new LeafCollector() {
+
@Override
public void setScorer(Scorer scorer) throws IOException {
- // ignore
}
@Override
@@ -179,10 +179,18 @@
getStringValue(BDV_METADATA, docId)));
}
- private int getNumericValue(String field, int docId) {
+ private long getNumericValue(String field, int docId) {
NumericDocValues docValues = fDocNumericValues.get(field);
if (docValues != null) {
- return (int) docValues.get(docId);
+ try {
+ docValues.advanceExact(docId);
+ if (!docValues.advanceExact(docId)) {
+ return 0;
+ }
+ return docValues.longValue();
+ } catch (IOException e) {
+ Logger.logException(e);
+ }
}
return 0;
}
@@ -190,12 +198,21 @@
private String getStringValue(String field, int docId) {
BinaryDocValues docValues = fDocBinaryValues.get(field);
if (docValues != null) {
- BytesRef bytesRef = docValues.get(docId);
- if (bytesRef.length > 0)
- return bytesRef.utf8ToString();
+ try {
+ if (!docValues.advanceExact(docId)) {
+ return null;
+ }
+ BytesRef bytesRef = docValues.binaryValue();
+ if (bytesRef.length > 0)
+ return bytesRef.utf8ToString();
+ } catch (IOException e) {
+ Logger.logException(e);
+ }
+
}
return null;
}
+
}
@Override
@@ -291,8 +308,9 @@
List<SearchMatch> results = new ArrayList<>();
for (String container : SearchScope.getContainers(scope)) {
SearcherManager searcherManager = LuceneManager.INSTANCE
- .findIndexSearcher(container, searchForRefs
- ? IndexType.REFERENCES : IndexType.DECLARATIONS,
+ .findIndexSearcher(container,
+ searchForRefs ? IndexType.REFERENCES
+ : IndexType.DECLARATIONS,
elementType);
try {
indexSearcher = searcherManager.acquire();
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/SearchMatch.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/SearchMatch.java
index 96194c3..441cb9d 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/SearchMatch.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/SearchMatch.java
@@ -48,18 +48,18 @@
* @param doc
* @param metadata
*/
- public SearchMatch(String container, int elementType, int offset,
- int length, int nameOffset, int nameLength, int flags,
+ public SearchMatch(String container, long elementType, long offset,
+ long length, long nameOffset, long nameLength, long flags,
String elementName, String path, String parent, String qualifier,
String doc, String metadata) {
super();
this.container = container;
- this.elementType = elementType;
- this.offset = offset;
- this.length = length;
- this.nameOffset = nameOffset;
- this.nameLength = nameLength;
- this.flags = flags;
+ this.elementType = (int) elementType;
+ this.offset = (int) offset;
+ this.length = (int) length;
+ this.nameOffset = (int) nameOffset;
+ this.nameLength = (int) nameLength;
+ this.flags = (int) flags;
this.elementName = elementName;
this.path = path;
this.parent = parent;
diff --git a/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Neon-4.6.target b/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Neon-4.6.target
index d26356f..72ac687 100644
--- a/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Neon-4.6.target
+++ b/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Neon-4.6.target
@@ -9,12 +9,12 @@
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
<unit id="com.google.guava" version="21.0.0.v20170206-1425"/>
<unit id="com.google.guava.source" version="21.0.0.v20170206-1425"/>
-<unit id="org.apache.lucene.analyzers-common" version="6.1.0.v20161115-1612"/>
-<unit id="org.apache.lucene.analyzers-common.source" version="6.1.0.v20161115-1612"/>
-<unit id="org.apache.lucene.core" version="6.1.0.v20170814-1820"/>
-<unit id="org.apache.lucene.core.source" version="6.1.0.v20170814-1820"/>
-<unit id="org.apache.lucene.misc" version="6.1.0.v20161115-1612"/>
-<unit id="org.apache.lucene.misc.source" version="6.1.0.v20161115-1612"/>
+<unit id="org.apache.lucene.analyzers-common" version="7.1.0.v20180122-2126"/>
+<unit id="org.apache.lucene.analyzers-common.source" version="7.1.0.v20180122-2126"/>
+<unit id="org.apache.lucene.core" version="7.1.0.v20171214-1510"/>
+<unit id="org.apache.lucene.core.source" version="7.1.0.v20171214-1510"/>
+<unit id="org.apache.lucene.misc" version="7.1.0.v20180220-1923"/>
+<unit id="org.apache.lucene.misc.source" version="7.1.0.v20180220-1923"/>
<unit id="org.h2" version="1.3.168.v201212121212"/>
<unit id="org.h2.source" version="1.3.168.v201212121212"/>
<unit id="org.hamcrest" version="0.0.0"/>
@@ -27,7 +27,7 @@
<unit id="org.junit.source" version="4.12.0.v201504281640"/>
<unit id="org.mockito" version="0.0.0"/>
<unit id="org.objenesis" version="0.0.0"/>
-<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/M20170818183741/repository/"/>
+<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20180302171354/repository/"/>
</location>
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
<unit id="org.eclipse.emf.sdk.feature.group" version="0.0.0"/>
diff --git a/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Oxygen-4.7.target b/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Oxygen-4.7.target
index 71b1383..3e297b9 100644
--- a/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Oxygen-4.7.target
+++ b/core/releng/org.eclipse.dltk.core.targetplatform/DLTK-Oxygen-4.7.target
@@ -9,12 +9,12 @@
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
<unit id="com.google.guava" version="21.0.0.v20170206-1425"/>
<unit id="com.google.guava.source" version="21.0.0.v20170206-1425"/>
-<unit id="org.apache.lucene.analyzers-common" version="6.1.0.v20161115-1612"/>
-<unit id="org.apache.lucene.analyzers-common.source" version="6.1.0.v20161115-1612"/>
-<unit id="org.apache.lucene.core" version="6.1.0.v20170814-1820"/>
-<unit id="org.apache.lucene.core.source" version="6.1.0.v20170814-1820"/>
-<unit id="org.apache.lucene.misc" version="6.1.0.v20161115-1612"/>
-<unit id="org.apache.lucene.misc.source" version="6.1.0.v20161115-1612"/>
+<unit id="org.apache.lucene.analyzers-common" version="7.1.0.v20180122-2126"/>
+<unit id="org.apache.lucene.analyzers-common.source" version="7.1.0.v20180122-2126"/>
+<unit id="org.apache.lucene.core" version="7.1.0.v20171214-1510"/>
+<unit id="org.apache.lucene.core.source" version="7.1.0.v20171214-1510"/>
+<unit id="org.apache.lucene.misc" version="7.1.0.v20180220-1923"/>
+<unit id="org.apache.lucene.misc.source" version="7.1.0.v20180220-1923"/>
<unit id="org.h2" version="1.3.168.v201212121212"/>
<unit id="org.h2.source" version="1.3.168.v201212121212"/>
<unit id="org.hamcrest" version="0.0.0"/>
@@ -27,7 +27,7 @@
<unit id="org.junit.source" version="4.12.0.v201504281640"/>
<unit id="org.mockito" version="0.0.0"/>
<unit id="org.objenesis" version="0.0.0"/>
-<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/M20170818183741/repository/"/>
+<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20180302171354/repository/"/>
</location>
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
<unit id="org.eclipse.emf.sdk.feature.group" version="0.0.0"/>
