Use Fork/Join framework for lucene search and index
Change-Id: I024fa0dc09f7443a0079155f0670a839360960ad
Signed-off-by: Dawid Pakuła <zulus@w3des.net>
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/IndexContainer.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/IndexContainer.java
index 2126fff..6b966b1 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/IndexContainer.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/IndexContainer.java
@@ -16,8 +16,9 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.index.ConcurrentMergeScheduler;
@@ -28,13 +29,12 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
-import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.SingleInstanceLockFactory;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
-import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.dltk.core.index.lucene.LucenePlugin;
@@ -129,7 +129,8 @@
private IndexWriter createWriter(Path path) throws IOException {
- Directory indexDir = FSDirectory.open(path);
+ Directory indexDir = FSDirectory.open(path,
+ new SingleInstanceLockFactory());
purgeLocks(path);
IndexWriterConfig config = new IndexWriterConfig(new SimpleAnalyzer());
config.setUseCompoundFile(false);
@@ -138,7 +139,6 @@
config.setMergeScheduler(mergeScheduler);
config.setOpenMode(OpenMode.CREATE_OR_APPEND);
config.setCommitOnClose(false);
- config.setCheckPendingFlushUpdate(false);
return new IndexWriter(indexDir, config);
}
@@ -171,14 +171,21 @@
return fTimestampsWriter;
}
- public synchronized SearcherManager getTimestampsSearcher() {
+ public SearcherManager getTimestampsSearcher() {
try {
if (fTimestampsSearcher == null) {
- fTimestampsSearcher = new SearcherManager(getTimestampsWriter(),
- true, false, new SearcherFactory());
+ synchronized (this) {
+ if (fTimestampsSearcher == null) {
+ fTimestampsSearcher = new SearcherManager(
+ getTimestampsWriter(), true, false,
+ new SearcherFactory());
+ } else {
+ fTimestampsSearcher.maybeRefresh();
+ }
+ }
+
}
- // Try to achieve the up-to-date index state
- fTimestampsSearcher.maybeRefresh();
+
} catch (IOException e) {
Logger.logException(e);
}
@@ -193,32 +200,41 @@
public IndexWriter getIndexWriter(IndexType dataType, int elementType) {
IndexWriter writer = fIndexWriters.get(dataType).get(elementType);
if (writer == null) {
- Path writerPath = getPath(dataType, elementType);
- writer = getWriter(writerPath);
- fIndexWriters.get(dataType).put(elementType, writer);
- fIndexSearchers.get(dataType).put(elementType, null);
+ synchronized (this) {
+ writer = fIndexWriters.get(dataType).get(elementType);
+ if (writer == null) {
+ Path writerPath = getPath(dataType, elementType);
+ writer = getWriter(writerPath);
+ fIndexWriters.get(dataType).put(elementType, writer);
+ fIndexSearchers.get(dataType).put(elementType, null);
+ }
+ }
}
return writer;
}
- public synchronized SearcherManager getIndexSearcher(IndexType dataType,
+ public SearcherManager getIndexSearcher(IndexType dataType,
int elementType) {
SearcherManager searcher = fIndexSearchers.get(dataType)
.get(elementType);
try {
- if (searcher != null) {
- try {
- searcher.maybeRefresh();
- } catch (AlreadyClosedException closed) {
- searcher = null;
- }
- }
if (searcher == null) {
- searcher = new SearcherManager(
- FSDirectory.open(getPath(dataType, elementType)),
- new SearcherFactory());
- fIndexSearchers.get(dataType).put(elementType, searcher);
+ synchronized (this) {
+ searcher = fIndexSearchers.get(dataType).get(elementType);
+ if (searcher == null) {
+ searcher = new SearcherManager(
+ getIndexWriter(dataType, elementType),
+ new SearcherFactory());
+ fIndexSearchers.get(dataType).put(elementType,
+ searcher);
+ } else {
+ searcher.maybeRefresh();
+ }
+
+ }
+ } else {
+ searcher.maybeRefresh();
}
} catch (IndexNotFoundException e) {
@@ -290,33 +306,27 @@
return false;
}
- void commit(IProgressMonitor monitor) {
- int ticks = 1;
- for (Map<?, ?> dataWriters : fIndexWriters.values()) {
- ticks += dataWriters.size();
- }
- SubMonitor subMonitor = SubMonitor.convert(monitor, ticks);
- try {
- for (Entry<IndexType, Map<Integer, IndexWriter>> entry : fIndexWriters
- .entrySet()) {
- Map<Integer, IndexWriter> dataWriters = entry.getValue();
- for (Entry<Integer, IndexWriter> writerEntry : dataWriters
- .entrySet()) {
- IndexWriter writer = writerEntry.getValue();
- if (writer != null && !subMonitor.isCanceled()) {
- writer.commit();
- subMonitor.worked(1);
+ void commit() {
+ List<IndexWriter> writers = new LinkedList<>();
+ synchronized (this) {
+ for (Map<Integer, IndexWriter> dataWriters : fIndexWriters
+ .values()) {
+ for (IndexWriter writer : dataWriters.values()) {
+ if (writer != null) {
+ writers.add(writer);
}
}
}
- if (fTimestampsWriter != null && !subMonitor.isCanceled()) {
- fTimestampsWriter.commit();
- subMonitor.worked(1);
+ }
+ try {
+ for (IndexWriter writer : writers) {
+ writer.commit();
}
- subMonitor.done();
+ if (fTimestampsWriter != null) {
+ fTimestampsWriter.commit();
+ }
} catch (IOException e) {
Logger.logException(e);
}
}
-
}
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneManager.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneManager.java
index f1dfa8b..e73a96e 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneManager.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneManager.java
@@ -19,12 +19,14 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ForkJoinTask;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.SearcherManager;
@@ -95,16 +97,24 @@
SubMonitor subMonitor = SubMonitor.convert(monitor,
containersNumber);
try {
- for (IndexContainer indexContainer : dirtyContainers) {
- if (!monitor.isCanceled()) {
- // Commit index data without merging deletions (better
- // performance)
- indexContainer.commit(subMonitor.newChild(1));
- }
+ if (subMonitor.isCanceled()) {
+ return Status.CANCEL_STATUS;
}
- monitor.done();
+ List<ForkJoinTask> tasks = new LinkedList<>();
+ if (dirtyContainers.size() == 1) {
+ dirtyContainers.get(0).commit();
+ } else {
+ for (IndexContainer indexContainer : dirtyContainers) {
+ tasks.add(ForkJoinTask.adapt(() -> {
+ indexContainer.commit();
+ }));
+ }
+ ForkJoinTask.invokeAll(tasks);
+ }
} catch (Exception e) {
Logger.logException(e);
+ } finally {
+ subMonitor.done();
}
return Status.OK_STATUS;
}
@@ -292,14 +302,15 @@
return fIndexContainers.get(containerId);
}
- private synchronized void deleteIndexContainer(String container,
- boolean wait) {
- String containerId = (String) fContainerMappings.remove(container);
- if (containerId != null) {
- IndexContainer containerEntry = fIndexContainers
- .remove(containerId);
- saveMappings();
- containerEntry.delete(wait);
+ private void deleteIndexContainer(String container, boolean wait) {
+ synchronized (fContainerMappings) {
+ String containerId = (String) fContainerMappings.remove(container);
+ if (containerId != null) {
+ IndexContainer containerEntry = fIndexContainers
+ .remove(containerId);
+ saveMappings();
+ containerEntry.delete(wait);
+ }
}
}
@@ -455,5 +466,4 @@
}
indexRoot.toFile().mkdir();
}
-
}
diff --git a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java
index a5ce77d..30b5806 100644
--- a/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java
+++ b/core/plugins/org.eclipse.dltk.core.index.lucene/src/org/eclipse/dltk/internal/core/index/lucene/LuceneSearchEngine.java
@@ -30,9 +30,13 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ForkJoinTask;
+import java.util.concurrent.RecursiveTask;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.LeafReader;
@@ -121,13 +125,15 @@
private Map<String, BinaryDocValues> fDocBinaryValues;
private String fContainer;
private int fElementType;
- private List<SearchMatch> fResult;
+ private List<SearchMatch> fResult = new LinkedList<>();
- public ResultsCollector(String container, int elementType,
- List<SearchMatch> result) {
+ public ResultsCollector(String container, int elementType) {
this.fContainer = container;
this.fElementType = elementType;
- this.fResult = result;
+ }
+
+ public List<SearchMatch> getfResult() {
+ return fResult;
}
@Override
@@ -235,24 +241,38 @@
|| searchFor == SearchFor.ALL_OCCURRENCES;
boolean searchForRefs = searchFor == SearchFor.REFERENCES
|| searchFor == SearchFor.ALL_OCCURRENCES;
+
+ List<SearchTask> tasks = new LinkedList<>();
+ List<String> containers = SearchScope.getContainers(scope);
+ List<String> scripts = SearchScope.getScripts(scope);
+ final SearchMatchHandler searchMatchHandler = new SearchMatchHandler(
+ scope, requestor);
if (searchForRefs) {
- doSearch(elementType, qualifier, elementName, parent, trueFlags,
- falseFlags, limit, true, matchRule, scope, requestor,
- monitor);
+ for (String container : containers) {
+ tasks.add(new SearchTask(elementType, qualifier, elementName,
+ parent, trueFlags, falseFlags, true, matchRule, scripts,
+ container));
+ }
+ ForkJoinTask.invokeAll(tasks).stream().forEach(t -> t.join()
+ .stream().forEach(m -> searchMatchHandler.handle(m, true)));
+
}
if (searchForDecls) {
- doSearch(elementType, qualifier, elementName, parent, trueFlags,
- falseFlags, limit, false, matchRule, scope, requestor,
- monitor);
+ for (String container : containers) {
+ tasks.add(new SearchTask(elementType, qualifier, elementName,
+ parent, trueFlags, falseFlags, false, matchRule,
+ scripts, container));
+ }
+ ForkJoinTask.invokeAll(tasks).stream().forEach(t -> t.join()
+ .stream().forEach(m -> searchMatchHandler.handle(m, true)));
}
}
private Query createQuery(final String elementName, final String qualifier,
final String parent, final int trueFlags, final int falseFlags,
final boolean searchForRefs, MatchRule matchRule,
- IDLTKSearchScope scope) {
+ List<String> scripts) {
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
- List<String> scripts = SearchScope.getScripts(scope);
if (!scripts.isEmpty()) {
BooleanQuery.Builder scriptQueryBuilder = new BooleanQuery.Builder();
for (String script : scripts) {
@@ -297,35 +317,58 @@
return query.clauses().isEmpty() ? null : query;
}
- private void doSearch(final int elementType, String qualifier,
- String elementName, String parent, final int trueFlags,
- final int falseFlags, int limit, final boolean searchForRefs,
- MatchRule matchRule, IDLTKSearchScope scope,
- ISearchRequestor requestor, IProgressMonitor monitor) {
- Query query = createQuery(elementName, qualifier, parent, trueFlags,
- falseFlags, searchForRefs, matchRule, scope);
- IndexSearcher indexSearcher = null;
- final SearchMatchHandler searchMatchHandler = new SearchMatchHandler(
- scope, requestor);
- List<SearchMatch> results = new ArrayList<>();
- for (String container : SearchScope.getContainers(scope)) {
+ private class SearchTask extends RecursiveTask<List<SearchMatch>> {
+ int elementType;
+ String qualifier;
+ String elementName;
+ String parent;
+ int trueFlags;
+ int falseFlags;
+ boolean searchForRefs;
+ MatchRule matchRule;
+ List<String> scripts;
+ String container;
+
+ private SearchTask(int elementType, String qualifier,
+ String elementName, String parent, int trueFlags,
+ final int falseFlags, boolean searchForRefs,
+ MatchRule matchRule, List<String> scripts, String container) {
+ this.elementType = elementType;
+ this.qualifier = qualifier;
+ this.elementName = elementName;
+ this.parent = parent;
+ this.trueFlags = trueFlags;
+ this.falseFlags = falseFlags;
+ this.searchForRefs = searchForRefs;
+ this.matchRule = matchRule;
+ this.scripts = scripts;
+ this.container = container;
+ }
+
+ @Override
+ protected List<SearchMatch> compute() {
SearcherManager searcherManager = LuceneManager.INSTANCE
.findIndexSearcher(container,
searchForRefs ? IndexType.REFERENCES
: IndexType.DECLARATIONS,
elementType);
if (searcherManager == null) {
- continue;
+ return Collections.emptyList();
}
+ IndexSearcher indexSearcher = null;
try {
indexSearcher = searcherManager.acquire();
+ Query query = createQuery(elementName, qualifier, parent,
+ trueFlags, falseFlags, searchForRefs, matchRule,
+ scripts);
ResultsCollector collector = new ResultsCollector(container,
- elementType, results);
+ elementType);
if (query != null) {
indexSearcher.search(query, collector);
} else {
indexSearcher.search(new MatchAllDocsQuery(), collector);
}
+ return collector.getfResult();
} catch (IOException e) {
Logger.logException(e);
} finally {
@@ -337,10 +380,7 @@
}
}
}
- }
- // Pass results to entity handler
- for (SearchMatch result : results) {
- searchMatchHandler.handle(result, searchForRefs);
+ return Collections.emptyList();
}
}
