| /******************************************************************************* |
| * Copyright (c) 2000, 2017 IBM Corporation and others. |
| * All rights reserved. This program and the accompanying materials |
| * are made available under the terms of the Eclipse Public License v1.0 |
| * which accompanies this distribution, and is available at |
| * http://www.eclipse.org/legal/epl-v10.html |
| * |
| * Contributors: |
| * IBM Corporation - initial API and implementation |
| * Holger Voormann - fix for bug 426785 (http://eclip.se/426785) |
| * Alexander Kurtakov - Bug 460787 |
| * Sopot Cela - Bug 466829 |
| *******************************************************************************/ |
| package org.eclipse.help.internal.search; |
| |
| import java.io.File; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.RandomAccessFile; |
| import java.net.MalformedURLException; |
| import java.net.URL; |
| import java.nio.channels.FileLock; |
| import java.nio.channels.OverlappingFileLockException; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.zip.ZipEntry; |
| import java.util.zip.ZipInputStream; |
| |
| import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.document.StoredField; |
| import org.apache.lucene.document.StringField; |
| import org.apache.lucene.index.DirectoryReader; |
| import org.apache.lucene.index.IndexFormatTooOldException; |
| import org.apache.lucene.index.IndexNotFoundException; |
| import org.apache.lucene.index.IndexReader; |
| import org.apache.lucene.index.IndexWriter; |
| import org.apache.lucene.index.IndexWriterConfig; |
| import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| import org.apache.lucene.index.LeafReaderContext; |
| import org.apache.lucene.index.LogByteSizeMergePolicy; |
| import org.apache.lucene.index.LogMergePolicy; |
| import org.apache.lucene.index.PostingsEnum; |
| import org.apache.lucene.index.Term; |
| import org.apache.lucene.search.BooleanQuery; |
| import org.apache.lucene.search.IndexSearcher; |
| import org.apache.lucene.search.Query; |
| import org.apache.lucene.search.TopDocs; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.store.NIOFSDirectory; |
| import org.eclipse.core.runtime.IExtension; |
| import org.eclipse.core.runtime.IExtensionPoint; |
| import org.eclipse.core.runtime.IExtensionRegistry; |
| import org.eclipse.core.runtime.IProgressMonitor; |
| import org.eclipse.core.runtime.IStatus; |
| import org.eclipse.core.runtime.InvalidRegistryObjectException; |
| import org.eclipse.core.runtime.OperationCanceledException; |
| import org.eclipse.core.runtime.Platform; |
| import org.eclipse.core.runtime.Status; |
| import org.eclipse.help.internal.HelpPlugin; |
| import org.eclipse.help.internal.base.BaseHelpSystem; |
| import org.eclipse.help.internal.base.HelpBasePlugin; |
| import org.eclipse.help.internal.base.util.HelpProperties; |
| import org.eclipse.help.internal.protocols.HelpURLConnection; |
| import org.eclipse.help.internal.protocols.HelpURLStreamHandler; |
| import org.eclipse.help.internal.toc.TocFileProvider; |
| import org.eclipse.help.internal.toc.TocManager; |
| import org.eclipse.help.internal.util.ResourceLocator; |
| import org.eclipse.help.search.IHelpSearchIndex; |
| import org.eclipse.help.search.ISearchDocument; |
| import org.eclipse.help.search.SearchParticipant; |
| import org.osgi.framework.Bundle; |
| import org.osgi.framework.Constants; |
| import org.osgi.framework.Version; |
| |
| /** |
| * Text search index. Documents added to this index can than be searched against a search query. |
| */ |
| public class SearchIndex implements IHelpSearchIndex { |
| |
| private IndexReader ir; |
| |
| private IndexWriter iw; |
| |
| private File indexDir; |
| |
| private Directory luceneDirectory; |
| |
| private String locale; |
| |
| private String relativePath; |
| |
| private TocManager tocManager; |
| |
| private AnalyzerDescriptor analyzerDescriptor; |
| |
| private PluginVersionInfo docPlugins; |
| |
| // table of all document names, used during indexing batches |
| private HelpProperties indexedDocs; |
| |
| public static final String INDEXED_CONTRIBUTION_INFO_FILE = "indexed_contributions"; //$NON-NLS-1$ |
| |
| public static final String INDEXED_DOCS_FILE = "indexed_docs"; //$NON-NLS-1$ |
| |
| public static final String DEPENDENCIES_VERSION_FILENAME = "indexed_dependencies"; //$NON-NLS-1$ |
| |
| public static final String DEPENDENCIES_KEY_LUCENE = "lucene"; //$NON-NLS-1$ |
| |
| public static final String DEPENDENCIES_KEY_ANALYZER = "analyzer"; //$NON-NLS-1$ |
| |
| private static final String LUCENE_BUNDLE_ID = "org.apache.lucene.core"; //$NON-NLS-1$ |
| |
| private static final String FIELD_NAME = "name"; //$NON-NLS-1$ |
| |
| private static final String FIELD_INDEX_ID = "index_path"; //$NON-NLS-1$ |
| |
| private File inconsistencyFile; |
| |
| private HTMLSearchParticipant htmlSearchParticipant; |
| |
| private IndexSearcher searcher; |
| |
| private Object searcherCreateLock = new Object(); |
| |
| private HelpProperties dependencies; |
| |
| private volatile boolean closed = false; |
| |
| // Collection of searches occuring now |
| private Collection<Thread> searches = new ArrayList<>(); |
| |
| private FileLock lock; |
| private RandomAccessFile raf = null; |
| |
| /** |
| * Constructor. |
| * |
| * @param locale |
| * the locale this index uses |
| * @param analyzerDesc |
| * the analyzer used to index |
| */ |
| public SearchIndex(String locale, AnalyzerDescriptor analyzerDesc, TocManager tocManager) { |
| this(new File(HelpBasePlugin.getConfigurationDirectory(), "index/" + locale), //$NON-NLS-1$ |
| locale, analyzerDesc, tocManager, null); |
| } |
| |
| /** |
| * Alternative constructor that provides index directory. |
| * |
| * @param indexDir |
| * @param locale |
| * @param analyzerDesc |
| * @param tocManager |
| * @since 3.1 |
| */ |
| |
| public SearchIndex(File indexDir, String locale, AnalyzerDescriptor analyzerDesc, TocManager tocManager, |
| String relativePath) { |
| this.locale = locale; |
| this.analyzerDescriptor = analyzerDesc; |
| this.tocManager = tocManager; |
| this.indexDir = indexDir; |
| |
| this.relativePath = relativePath; |
| // System.out.println("Index for a relative path: "+relativePath); |
| inconsistencyFile = new File(indexDir.getParentFile(), locale + ".inconsistent"); //$NON-NLS-1$ |
| htmlSearchParticipant = new HTMLSearchParticipant(indexDir.getAbsolutePath()); |
| try { |
| luceneDirectory = new NIOFSDirectory(indexDir.toPath()); |
| } catch (IOException e) { |
| } |
| if (!exists()) { |
| try { |
| if (tryLock()) { |
| // don't block or unzip when another instance is indexing |
| try { |
| unzipProductIndex(); |
| } finally { |
| releaseLock(); |
| } |
| } |
| } catch (OverlappingFileLockException ofle) { |
| // another thread in this process is unzipping |
| // should never be here - one index instance per locale exists |
| // in vm |
| } |
| } |
| |
| try { |
| DirectoryReader.open(luceneDirectory); |
| } catch (IndexFormatTooOldException | IndexNotFoundException | IllegalArgumentException e) { |
| deleteDir(indexDir); |
| indexDir.delete(); |
| } catch (IOException e) { |
| e.printStackTrace(); |
| } |
| |
| } |
| |
| private void deleteDir(File indexDir) { |
| File[] files = indexDir.listFiles(); |
| if(files == null) { |
| files = new File[0]; |
| } |
| for (File file : files) { |
| if (file.isDirectory()) |
| deleteDir(file); |
| file.delete(); |
| } |
| } |
| |
| /** |
| * Indexes one document from a stream. Index has to be open and close outside of this method |
| * |
| * @param name |
| * the document identifier (could be a URL) |
| * @param url |
| * the URL of the document |
| * @return IStatus |
| */ |
| public IStatus addDocument(String name, URL url) { |
| try { |
| Document doc = new Document(); |
| doc.add(new StringField(FIELD_NAME, name, Field.Store.YES)); |
| addExtraFields(doc); |
| String pluginId = LocalSearchManager.getPluginId(name); |
| if (relativePath != null) { |
| doc.add(new StringField(FIELD_INDEX_ID, relativePath, Field.Store.YES)); |
| } |
| // check for the explicit search participant. |
| SearchParticipant participant = null; |
| HelpURLConnection urlc = new HelpURLConnection(url); |
| String id = urlc.getValue("id"); //$NON-NLS-1$ |
| String pid = urlc.getValue("participantId"); //$NON-NLS-1$ |
| if (pid != null) |
| participant = BaseHelpSystem.getLocalSearchManager().getGlobalParticipant(pid); |
| // NEW: check for file extension-based search participant; |
| if (participant == null) |
| participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name); |
| if (participant != null) { |
| IStatus status = participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); |
| if (status.getSeverity() == IStatus.OK) { |
| String filters = doc.get("filters"); //$NON-NLS-1$ |
| indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$ |
| if (id != null) |
| doc.add(new StoredField("id", id)); //$NON-NLS-1$ |
| if (pid != null) |
| doc.add(new StoredField("participantId", pid)); //$NON-NLS-1$ |
| iw.addDocument(doc); |
| } |
| return status; |
| } |
| // default to html |
| IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); |
| if (status.getSeverity() == IStatus.OK) { |
| String filters = doc.get("filters"); //$NON-NLS-1$ |
| indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$ |
| iw.addDocument(doc); |
| } |
| return status; |
| } catch (IOException e) { |
| return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, |
| "IO exception occurred while adding document " + name //$NON-NLS-1$ |
| + " to index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ |
| e); |
| } |
| catch (Exception e) { |
| return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, |
| "An unexpected internal error occurred while adding document " //$NON-NLS-1$ |
| + name + " to index " + indexDir.getAbsolutePath() //$NON-NLS-1$ |
| + ".", e); //$NON-NLS-1$ |
| } |
| } |
| |
| /** |
| * Add any extra fields that need to be added to this document. Subclasses |
| * should override to add more fields. |
| * |
| * @param doc the document to add fields to |
| */ |
| protected void addExtraFields(Document doc) { |
| } |
| |
| /** |
| * Starts additions. To be called before adding documents. |
| */ |
| @SuppressWarnings("resource") |
| public synchronized boolean beginAddBatch(boolean firstOperation) { |
| try { |
| if (iw != null) { |
| iw.close(); |
| } |
| boolean create = false; |
| if (!indexDir.exists() || !isLuceneCompatible() || !isAnalyzerCompatible() |
| || inconsistencyFile.exists() && firstOperation) { |
| create = true; |
| indexDir.mkdirs(); |
| if (!indexDir.exists()) |
| return false; // unable to setup index directory |
| } |
| indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); |
| indexedDocs.restore(); |
| setInconsistent(true); |
| LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 1000000); |
| IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer); |
| writerConfig.setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND); |
| LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); |
| mergePolicy.setMergeFactor(20); |
| writerConfig.setMergePolicy(mergePolicy); |
| iw = new IndexWriter(luceneDirectory, writerConfig); |
| return true; |
| } catch (IOException e) { |
| HelpBasePlugin.logError("Exception occurred in search indexing at beginAddBatch.", e); //$NON-NLS-1$ |
| return false; |
| } |
| } |
| |
| /** |
| * Starts deletions. To be called before deleting documents. |
| */ |
| public synchronized boolean beginDeleteBatch() { |
| try { |
| if (iw != null) { |
| iw.close(); |
| } |
| indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); |
| indexedDocs.restore(); |
| setInconsistent(true); |
| iw = new IndexWriter(luceneDirectory, new IndexWriterConfig(analyzerDescriptor.getAnalyzer())); |
| return true; |
| } catch (IOException e) { |
| HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ |
| return false; |
| } |
| } |
| |
| /** |
| * Starts deletions. To be called before deleting documents. |
| */ |
| public synchronized boolean beginRemoveDuplicatesBatch() { |
| try { |
| if (ir != null) { |
| ir.close(); |
| } |
| ir = DirectoryReader.open(luceneDirectory); |
| if (iw == null) { |
| return beginDeleteBatch(); |
| } |
| return true; |
| } catch (IOException e) { |
| HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ |
| return false; |
| } |
| } |
| |
| /** |
| * Deletes a single document from the index. |
| * |
| * @param name - |
| * document name |
| * @return IStatus |
| */ |
| public IStatus removeDocument(String name) { |
| Term term = new Term(FIELD_NAME, name); |
| try { |
| iw.deleteDocuments(term); |
| indexedDocs.remove(name); |
| } catch (IOException e) { |
| return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, |
| "IO exception occurred while removing document " + name //$NON-NLS-1$ |
| + " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ |
| e); |
| } |
| return Status.OK_STATUS; |
| } |
| |
| /** |
| * Finish additions. To be called after adding documents. |
| */ |
| public synchronized boolean endAddBatch(boolean optimize, boolean lastOperation) { |
| try { |
| if (iw == null) |
| return false; |
| if (optimize) |
| iw.forceMerge(1, true); |
| iw.close(); |
| iw = null; |
| // save the update info: |
| // - all the docs |
| // - plugins (and their version) that were indexed |
| getDocPlugins().save(); |
| saveDependencies(); |
| if (lastOperation) { |
| indexedDocs.save(); |
| indexedDocs = null; |
| setInconsistent(false); |
| } |
| |
| /* |
| * The searcher's index reader has it's stuff in memory so it won't |
| * know about this change. Close it so that it gets reloaded next search. |
| */ |
| if (searcher != null) { |
| searcher.getIndexReader().close(); |
| searcher = null; |
| } |
| return true; |
| } catch (IOException e) { |
| HelpBasePlugin.logError("Exception occurred in search indexing at endAddBatch.", e); //$NON-NLS-1$ |
| return false; |
| } |
| } |
| |
| /** |
| * Finish deletions. To be called after deleting documents. |
| */ |
| public synchronized boolean endDeleteBatch() { |
| try { |
| if (iw == null) |
| return false; |
| iw.close(); |
| iw = null; |
| // save the update info: |
| // - all the docs |
| // - plugins (and their version) that were indexed |
| indexedDocs.save(); |
| indexedDocs = null; |
| getDocPlugins().save(); |
| saveDependencies(); |
| |
| /* |
| * The searcher's index reader has it's stuff in memory so it won't |
| * know about this change. Close it so that it gets reloaded next search. |
| */ |
| if (searcher != null) { |
| searcher.getIndexReader().close(); |
| searcher = null; |
| } |
| return true; |
| } catch (IOException e) { |
| HelpBasePlugin.logError("Exception occurred in search indexing at endDeleteBatch.", e); //$NON-NLS-1$ |
| return false; |
| } |
| } |
| |
| /** |
| * Finish deletions. To be called after deleting documents. |
| */ |
| public synchronized boolean endRemoveDuplicatesBatch() { |
| try { |
| if (ir == null) |
| return false; |
| ir.close(); |
| ir = null; |
| iw.close(); |
| iw = null; |
| // save the update info: |
| // - all the docs |
| // - plugins (and their version) that were indexed |
| indexedDocs.save(); |
| indexedDocs = null; |
| getDocPlugins().save(); |
| saveDependencies(); |
| setInconsistent(false); |
| return true; |
| } catch (IOException e) { |
| HelpBasePlugin.logError("Exception occurred in search indexing at endDeleteBatch.", e); //$NON-NLS-1$ |
| return false; |
| } |
| } |
| |
| /** |
| * If |
| * |
| * @param dirs |
| * @param monitor |
| * @return Map. Keys are /pluginid/href of all merged Docs. Values are null for added document, |
| * or String[] of indexIds with duplicates of the document |
| */ |
| public Map<String, String[]> merge(PluginIndex[] pluginIndexes, IProgressMonitor monitor) { |
| ArrayList<NIOFSDirectory> dirList = new ArrayList<>(pluginIndexes.length); |
| Map<String, String[]> mergedDocs = new HashMap<>(); |
| // Create directories to merge and calculate all documents added |
| // and which are duplicates (to delete later) |
| for (int p = 0; p < pluginIndexes.length; p++) { |
| List<String> indexIds = pluginIndexes[p].getIDs(); |
| List<String> indexPaths = pluginIndexes[p].getPaths(); |
| if (monitor.isCanceled()) { |
| throw new OperationCanceledException(); |
| } |
| |
| for (int i = 0; i < indexPaths.size(); i++) { |
| String indexId = indexIds.get(i); |
| String indexPath = indexPaths.get(i); |
| try { |
| // can't use try-with-resources as 'dir' needs to stay open |
| @SuppressWarnings("resource") |
| NIOFSDirectory dir = new NIOFSDirectory(new File(indexPath).toPath()); |
| dirList.add(dir); |
| } catch (IOException ioe) { |
| HelpBasePlugin |
| .logError( |
| "Help search indexing directory could not be created for directory " + indexPath, ioe); //$NON-NLS-1$ |
| continue; |
| } |
| |
| HelpProperties prebuiltDocs = new HelpProperties(INDEXED_DOCS_FILE, new File(indexPath)); |
| prebuiltDocs.restore(); |
| Set<?> prebuiltHrefs = prebuiltDocs.keySet(); |
| for (Iterator<?> it = prebuiltHrefs.iterator(); it.hasNext();) { |
| String href = (String) it.next(); |
| if (i == 0) { |
| // optimization for first prebuilt index of a plug-in |
| mergedDocs.put(href, null); |
| } else { |
| if (mergedDocs.containsKey(href)) { |
| // this is duplicate |
| String[] dups = mergedDocs.get(href); |
| if (dups == null) { |
| // first duplicate |
| mergedDocs.put(href, new String[] { indexId }); |
| } else { |
| // next duplicate |
| String[] newDups = new String[dups.length + 1]; |
| System.arraycopy(dups, 0, newDups, 0, dups.length); |
| newDups[dups.length] = indexId; |
| mergedDocs.put(href, newDups); |
| } |
| } else { |
| // document does not exist in more specific indexes |
| // for this plugin |
| mergedDocs.put(href, null); |
| } |
| |
| } |
| } |
| } |
| } |
| // perform actual merging |
| for (Iterator<String> it = mergedDocs.keySet().iterator(); it.hasNext();) { |
| indexedDocs.put(it.next(), "0"); //$NON-NLS-1$ |
| } |
| Directory[] luceneDirs = dirList.toArray(new Directory[dirList.size()]); |
| try { |
| iw.addIndexes(luceneDirs); |
| iw.forceMerge(1, true); |
| } catch (IOException ioe) { |
| HelpBasePlugin.logError("Merging search indexes failed.", ioe); //$NON-NLS-1$ |
| return new HashMap<>(); |
| } |
| return mergedDocs; |
| } |
| |
| public IStatus removeDuplicates(String name, String[] index_paths) { |
| |
| try (DirectoryReader ar = DirectoryReader.open(luceneDirectory)) { |
| PostingsEnum hrefDocs = null; |
| PostingsEnum indexDocs = null; |
| Term hrefTerm = new Term(FIELD_NAME, name); |
| for (int i = 0; i < index_paths.length; i++) { |
| Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]); |
| List<LeafReaderContext> leaves = ar.leaves(); |
| for (LeafReaderContext c : leaves) { |
| indexDocs = c.reader().postings(indexTerm); |
| hrefDocs = c.reader().postings(hrefTerm); |
| removeDocuments(hrefDocs, indexDocs); |
| } |
| } |
| } catch (IOException ioe) { |
| return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, |
| "IO exception occurred while removing duplicates of document " + name //$NON-NLS-1$ |
| + " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ |
| ioe); |
| } |
| return Status.OK_STATUS; |
| } |
| |
| /** |
| * Removes documents containing term1 and term2 |
| * |
| * @param doc1 |
| * @param docs2 |
| * @throws IOException |
| */ |
| private void removeDocuments(PostingsEnum doc1, PostingsEnum docs2) throws IOException { |
| if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { |
| return; |
| } |
| if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) { |
| return; |
| } |
| while (true) { |
| if (doc1.docID() < docs2.docID()) { |
| if (doc1.advance(docs2.docID()) == PostingsEnum.NO_MORE_DOCS) { |
| if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { |
| return; |
| } |
| } |
| } else if (doc1.docID() > docs2.docID()) { |
| if (docs2.advance(doc1.docID()) == PostingsEnum.NO_MORE_DOCS) { |
| if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { |
| return; |
| } |
| } |
| } |
| if (doc1.docID() == docs2.docID()) { |
| iw.tryDeleteDocument(ir, doc1.docID()); |
| if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { |
| return; |
| } |
| if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) { |
| return; |
| } |
| } |
| } |
| } |
| |
| /** |
| * Checks if index exists and is usable. |
| * |
| * @return true if index exists |
| */ |
| public boolean exists() { |
| return indexDir.exists() && !isInconsistent(); |
| // assume index exists if directory does |
| } |
| |
| /** |
| * Performs a query search on this index |
| */ |
| public void search(ISearchQuery searchQuery, ISearchHitCollector collector) |
| throws QueryTooComplexException { |
| try { |
| if (closed) |
| return; |
| registerSearch(Thread.currentThread()); |
| if (closed) |
| return; |
| QueryBuilder queryBuilder = new QueryBuilder(searchQuery.getSearchWord(), analyzerDescriptor); |
| Query luceneQuery = queryBuilder.getLuceneQuery(searchQuery.getFieldNames(), searchQuery |
| .isFieldSearch()); |
| if (HelpPlugin.DEBUG_SEARCH) { |
| System.out.println("Search Query: " + luceneQuery.toString()); //$NON-NLS-1$ |
| } |
| String highlightTerms = queryBuilder.gethighlightTerms(); |
| if (luceneQuery != null) { |
| if (searcher == null) { |
| openSearcher(); |
| } |
| TopDocs topDocs = searcher.search(luceneQuery, 1000); |
| collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlightTerms); |
| } |
| } catch (BooleanQuery.TooManyClauses tmc) { |
| collector.addQTCException(new QueryTooComplexException()); |
| } catch (QueryTooComplexException qe) { |
| collector.addQTCException(qe); |
| } catch (Exception e) { |
| HelpBasePlugin.logError("Exception occurred performing search for: " //$NON-NLS-1$ |
| + searchQuery.getSearchWord() + ".", e); //$NON-NLS-1$ |
| } finally { |
| unregisterSearch(Thread.currentThread()); |
| } |
| } |
| |
| @Override |
| public String getLocale() { |
| return locale; |
| } |
| |
| /** |
| * Returns the list of all the plugins in this session that have declared a help contribution. |
| */ |
| public PluginVersionInfo getDocPlugins() { |
| if (docPlugins == null) { |
| Set<String> totalIds = new HashSet<>(); |
| IExtensionRegistry registry = Platform.getExtensionRegistry(); |
| IExtensionPoint extensionPoint = registry.getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC); |
| IExtension[] extensions = extensionPoint.getExtensions(); |
| for (int i=0;i<extensions.length;++i) { |
| try { |
| totalIds.add(extensions[i].getContributor().getName()); |
| } |
| catch (InvalidRegistryObjectException e) { |
| // ignore this extension and move on |
| } |
| } |
| Collection<String> additionalPluginIds = BaseHelpSystem.getLocalSearchManager() |
| .getPluginsWithSearchParticipants(); |
| totalIds.addAll(additionalPluginIds); |
| docPlugins = new PluginVersionInfo(INDEXED_CONTRIBUTION_INFO_FILE, totalIds, indexDir, !exists()); |
| } |
| return docPlugins; |
| } |
| |
| /** |
| * Sets the list of all plug-ns in this session. This method is used for external indexer. |
| * |
| * @param docPlugins |
| */ |
| public void setDocPlugins(PluginVersionInfo docPlugins) { |
| this.docPlugins = docPlugins; |
| } |
| |
| /** |
| * We use HelpProperties, but a list would suffice. We only need the key values. |
| * |
| * @return HelpProperties, keys are URLs of indexed documents |
| */ |
| public HelpProperties getIndexedDocs() { |
| HelpProperties indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); |
| if (exists()) |
| indexedDocs.restore(); |
| return indexedDocs; |
| } |
| |
| /** |
| * Gets properties with versions of Lucene plugin and Analyzer used in existing index |
| */ |
| private HelpProperties getDependencies() { |
| if (dependencies == null) { |
| dependencies = new HelpProperties(DEPENDENCIES_VERSION_FILENAME, indexDir); |
| dependencies.restore(); |
| } |
| return dependencies; |
| } |
| |
| private boolean isLuceneCompatible() { |
| String usedLuceneVersion = getDependencies().getProperty(DEPENDENCIES_KEY_LUCENE); |
| return isLuceneCompatible(usedLuceneVersion); |
| } |
| |
| /** |
| * Determines whether an index can be read by the Lucene bundle |
| * @param indexVersionString The version of an Index directory |
| * @return |
| */ |
| public boolean isLuceneCompatible(String indexVersionString) { |
| if (indexVersionString==null) return false; |
| String luceneVersionString = ""; //$NON-NLS-1$ |
| Bundle luceneBundle = Platform.getBundle(LUCENE_BUNDLE_ID); |
| if (luceneBundle != null) { |
| luceneVersionString += luceneBundle.getHeaders() |
| .get(Constants.BUNDLE_VERSION); |
| } |
| Version luceneVersion = new Version(luceneVersionString); |
| Version indexVersion = new Version(indexVersionString); |
| Version v700 = new Version(7, 0, 0); |
| if (indexVersion.compareTo(v700) < 0) { |
| // index is older than Lucene 7.0.0 |
| return false; |
| } |
| if ( luceneVersion.compareTo(indexVersion) >= 0 ) { |
| // Lucene bundle is newer than the index |
| return true; |
| } |
| return luceneVersion.getMajor() == indexVersion.getMajor() |
| && luceneVersion.getMinor() == indexVersion.getMinor() |
| && luceneVersion.getMicro() == indexVersion.getMicro(); |
| } |
| |
| private boolean isAnalyzerCompatible() { |
| String usedAnalyzer = getDependencies().getProperty(DEPENDENCIES_KEY_ANALYZER); |
| return isAnalyzerCompatible(usedAnalyzer); |
| } |
| |
| public boolean isAnalyzerCompatible(String analyzerId) { |
| if (analyzerId == null) { |
| analyzerId = ""; //$NON-NLS-1$ |
| } |
| return analyzerDescriptor.isCompatible(analyzerId); |
| } |
| |
| /** |
| * Saves Lucene version and analyzer identifier to a file. |
| */ |
| private void saveDependencies() { |
| getDependencies().put(DEPENDENCIES_KEY_ANALYZER, analyzerDescriptor.getId()); |
| Bundle luceneBundle = Platform.getBundle(LUCENE_BUNDLE_ID); |
| if (luceneBundle != null) { |
| String luceneBundleVersion = "" //$NON-NLS-1$ |
| + luceneBundle.getHeaders().get(Constants.BUNDLE_VERSION); |
| getDependencies().put(DEPENDENCIES_KEY_LUCENE, luceneBundleVersion); |
| } else { |
| getDependencies().put(DEPENDENCIES_KEY_LUCENE, ""); //$NON-NLS-1$ |
| } |
| getDependencies().save(); |
| } |
| |
| /** |
| * @return Returns true if index has been left in inconsistent state If analyzer has changed to |
| * incompatible one, index is treated as inconsistent as well. |
| */ |
| public boolean isInconsistent() { |
| if (inconsistencyFile.exists()) { |
| return true; |
| } |
| return !isLuceneCompatible() || !isAnalyzerCompatible(); |
| } |
| |
| /** |
| * Writes or deletes inconsistency flag file |
| */ |
| public void setInconsistent(boolean inconsistent) { |
| if (inconsistent) { |
| try (FileOutputStream fos = new FileOutputStream(inconsistencyFile)) { |
| // parent directory already created by beginAddBatch on new |
| // index |
| } catch (IOException ioe) { |
| } |
| } else |
| inconsistencyFile.delete(); |
| } |
| |
| public void openSearcher() throws IOException { |
| synchronized (searcherCreateLock) { |
| if (searcher == null) { |
| searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); |
| } |
| } |
| } |
| |
| /** |
| * Closes IndexReader used by Searcher. Should be called on platform shutdown, or when TOCs have |
| * changed when no more reading from this index is to be performed. |
| */ |
| public void close() { |
| closed = true; |
| |
| // wait for all searches to finish |
| while (true) { |
| synchronized (searches) { |
| if (searches.isEmpty()) { |
| if (searcher != null) { |
| try { |
| searcher.getIndexReader().close(); |
| } catch (IOException ioe) { |
| } |
| } |
| break; |
| } |
| } |
| try { |
| Thread.sleep(50); |
| } catch (InterruptedException ie) { |
| } |
| } |
| } |
| |
| /** |
| * Finds and unzips prebuild index specified in preferences |
| */ |
| private void unzipProductIndex() { |
| String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID, "productIndex", null, null); //$NON-NLS-1$ |
| if (indexPluginId == null || indexPluginId.length() <= 0) { |
| return; |
| } |
| InputStream zipIn = ResourceLocator.openFromPlugin(indexPluginId, "doc_index.zip", getLocale()); //$NON-NLS-1$ |
| if (zipIn == null) { |
| return; |
| } |
| setInconsistent(true); |
| cleanOldIndex(); |
| byte[] buf = new byte[8192]; |
| File destDir = indexDir; |
| |
| FileOutputStream fos = null; |
| try (ZipInputStream zis = new ZipInputStream(zipIn)) { |
| ZipEntry zEntry; |
| while ((zEntry = zis.getNextEntry()) != null) { |
| // if it is empty directory, create it |
| if (zEntry.isDirectory()) { |
| new File(destDir, zEntry.getName()).mkdirs(); |
| continue; |
| } |
| // if it is a file, extract it |
| String filePath = zEntry.getName(); |
| int lastSeparator = filePath.lastIndexOf("/"); //$NON-NLS-1$ |
| String fileDir = ""; //$NON-NLS-1$ |
| if (lastSeparator >= 0) { |
| fileDir = filePath.substring(0, lastSeparator); |
| } |
| // create directory for a file |
| new File(destDir, fileDir).mkdirs(); |
| // write file |
| File outFile = new File(destDir, filePath); |
| fos = new FileOutputStream(outFile); |
| int n = 0; |
| while ((n = zis.read(buf)) >= 0) { |
| fos.write(buf, 0, n); |
| } |
| fos.close(); |
| } |
| setInconsistent(false); |
| } catch (IOException ioe) { |
| if (fos != null) { |
| try { |
| fos.close(); |
| } catch (IOException ioe2) { |
| } |
| } |
| } finally { |
| try { |
| zipIn.close(); |
| } catch (IOException ioe) { |
| } |
| } |
| } |
| |
| /** |
| * Cleans any old index and Lucene lock files by initializing a new index. |
| */ |
| private void cleanOldIndex() { |
| try (LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 10000); |
| IndexWriter cleaner = new IndexWriter(luceneDirectory, |
| new IndexWriterConfig(analyzer) |
| .setOpenMode(OpenMode.CREATE))) { |
| |
| } catch (IOException ioe) { |
| } |
| } |
| |
| /** |
| * Returns true when the index must be updated. |
| */ |
| public synchronized boolean needsUpdating() { |
| if (!exists()) { |
| return true; |
| } |
| return getDocPlugins().detectChange(); |
| } |
| |
| /** |
| * @return Returns the tocManager. |
| */ |
| public TocManager getTocManager() { |
| return tocManager; |
| } |
| |
| private void registerSearch(Thread t) { |
| synchronized (searches) { |
| searches.add(t); |
| } |
| } |
| |
| private void unregisterSearch(Thread t) { |
| synchronized (searches) { |
| searches.remove(t); |
| } |
| } |
| |
| /** |
| * @return Returns the closed. |
| */ |
| public boolean isClosed() { |
| return closed; |
| } |
| |
| /** |
| * @return true if lock obtained for this Eclipse instance |
| * @throws OverlappingFileLockException |
| * if lock already obtained |
| */ |
| public synchronized boolean tryLock() throws OverlappingFileLockException { |
| if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$ |
| return true; // Act as if lock succeeded |
| } |
| if (lock != null) { |
| throw new OverlappingFileLockException(); |
| } |
| File lockFile = getLockFile(); |
| lockFile.getParentFile().mkdirs(); |
| try { |
| raf = new RandomAccessFile(lockFile, "rw"); //$NON-NLS-1$ |
| FileLock l = raf.getChannel().tryLock(); |
| if (l != null) { |
| // The RandomAccessFile raf cannot be closed yet because closing it will release the |
| // lock. It will be closed when the lock is released. |
| lock = l; |
| return true; |
| } |
| logLockFailure(null); |
| } catch (IOException ioe) { |
| lock = null; |
| logLockFailure(ioe); |
| } |
| if ( raf != null ) { |
| try { |
| raf.close(); |
| } catch (IOException e) { |
| } |
| raf = null; |
| } |
| return false; |
| } |
| |
| private static boolean errorReported = false; |
| |
| private void logLockFailure(IOException ioe) { |
| if (!errorReported) { |
| HelpBasePlugin.logError("Unable to Lock Help Search Index", ioe); //$NON-NLS-1$ |
| errorReported = true; |
| } |
| } |
| |
| private File getLockFile() { |
| return new File(indexDir.getParentFile(), locale + ".lock"); //$NON-NLS-1$ |
| } |
| |
| /** |
| * Deletes the lock file. The lock must be released prior to this call. |
| * |
| * @return <code>true</code> if the file has been deleted, <code>false</code> otherwise. |
| */ |
| |
| public synchronized boolean deleteLockFile() { |
| if (lock != null) |
| return false; |
| File lockFile = getLockFile(); |
| if (lockFile.exists()) |
| return lockFile.delete(); |
| return true; |
| } |
| |
| public synchronized void releaseLock() { |
| if (lock != null) { |
| try { |
| lock.channel().close(); |
| } catch (IOException ioe) { |
| } |
| lock = null; |
| } |
| if (raf != null ) { |
| try { |
| raf.close(); |
| } catch (IOException ioe) { |
| } |
| raf = null; |
| } |
| } |
| |
| public static String getIndexableHref(String url) { |
| String fileName = url.toLowerCase(Locale.ENGLISH); |
| if (fileName.endsWith(".htm") //$NON-NLS-1$ |
| || fileName.endsWith(".html") //$NON-NLS-1$ |
| || fileName.endsWith(".xhtml") //$NON-NLS-1$ |
| || fileName.endsWith(".xml") //$NON-NLS-1$ |
| || fileName.endsWith(".txt")) { //$NON-NLS-1$ |
| // indexable |
| } else if (fileName.indexOf(".htm#") >= 0 //$NON-NLS-1$ |
| || fileName.indexOf(".html#") >= 0 //$NON-NLS-1$ |
| || fileName.indexOf(".xhtml#") >= 0 //$NON-NLS-1$ |
| || fileName.indexOf(".xml#") >= 0) { //$NON-NLS-1$ |
| url = url.substring(0, url.lastIndexOf('#')); |
| // its a fragment, index whole document |
| } else { |
| // try search participants |
| return BaseHelpSystem.getLocalSearchManager().isIndexable(url) ? url : null; |
| } |
| return url; |
| } |
| |
| /** |
| * Checks if document is indexable, and creates a URL to obtain contents. |
| * |
| * @param locale |
| * @param url |
| * specified in the navigation |
| * @return URL to obtain document content or null |
| */ |
| public static URL getIndexableURL(String locale, String url) { |
| return getIndexableURL(locale, url, null, null); |
| } |
| |
| /** |
| * Checks if document is indexable, and creates a URL to obtain contents. |
| * |
| * @param locale |
| * @param url |
| * @param participantId |
| * the search participant or <code>null</code> specified in the navigation |
| * @return URL to obtain document content or null |
| */ |
| public static URL getIndexableURL(String locale, String url, String id, String participantId) { |
| if (participantId == null) |
| url = getIndexableHref(url); |
| if (url == null) |
| return null; |
| |
| try { |
| StringBuilder query = new StringBuilder(); |
| query.append("?"); //$NON-NLS-1$ |
| query.append("lang=" + locale); //$NON-NLS-1$ |
| if (id != null) |
| query.append("&id=" + id); //$NON-NLS-1$ |
| if (participantId != null) |
| query.append("&participantId=" + participantId); //$NON-NLS-1$ |
| return new URL("localhelp", //$NON-NLS-1$ |
| null, -1, url + query.toString(), HelpURLStreamHandler.getDefault()); |
| |
| } catch (MalformedURLException mue) { |
| return null; |
| } |
| } |
| |
| public IStatus addDocument(String pluginId, String name, URL url, String id, Document doc) { |
| // try a registered participant for the file format |
| SearchParticipant participant = BaseHelpSystem.getLocalSearchManager() |
| .getParticipant(pluginId, name); |
| if (participant != null) { |
| try { |
| return participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); |
| } |
| catch (Throwable t) { |
| return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, |
| "Error while adding document to search participant (addDocument()): " //$NON-NLS-1$ |
| + name + ", " + url + "for participant " + participant.getClass().getName(), t); //$NON-NLS-1$ //$NON-NLS-2$ |
| } |
| } |
| // default to html |
| return htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); |
| } |
| |
| @Override |
| public IStatus addSearchableDocument(String pluginId, String name, URL url, String id, ISearchDocument doc) { |
| // In the help system the only class that implements ISearchDocument is LuceneSearchDocument |
| LuceneSearchDocument luceneDoc = (LuceneSearchDocument)doc; |
| return addDocument(pluginId, name, url, id, luceneDoc.getDocument()); |
| } |
| } |