| /*=============================================================================# |
| # Copyright (c) 2010, 2020 Stephan Wahlbrink and others. |
| # |
| # This program and the accompanying materials are made available under the |
| # terms of the Eclipse Public License 2.0 which is available at |
| # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 |
| # which is available at https://www.apache.org/licenses/LICENSE-2.0. |
| # |
| # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 |
| # |
| # Contributors: |
| # Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation |
| #=============================================================================*/ |
| |
| package org.eclipse.statet.internal.rhelp.core.index; |
| |
| import static org.eclipse.statet.internal.rhelp.core.RHelpCoreInternals.DEBUG; |
| import static org.eclipse.statet.internal.rhelp.core.index.RHelpHtmlUtils.indexOfLastHr; |
| import static org.eclipse.statet.internal.rhelp.core.index.RHelpHtmlUtils.indexOfSectionEnd; |
| |
| import java.io.IOException; |
| import java.nio.file.Files; |
| import java.nio.file.Path; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.HashMap; |
| import java.util.Iterator; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.ListIterator; |
| import java.util.Map; |
| |
| import org.apache.lucene.analysis.Analyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.index.CorruptIndexException; |
| import org.apache.lucene.index.DirectoryReader; |
| import org.apache.lucene.index.IndexFormatTooOldException; |
| import org.apache.lucene.index.IndexReader; |
| import org.apache.lucene.index.IndexWriter; |
| import org.apache.lucene.index.IndexWriterConfig; |
| import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| import org.apache.lucene.index.LeafReader; |
| import org.apache.lucene.index.LeafReaderContext; |
| import org.apache.lucene.index.LiveIndexWriterConfig; |
| import org.apache.lucene.index.Term; |
| import org.apache.lucene.index.Terms; |
| import org.apache.lucene.index.TermsEnum; |
| import org.apache.lucene.store.FSDirectory; |
| import org.apache.lucene.util.BytesRef; |
| |
| import org.eclipse.statet.jcommons.collections.ImCollections; |
| import org.eclipse.statet.jcommons.collections.ImList; |
| import org.eclipse.statet.jcommons.io.FileUtils; |
| import org.eclipse.statet.jcommons.lang.NonNullByDefault; |
| import org.eclipse.statet.jcommons.lang.Nullable; |
| import org.eclipse.statet.jcommons.status.CancelStatus; |
| import org.eclipse.statet.jcommons.status.ErrorStatus; |
| import org.eclipse.statet.jcommons.status.InfoStatus; |
| import org.eclipse.statet.jcommons.status.MultiStatus; |
| import org.eclipse.statet.jcommons.status.Status; |
| import org.eclipse.statet.jcommons.status.StatusException; |
| import org.eclipse.statet.jcommons.text.core.input.StringParserInput; |
| import org.eclipse.statet.jcommons.text.core.util.HtmlStripParserInput; |
| |
| import org.eclipse.statet.internal.rhelp.core.REnvHelpImpl; |
| import org.eclipse.statet.internal.rhelp.core.RHelpCoreInternals; |
| import org.eclipse.statet.internal.rhelp.core.RHelpKeywordGroupImpl; |
| import org.eclipse.statet.internal.rhelp.core.RHelpKeywordImpl; |
| import org.eclipse.statet.internal.rhelp.core.RHelpManagerIntern; |
| import org.eclipse.statet.internal.rhelp.core.RHelpPageImpl; |
| import org.eclipse.statet.internal.rhelp.core.RPkgHelpImpl; |
| import org.eclipse.statet.internal.rhelp.core.SerUtil; |
| import org.eclipse.statet.rhelp.core.DocResource; |
| import org.eclipse.statet.rhelp.core.REnvHelpConfiguration; |
| import org.eclipse.statet.rhelp.core.RHelpCore; |
| import org.eclipse.statet.rhelp.core.RHelpKeywordGroup; |
| import org.eclipse.statet.rhelp.core.RHelpKeywordNode; |
| import org.eclipse.statet.rhelp.core.RPkgHelp; |
| import org.eclipse.statet.rj.renv.core.RLibLocation; |
| import org.eclipse.statet.rj.renv.core.RNumVersion; |
| import org.eclipse.statet.rj.renv.core.RPkgDescription; |
| |
| |
| @NonNullByDefault |
| public class REnvIndexWriter implements REnvIndexSchema { |
| |
| |
| public static class RdItem { |
| |
| |
| private final String pkg; |
| private final String name; |
| |
| private final List<String> topics= new ArrayList<>(8); |
| private ImList<String> imTopics; |
| private String title= ""; //$NON-NLS-1$ |
| private @Nullable List<String> keywords; |
| private @Nullable List<String> concepts; |
| |
| private @Nullable String html; |
| |
| private @Nullable String descrTxt; |
| private @Nullable String mainTxt; |
| private @Nullable String examplesTxt; |
| |
| |
| public RdItem(final String pkg, final String name) { |
| this.pkg= pkg; |
| this.name= name.intern(); |
| } |
| |
| |
| void finish() { |
| this.imTopics= ImCollections.toList(this.topics); |
| } |
| |
| |
| public String getPkg() { |
| return this.pkg; |
| } |
| |
| public String getName() { |
| return this.name; |
| } |
| |
| private ImList<String> getTopics() { |
| return this.imTopics; |
| } |
| |
| public void addTopic(final String alias) { |
| if (!this.topics.contains(alias)) { |
| this.topics.add(alias.intern()); |
| } |
| } |
| |
| private String getTitle() { |
| return this.title; |
| } |
| |
| public void setTitle(final @Nullable String title) { |
| if (title != null) { |
| this.title= title; |
| } |
| } |
| |
| public void addKeyword(final String keyword) { |
| List<String> keywords= this.keywords; |
| if (keywords == null) { |
| keywords= new ArrayList<>(8); |
| this.keywords= keywords; |
| } |
| if (!keywords.contains(keyword)) { |
| keywords.add(keyword); |
| } |
| } |
| |
| public boolean isInternal() { |
| final List<String> keywords= this.keywords; |
| return (keywords != null && keywords.contains(RHelpPageImpl.INTERNAL_KEYWORD)); |
| } |
| |
| public void addConcept(final String concept) { |
| if (this.concepts == null) { |
| this.concepts= new ArrayList<>(8); |
| } |
| this.concepts.add(concept); |
| } |
| |
| public void setHtml(final String html) { |
| this.html= html; |
| } |
| |
| } |
| |
| |
| public static final Collection<String> IGNORE_PKG_NAMES; |
| static { |
| IGNORE_PKG_NAMES= new ArrayList<>(); |
| IGNORE_PKG_NAMES.add("translations"); //$NON-NLS-1$ |
| |
| String s= System.getProperty("org.eclipse.statet.r.rhelp.PkgsToExclude.names"); //$NON-NLS-1$ |
| if (s != null && !(s= s.trim()).isEmpty()) { |
| for (final String name : s.split(",")) { //$NON-NLS-1$ |
| if (!name.isEmpty() && !IGNORE_PKG_NAMES.contains(name)) { |
| IGNORE_PKG_NAMES.add(name); |
| } |
| } |
| } |
| } |
| |
| |
| private static Analyzer WRITE_ANALYZER= new WriteAnalyzer(); |
| |
| |
| /** Worker for packages. Single thread! */ |
| class PackageWorker { |
| |
| |
| private final FlagField doctypeField_PKG_DESCRIPTION= new FlagField(DOCTYPE_FIELD_NAME, PKG_DESCRIPTION_DOCTYPE); |
| private final FlagField doctypeField_PAGE= new FlagField(DOCTYPE_FIELD_NAME, PAGE_DOCTYPE); |
| private final NameField packageField= new NameField(PACKAGE_FIELD_NAME); |
| private final NameField pageField= new NameField(PAGE_FIELD_NAME); |
| private final TxtField titleTxtField= new TxtField(TITLE_TXT_FIELD_NAME); |
| private final MultiValueFieldList<NameField> aliasFields= MultiValueFieldList.forNameField( |
| ALIAS_FIELD_NAME ); |
| private final MultiValueFieldList<TxtField> aliasTxtFields= MultiValueFieldList.forTxtField( |
| ALIAS_TXT_FIELD_NAME ); |
| private final TxtField descriptionTxtField= new TxtField(DESCRIPTION_TXT_FIELD_NAME); |
| private final TxtField authorsTxtField= new TxtField(AUTHORS_TXT_FIELD_NAME); |
| private final TxtField maintainerTxtField= new TxtField(MAINTAINER_TXT_FIELD_NAME); |
| private final TxtField urlTxtField= new TxtField(URL_TXT_FIELD_NAME); |
| private final MultiValueFieldList<KeywordField> keywordTxtFields= MultiValueFieldList.forKeywordField( |
| KEYWORD_FIELD_NAME ); |
| private final MultiValueFieldList<TxtField> conteptTxtFields= MultiValueFieldList.forTxtField( |
| CONCEPT_TXT_FIELD_NAME ); |
| private final TxtField docTxtField= new TxtField(DOC_TXT_FIELD_NAME); |
| private final TxtField.OmitNorm docHtmlField= new TxtField.OmitNorm(DOC_HTML_FIELD_NAME); |
| private final TxtField examplesTxtField= new TxtField(EXAMPLES_TXT_FIELD_NAME); |
| |
| private final StringBuilder tempBuilder= new StringBuilder(65536); |
| private final HtmlStripParserInput tempHtmlInput= new HtmlStripParserInput( |
| new StringParserInput(0x800), 0x800 ); |
| |
| |
| public PackageWorker() { |
| } |
| |
| |
| private void addToLucene(final RPkgDescription item) throws CorruptIndexException, IOException { |
| final Document doc= new Document(); |
| doc.add(this.doctypeField_PKG_DESCRIPTION); |
| this.packageField.setStringValue(item.getName()); |
| doc.add(this.packageField); |
| this.descriptionTxtField.setStringValue(item.getDescription()); |
| doc.add(this.descriptionTxtField); |
| if (item.getAuthor() != null) { |
| this.authorsTxtField.setStringValue(item.getAuthor()); |
| doc.add(this.authorsTxtField); |
| } |
| if (item.getMaintainer() != null) { |
| this.maintainerTxtField.setStringValue(item.getMaintainer()); |
| doc.add(this.maintainerTxtField); |
| } |
| if (item.getUrl() != null) { |
| this.urlTxtField.setStringValue(item.getUrl()); |
| doc.add(this.urlTxtField); |
| } |
| REnvIndexWriter.this.luceneWriter.addDocument(doc); |
| } |
| |
| private void addToLucene(final RdItem item) throws CorruptIndexException, IOException { |
| final Document doc= new Document(); |
| doc.add(this.doctypeField_PAGE); |
| this.packageField.setStringValue(item.getPkg()); |
| doc.add(this.packageField); |
| this.pageField.setStringValue(item.getName()); |
| doc.add(this.pageField); |
| this.titleTxtField.setStringValue(item.getTitle()); |
| doc.add(this.titleTxtField); |
| { final ImList<String> topics= item.getTopics(); |
| for (int i= 0; i < topics.size(); i++) { |
| final NameField nameField= this.aliasFields.get(i); |
| nameField.setStringValue(topics.get(i)); |
| doc.add(nameField); |
| final TxtField txtField= this.aliasTxtFields.get(i); |
| txtField.setStringValue(topics.get(i)); |
| doc.add(txtField); |
| } |
| } |
| if (item.keywords != null) { |
| final List<String> keywords= item.keywords; |
| for (int i= 0; i < keywords.size(); i++) { |
| final KeywordField field= this.keywordTxtFields.get(i); |
| field.setStringValue(keywords.get(i)); |
| doc.add(field); |
| } |
| } |
| if (item.concepts != null) { |
| final List<String> concepts= item.concepts; |
| for (int i= 0; i < concepts.size(); i++) { |
| final TxtField txtField= this.conteptTxtFields.get(i); |
| txtField.setStringValue(concepts.get(i)); |
| doc.add(txtField); |
| } |
| } |
| if (item.html != null) { |
| createSectionsTxt(item); |
| if (item.descrTxt != null) { |
| this.descriptionTxtField.setStringValue(item.descrTxt); |
| doc.add(this.descriptionTxtField); |
| } |
| this.docTxtField.setStringValue(item.mainTxt); |
| doc.add(this.docTxtField); |
| if (item.examplesTxt != null) { |
| this.examplesTxtField.setStringValue(item.examplesTxt); |
| doc.add(this.examplesTxtField); |
| } |
| this.docHtmlField.setStringValue(item.html); |
| doc.add(this.docHtmlField); |
| } |
| REnvIndexWriter.this.luceneWriter.addDocument(doc); |
| } |
| |
| private void createSectionsTxt(final RdItem item) throws IOException { |
| String html= item.html; |
| this.tempBuilder.setLength(0); |
| { final int idx1= html.indexOf("</h2>"); //$NON-NLS-1$ |
| if (idx1 >= 0) { |
| html= html.substring(idx1 + 5); |
| } |
| } |
| { final int idx1= indexOfLastHr(html); |
| if (idx1 >= 0) { |
| html= html.substring(0, idx1); |
| } |
| } |
| { int idxBegin= html.indexOf("<h3 id=\"description\""); //$NON-NLS-1$ |
| if (idxBegin >= 0) { |
| idxBegin= html.indexOf('>', idxBegin + 20); |
| if (idxBegin >= 0) { |
| idxBegin= html.indexOf("</h3>", idxBegin + 1); //$NON-NLS-1$ |
| if (idxBegin >= 0) { |
| idxBegin+= 5; |
| final int idxEnd= indexOfSectionEnd(html, idxBegin); |
| if (idxEnd >= 0) { |
| item.descrTxt= html2txt(html.substring(idxBegin, idxEnd)); |
| html= html.substring(idxEnd); |
| } |
| } |
| } |
| } |
| } |
| { final String[] s= new String[] { html, null }; |
| if (extract(s, "<h3 id=\"examples\"")) { //$NON-NLS-1$ |
| item.examplesTxt= html2txt(s[1]); |
| } |
| item.mainTxt= html2txt(s[0]); |
| } |
| } |
| |
| private boolean extract(final String[] s, final String h3) { |
| final String html= s[0]; |
| final int idx0= html.indexOf(h3); |
| if (idx0 >= 0) { |
| int idxBegin= html.indexOf('>', idx0 + h3.length()); |
| if (idxBegin >= 0) { |
| idxBegin= html.indexOf("</h3>", idxBegin + 1); //$NON-NLS-1$ |
| if (idxBegin >= 0) { |
| idxBegin+= 5; |
| final int idxEnd= indexOfSectionEnd(html, idxBegin); |
| if (idxEnd >= 0) { |
| this.tempBuilder.setLength(0); |
| this.tempBuilder.append(html, 0, idx0); |
| this.tempBuilder.append(html, idxEnd, html.length()); |
| s[0]= this.tempBuilder.toString(); |
| s[1]= html.substring(idxBegin, idxEnd); |
| } |
| else { |
| s[0]= html.substring(0, idx0); |
| s[1]= html.substring(idxBegin, html.length()); |
| } |
| return true; |
| } |
| } |
| } |
| return false; |
| } |
| |
| private String html2txt(final String html) { |
| this.tempBuilder.setLength(0); |
| ((StringParserInput) this.tempHtmlInput.getSource()).reset(html); |
| this.tempHtmlInput.init(); |
| int c; |
| boolean blank= true; |
| while ((c= this.tempHtmlInput.get(0)) >= 0) { |
| if (c <= 0x20) { |
| if (!blank) { |
| blank= true; |
| this.tempBuilder.append(' '); |
| } |
| } |
| else { |
| if (blank) { |
| blank= false; |
| } |
| this.tempBuilder.append((char) c); |
| } |
| this.tempHtmlInput.consume(1); |
| } |
| c= this.tempBuilder.length(); |
| return (c > 0 && this.tempBuilder.charAt(c - 1) == ' ') ? |
| this.tempBuilder.substring(0, c - 1) : this.tempBuilder.toString(); |
| } |
| |
| } |
| |
| |
| private final RHelpManagerIntern rHelpManager; |
| private final REnvHelpConfiguration rEnvConfig; |
| |
| private long stamp; |
| |
| private @Nullable String docDir; |
| private @Nullable Path docDirPath; |
| private final List<DocResource> manuals= new ArrayList<>(); |
| private final List<DocResource> miscRes= new ArrayList<>(); |
| |
| private Map<String, RPkgHelp> existingPackages; |
| private Map<String, @Nullable RPkgHelp> packages; |
| private LinkedHashMap<String, RHelpKeywordGroupImpl> keywordGroups; |
| |
| private final Path indexDirectory; |
| private FSDirectory luceneDirectory; |
| private IndexWriter luceneWriter; |
| |
| private @Nullable RPkgHelpImpl currentPkg; |
| private final List<RHelpPageImpl> currentPkgPages= new ArrayList<>(1024); |
| |
| private @Nullable Object indexLock; |
| |
| private @Nullable Map<String, String> rEnvSharedProperties; |
| |
| private boolean reset; |
| |
| // At moment we use a single worker; multi-threading is not worth, because R is the bottleneck. |
| // For multi-threading: thread pool / jobs with worker / thread, currentPackage to worker, ... |
| private final PackageWorker worker= new PackageWorker(); |
| |
| private @Nullable MultiStatus status; |
| |
| |
| public REnvIndexWriter(final REnvHelpConfiguration rEnvConfig, |
| final RHelpManagerIntern rHelpManager) { |
| this.rEnvConfig= rEnvConfig; |
| this.rHelpManager= rHelpManager; |
| |
| this.indexDirectory= SerUtil.getIndexDirectoryChecked(rEnvConfig); |
| } |
| |
| |
| public void log(final Status status) { |
| final MultiStatus multiStatus= this.status; |
| if (multiStatus != null) { |
| multiStatus.add(status); |
| } |
| else { |
| RHelpCoreInternals.log(status); |
| } |
| } |
| |
| public void beginBatch(final boolean reset) throws AbortIndexOperationException, StatusException { |
| if (this.luceneWriter != null) { |
| throw new IllegalStateException(); |
| } |
| |
| this.status= new MultiStatus(RHelpCore.BUNDLE_ID, |
| String.format("Indexing: '%1$s'.", this.rEnvConfig.getName()) ); //$NON-NLS-1$ |
| |
| try { |
| this.indexLock= this.rHelpManager.beginIndexUpdate(this.rEnvConfig.getREnv()); |
| if (this.indexLock == null) { |
| final Status status= new CancelStatus(RHelpCore.BUNDLE_ID, |
| "Indexing is already running in another task." ); |
| this.status.add(status); |
| throw new StatusException(status); |
| } |
| |
| this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, |
| String.format("Beginning batch (index directory= '%1$s').", |
| this.indexDirectory.toString() ))); |
| |
| this.luceneDirectory= REnvIndexUtils.getDirectory(this.indexDirectory); |
| |
| this.reset= true; |
| if (!reset) { |
| final REnvHelpImpl oldHelp= this.rHelpManager.getHelpIntern(this.rEnvConfig.getREnv()); |
| try { |
| if (oldHelp != null && oldHelp.getVersion() == SerUtil.CURRENT_VERSION) { |
| try (final IndexReader dirReader= DirectoryReader.open(this.luceneDirectory)) { |
| this.existingPackages= new HashMap<>(64); |
| for (final LeafReaderContext leave : dirReader.leaves()) { |
| final LeafReader aReader= leave.reader(); |
| final Terms terms= aReader.terms(PACKAGE_FIELD_NAME); |
| if (terms != null) { |
| BytesRef term; |
| for (final TermsEnum termsEnum= terms.iterator(); |
| ((term= termsEnum.next()) != null); ) { |
| final String name= term.utf8ToString(); |
| final RPkgHelp pkgHelp= (oldHelp != null) ? oldHelp.getPkgHelp(name) : null; |
| this.existingPackages.put(name, pkgHelp); |
| } |
| } |
| } |
| |
| final IndexWriterConfig config= createWriterConfig(); |
| config.setOpenMode(OpenMode.CREATE_OR_APPEND); |
| this.luceneWriter= new IndexWriter(this.luceneDirectory, config); |
| this.reset= false; |
| } |
| catch (final IOException e) { |
| assert (this.luceneWriter == null); |
| // try again new |
| } |
| } |
| } |
| finally { |
| if (oldHelp != null) { |
| oldHelp.unlock(); |
| } |
| } |
| } |
| |
| if (this.luceneWriter == null) { |
| this.existingPackages= new HashMap<>(0); |
| |
| try { |
| final IndexWriterConfig config= createWriterConfig(); |
| config.setOpenMode(OpenMode.CREATE); |
| this.luceneWriter= new IndexWriter(this.luceneDirectory, config); |
| } |
| catch (final IndexFormatTooOldException e) { |
| if (Files.exists(this.indexDirectory)) { |
| FileUtils.cleanDirectory(this.indexDirectory); |
| final IndexWriterConfig config= createWriterConfig(); |
| config.setOpenMode(OpenMode.CREATE); |
| this.luceneWriter= new IndexWriter(this.luceneDirectory, config); |
| } |
| else { |
| throw e; |
| } |
| } |
| } |
| |
| this.stamp= REnvHelpImpl.createStamp(); |
| |
| this.packages= new LinkedHashMap<>(); |
| this.keywordGroups= new LinkedHashMap<>(); |
| } |
| catch (final IOException e) { |
| throw new AbortIndexOperationException(e); |
| } |
| catch (final OutOfMemoryError e) { |
| throw new AbortIndexOperationException(e); |
| } |
| } |
| |
| private IndexWriterConfig createWriterConfig() { |
| final IndexWriterConfig config= new IndexWriterConfig(WRITE_ANALYZER); |
| config.setSimilarity(SIMILARITY); |
| config.setRAMPerThreadHardLimitMB(512); |
| config.setCommitOnClose(false); |
| return config; |
| } |
| |
| public String getDocDir() { |
| return this.docDir; |
| } |
| |
| public void setDocDir(@Nullable String docDir, final @Nullable Path docDirPath) { |
| if (docDir != null && docDir.isEmpty()) { |
| docDir= null; |
| } |
| this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, |
| String.format("Setting doc dir to: %1$s.", //$NON-NLS-1$ |
| (docDir != null) ? ('\'' + docDir + '\'') : "<missing>" ))); //$NON-NLS-1$ |
| this.docDir= docDir; |
| this.docDirPath= docDirPath; |
| } |
| |
| public void addManual(final DocResource res) { |
| this.manuals.add(res); |
| } |
| |
| public void addManuals(final Collection<DocResource> resources) { |
| this.manuals.addAll(resources); |
| } |
| |
| public void addMiscResource(final DocResource res) { |
| this.miscRes.add(res); |
| } |
| |
| public void addMiscResources(final Collection<DocResource> resources) { |
| this.miscRes.addAll(resources); |
| } |
| |
| public void setREnvSharedProperties(final @Nullable Map<String, String> properties) { |
| this.rEnvSharedProperties= properties; |
| } |
| |
| |
| public void addDefaultKeyword(final String[] path, final String description) { |
| if (path == null || path.length == 0) { |
| return; |
| } |
| if (path.length == 1) { // group |
| final String key= path[0].trim().intern(); |
| if (key.length() > 0) { |
| this.keywordGroups.put(key, new RHelpKeywordGroupImpl(key, description)); |
| } |
| return; |
| } |
| else { |
| RHelpKeywordNode node= this.keywordGroups.get(path[0]); |
| int i= 1; |
| while (node != null) { |
| if (i == path.length - 1) { |
| if (path[i].length() > 0) { |
| final String key= path[i].intern(); |
| node.getNestedKeywords().add(new RHelpKeywordImpl(key, description)); |
| return; |
| } |
| } |
| else { |
| node= node.getNestedKeyword(path[i++]); |
| continue; |
| } |
| } |
| return; |
| } |
| } |
| |
| /** |
| * |
| * @param name package name |
| * @param version |
| * @return <code>true</code> if seems OK, otherwise false |
| */ |
| public boolean checkPackage(final String name, final RNumVersion version, final String built, |
| final RLibLocation libLocation) { |
| if (IGNORE_PKG_NAMES.contains(name)) { |
| return true; |
| } |
| synchronized (this.packages) { |
| if (this.packages.containsKey(name)) { |
| return true; |
| } |
| RPkgHelp pkgHelp= this.existingPackages.remove(name); |
| if (!this.reset && pkgHelp != null |
| && version.equals(pkgHelp.getVersion()) |
| && built.equals(pkgHelp.getPkgDescription().getBuilt()) ) { |
| if (libLocation != pkgHelp.getPkgDescription().getLibLocation()) { |
| pkgHelp= new RPkgHelpImpl(pkgHelp, libLocation); |
| } |
| this.packages.put(name, pkgHelp); // reuse |
| return true; |
| } |
| else { |
| this.packages.put(name, null); // placeholder |
| return false; |
| } |
| } |
| } |
| |
| public void beginPackage(final RPkgDescription pkgDesription) |
| throws AbortIndexOperationException { |
| final String name= pkgDesription.getName(); |
| if (this.currentPkg != null) { |
| throw new IllegalArgumentException(); |
| } |
| try { |
| this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, |
| String.format("Beginning package: '%1$s'.", name) )); //$NON-NLS-1$ |
| |
| this.currentPkg= new RPkgHelpImpl(pkgDesription, this.rEnvConfig.getREnv()); |
| synchronized (this.packages) { |
| this.existingPackages.remove(name); |
| this.packages.put(name, this.currentPkg); |
| } |
| this.currentPkgPages.clear(); |
| this.luceneWriter.deleteDocuments(new Term(PACKAGE_FIELD_NAME, name)); |
| this.worker.addToLucene(pkgDesription); |
| } |
| catch (final IOException e) { |
| throw new AbortIndexOperationException(e); |
| } |
| catch (final OutOfMemoryError e) { |
| throw new AbortIndexOperationException(e); |
| } |
| } |
| |
| public void add(final RdItem item) throws AbortIndexOperationException { |
| final RPkgHelpImpl currentPkg= this.currentPkg; |
| if (currentPkg == null || !currentPkg.getName().equals(item.getPkg())) { |
| throw new IllegalArgumentException(); |
| } |
| try { |
| item.finish(); |
| this.currentPkgPages.add(new RHelpPageImpl(currentPkg, item.getName(), |
| RHelpPageImpl.createFlags(item.isInternal()), |
| item.getTopics(), |
| item.getTitle() )); |
| this.worker.addToLucene(item); |
| } |
| catch (final IOException e) { |
| throw new AbortIndexOperationException(e); |
| } |
| catch (final OutOfMemoryError e) { |
| throw new AbortIndexOperationException(e); |
| } |
| } |
| |
| public void endPackage() throws AbortIndexOperationException { |
| if (DEBUG) { |
| this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, "Finishing package.")); //$NON-NLS-1$ |
| |
| final Runtime runtime= Runtime.getRuntime(); |
| final long maxMemory= runtime.maxMemory(); |
| final long allocatedMemory= runtime.totalMemory(); |
| final long freeMemory= runtime.freeMemory(); |
| final LiveIndexWriterConfig config= this.luceneWriter.getConfig(); |
| final StringBuilder sb= new StringBuilder("Memory status:\n"); //$NON-NLS-1$ |
| sb.append("TempBuilder-capycity: ").append(this.worker.tempBuilder.capacity()).append('\n'); //$NON-NLS-1$ |
| sb.append("Lucene-buffersize: ").append((long) (config.getRAMBufferSizeMB() * 1024.0)).append('\n'); //$NON-NLS-1$ |
| sb.append("Memory-free: ").append(freeMemory / 1024L).append('\n'); //$NON-NLS-1$ |
| sb.append("Memory-total: ").append(allocatedMemory / 1024L).append('\n'); //$NON-NLS-1$ |
| sb.append("Memory-max: ").append(maxMemory / 1024L).append('\n'); //$NON-NLS-1$ |
| this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, sb.toString())); |
| } |
| |
| if (this.currentPkg == null) { |
| return; |
| } |
| |
| this.currentPkg.setPages(ImCollections.toList(this.currentPkgPages)); |
| this.currentPkg= null; |
| } |
| |
| public @Nullable Status endBatch() throws AbortIndexOperationException { |
| if (this.luceneWriter == null) { |
| return null; |
| } |
| |
| final MultiStatus status= this.status; |
| this.status= null; |
| if (status != null) { |
| status.add(new InfoStatus(RHelpCore.BUNDLE_ID, "Finishing batch.")); //$NON-NLS-1$ |
| RHelpCoreInternals.log(status); |
| } |
| try { |
| for (final String pkgName : this.existingPackages.keySet()) { |
| this.luceneWriter.deleteDocuments(new Term(PACKAGE_FIELD_NAME, pkgName)); |
| } |
| this.existingPackages.clear(); |
| |
| final ImList<DocResource> manuals= checkDocResources(this.manuals, this.docDirPath); |
| final ImList<DocResource> miscRes= checkDocResources(this.miscRes, this.docDirPath); |
| |
| final ImList<RHelpKeywordGroup> keywords; |
| { final Collection<RHelpKeywordGroupImpl> values= this.keywordGroups.values(); |
| for (final RHelpKeywordGroupImpl group : values) { |
| group.freeze(); |
| } |
| keywords= ImCollections.<RHelpKeywordGroup>toList(values); |
| } |
| final ImList<RPkgHelp> packages; |
| { for (final Iterator<RPkgHelp> iter= this.packages.values().iterator(); iter.hasNext(); ) { |
| if (iter.next() == null) { |
| iter.remove(); |
| } |
| } |
| final RPkgHelp[] array= this.packages.values().toArray(new RPkgHelp[this.packages.size()]); |
| Arrays.sort(array); |
| packages= ImCollections.newList(array); |
| } |
| |
| final REnvHelpImpl help= new REnvHelpImpl(this.rEnvConfig.getREnv(), |
| SerUtil.CURRENT_VERSION, this.stamp, |
| this.docDir, manuals, miscRes, |
| keywords, packages ); |
| |
| // this.luceneWriter.maybeMerge(); |
| this.luceneWriter.commit(); |
| |
| this.rHelpManager.updateLocalHelp(this.rEnvConfig, this.rEnvSharedProperties, help); |
| |
| this.luceneWriter.close(); |
| this.luceneWriter= null; |
| |
| if (status != null && status.getSeverity() >= Status.WARNING) { |
| return status; |
| } |
| return null; |
| } |
| catch (final IOException e) { |
| cancel(); |
| throw new AbortIndexOperationException(e); |
| } |
| catch (final OutOfMemoryError e) { |
| throw new AbortIndexOperationException(e); |
| } |
| finally { |
| clear(); |
| } |
| } |
| |
| public @Nullable Status cancel() { |
| final MultiStatus status; |
| try { |
| if (this.luceneWriter != null) { |
| try { |
| this.luceneWriter.rollback(); |
| } |
| catch (final Exception e) { |
| if (this.status != null) { |
| this.status.add(new ErrorStatus(RHelpCore.BUNDLE_ID, |
| "Error when rolling back.", null )); //$NON-NLS-1$ |
| } |
| } |
| |
| this.luceneWriter.close(); |
| this.luceneWriter= null; |
| } |
| } |
| catch (final Exception close) { |
| } |
| finally { |
| clear(); |
| |
| status= this.status; |
| this.status= null; |
| if (status != null) { |
| status.add(new InfoStatus(RHelpCore.BUNDLE_ID, "Canceling batch.")); //$NON-NLS-1$ |
| RHelpCoreInternals.log(status); |
| } |
| } |
| if (status != null && status.getSeverity() >= Status.WARNING) { |
| return status; |
| } |
| return null; |
| } |
| |
| private void clear() { |
| if (this.luceneWriter != null) { |
| try { |
| if (this.luceneWriter.isOpen()) { |
| this.luceneWriter.close(); |
| } |
| } catch (final Exception ignore) {} |
| } |
| this.luceneWriter= null; |
| |
| if (this.indexLock != null) { |
| this.rHelpManager.endIndexUpdate(this.indexLock); |
| this.indexLock= null; |
| } |
| |
| this.luceneDirectory= null; |
| this.currentPkg= null; |
| this.currentPkgPages.clear(); |
| this.indexLock= null; |
| } |
| |
| |
| private ImList<DocResource> checkDocResources(final List<DocResource> resources, |
| final @Nullable Path basePath) { |
| if (basePath == null) { |
| ImCollections.emptyList(); |
| } |
| for (final ListIterator<DocResource> iter= resources.listIterator(); iter.hasNext();) { |
| final DocResource res= iter.next(); |
| if (Files.isRegularFile(basePath.resolve(res.getPath()))) { |
| if (res.getPdfPath() != null |
| && !Files.isRegularFile(basePath.resolve(res.getPdfPath())) ) { |
| iter.set(new DocResource(res.getTitle(), res.getPath(), null)); |
| } |
| } |
| else { |
| iter.remove(); |
| } |
| } |
| return ImCollections.toList(resources); |
| } |
| |
| } |