rhelp/org.eclipse.statet.rhelp.core/src/org/eclipse/statet/internal/rhelp/core/index/REnvIndexWriter.java

/*=============================================================================#
 # Copyright (c) 2010, 2021 Stephan Wahlbrink and others.
 # 
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
 # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
 # which is available at https://www.apache.org/licenses/LICENSE-2.0.
 # 
 # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
 # 
 # Contributors:
 #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
 #=============================================================================*/

package org.eclipse.statet.internal.rhelp.core.index;

import static org.eclipse.statet.internal.rhelp.core.RHelpCoreInternals.DEBUG;
import static org.eclipse.statet.internal.rhelp.core.index.RHelpHtmlUtils.indexOfLastHr;
import static org.eclipse.statet.internal.rhelp.core.index.RHelpHtmlUtils.indexOfSectionEnd;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.LiveIndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;

import org.eclipse.statet.jcommons.collections.ImCollections;
import org.eclipse.statet.jcommons.collections.ImList;
import org.eclipse.statet.jcommons.io.FileUtils;
import org.eclipse.statet.jcommons.lang.NonNullByDefault;
import org.eclipse.statet.jcommons.lang.Nullable;
import org.eclipse.statet.jcommons.status.CancelStatus;
import org.eclipse.statet.jcommons.status.ErrorStatus;
import org.eclipse.statet.jcommons.status.InfoStatus;
import org.eclipse.statet.jcommons.status.MultiStatus;
import org.eclipse.statet.jcommons.status.Status;
import org.eclipse.statet.jcommons.status.StatusException;
import org.eclipse.statet.jcommons.text.core.input.StringParserInput;
import org.eclipse.statet.jcommons.text.core.util.HtmlStripParserInput;

import org.eclipse.statet.internal.rhelp.core.REnvHelpImpl;
import org.eclipse.statet.internal.rhelp.core.RHelpCoreInternals;
import org.eclipse.statet.internal.rhelp.core.RHelpKeywordGroupImpl;
import org.eclipse.statet.internal.rhelp.core.RHelpKeywordImpl;
import org.eclipse.statet.internal.rhelp.core.RHelpManagerIntern;
import org.eclipse.statet.internal.rhelp.core.RHelpPageImpl;
import org.eclipse.statet.internal.rhelp.core.RPkgHelpImpl;
import org.eclipse.statet.internal.rhelp.core.SerUtil;
import org.eclipse.statet.rhelp.core.DocResource;
import org.eclipse.statet.rhelp.core.REnvHelpConfiguration;
import org.eclipse.statet.rhelp.core.RHelpCore;
import org.eclipse.statet.rhelp.core.RHelpKeywordGroup;
import org.eclipse.statet.rhelp.core.RHelpKeywordNode;
import org.eclipse.statet.rhelp.core.RPkgHelp;
import org.eclipse.statet.rj.renv.core.RLibLocation;
import org.eclipse.statet.rj.renv.core.RNumVersion;
import org.eclipse.statet.rj.renv.core.RPkgDescription;


@NonNullByDefault
public class REnvIndexWriter implements REnvIndexSchema {
	
	
	public static class RdItem {
		
		
		private final String pkg;
		private final String name;
		
		private final List<String> topics= new ArrayList<>(8);
		private ImList<String> imTopics;
		private String title= ""; //$NON-NLS-1$
		private @Nullable List<String> keywords;
		private @Nullable List<String> concepts;
		
		private @Nullable String html;
		
		private @Nullable String descrTxt;
		private @Nullable String mainTxt;
		private @Nullable String examplesTxt;
		
		
		public RdItem(final String pkg, final String name) {
			this.pkg= pkg;
			this.name= name.intern();
		}
		
		
		void finish() {
			this.imTopics= ImCollections.toList(this.topics);
		}
		
		
		public String getPkg() {
			return this.pkg;
		}
		
		public String getName() {
			return this.name;
		}
		
		private ImList<String> getTopics() {
			return this.imTopics;
		}
		
		public void addTopic(final String alias) {
			if (!this.topics.contains(alias)) {
				this.topics.add(alias.intern());
			}
		}
		
		private String getTitle() {
			return this.title;
		}
		
		public void setTitle(final @Nullable String title) {
			if (title != null) {
				this.title= title;
			}
		}
		
		public void addKeyword(final String keyword) {
			List<String> keywords= this.keywords;
			if (keywords == null) {
				keywords= new ArrayList<>(8);
				this.keywords= keywords;
			}
			if (!keywords.contains(keyword)) {
				keywords.add(keyword);
			}
		}
		
		public boolean isInternal() {
			final List<String> keywords= this.keywords;
			return (keywords != null && keywords.contains(RHelpPageImpl.INTERNAL_KEYWORD));
		}
		
		public void addConcept(final String concept) {
			if (this.concepts == null) {
				this.concepts= new ArrayList<>(8);
			}
			this.concepts.add(concept);
		}
		
		public void setHtml(final String html) {
			this.html= html;
		}
		
	}
	
	
	public static final Collection<String> IGNORE_PKG_NAMES;
	static {
		IGNORE_PKG_NAMES= new ArrayList<>();
		IGNORE_PKG_NAMES.add("translations"); //$NON-NLS-1$
		
		String s= System.getProperty("org.eclipse.statet.r.rhelp.PkgsToExclude.names"); //$NON-NLS-1$
		if (s != null && !(s= s.trim()).isEmpty()) {
			for (final String name : s.split(",")) { //$NON-NLS-1$
				if (!name.isEmpty() && !IGNORE_PKG_NAMES.contains(name)) {
					IGNORE_PKG_NAMES.add(name);
				}
			}
		}
	}
	
	
	private static Analyzer WRITE_ANALYZER= new WriteAnalyzer();
	
	
	/** Worker for packages. Single thread! */
	class PackageWorker {
		
		
		private final FlagField doctypeField_PKG_DESCRIPTION= new FlagField(DOCTYPE_FIELD_NAME, PKG_DESCRIPTION_DOCTYPE);
		private final FlagField doctypeField_PAGE= new FlagField(DOCTYPE_FIELD_NAME, PAGE_DOCTYPE);
		private final NameField packageField= new NameField(PACKAGE_FIELD_NAME);
		private final NameField pageField= new NameField(PAGE_FIELD_NAME);
		private final TxtField titleTxtField= new TxtField(TITLE_TXT_FIELD_NAME);
		private final MultiValueFieldList<NameField> aliasFields=
				MultiValueFieldList.forNameField(ALIAS_FIELD_NAME);
		private final MultiValueFieldList<TxtField> aliasTxtFields=
				MultiValueFieldList.forTxtField(ALIAS_TXT_FIELD_NAME);
		private final TxtField descriptionTxtField= new TxtField(DESCRIPTION_TXT_FIELD_NAME);
		private final TxtField authorsTxtField= new TxtField(AUTHORS_TXT_FIELD_NAME);
		private final TxtField maintainerTxtField= new TxtField(MAINTAINER_TXT_FIELD_NAME);
		private final MultiValueFieldList<TxtField> urlTxtFields=
				MultiValueFieldList.forTxtField(URL_TXT_FIELD_NAME);
		private final MultiValueFieldList<KeywordField> keywordTxtFields=
				MultiValueFieldList.forKeywordField(KEYWORD_FIELD_NAME);
		private final MultiValueFieldList<TxtField> conteptTxtFields=
				MultiValueFieldList.forTxtField(CONCEPT_TXT_FIELD_NAME);
		private final TxtField docTxtField= new TxtField(DOC_TXT_FIELD_NAME);
		private final TxtField.OmitNorm docHtmlField= new TxtField.OmitNorm(DOC_HTML_FIELD_NAME);
		private final TxtField examplesTxtField= new TxtField(EXAMPLES_TXT_FIELD_NAME);
		
		private final StringBuilder tempBuilder= new StringBuilder(65536);
		private final HtmlStripParserInput tempHtmlInput= new HtmlStripParserInput(
				new StringParserInput(0x800), 0x800 );
		
		
		public PackageWorker() {
		}
		
		
		private void addToLucene(final RPkgDescription item) throws CorruptIndexException, IOException {
			final Document doc= new Document();
			doc.add(this.doctypeField_PKG_DESCRIPTION);
			this.packageField.setStringValue(item.getName());
			doc.add(this.packageField);
			this.descriptionTxtField.setStringValue(item.getDescription());
			doc.add(this.descriptionTxtField);
			if (item.getAuthor() != null) {
				this.authorsTxtField.setStringValue(item.getAuthor());
				doc.add(this.authorsTxtField);
			}
			if (item.getMaintainer() != null) {
				this.maintainerTxtField.setStringValue(item.getMaintainer());
				doc.add(this.maintainerTxtField);
			}
			{	final ImList<String> urls= item.getUrls();
				for (int i= 0; i < urls.size(); i++) {
					final TxtField txtField= this.urlTxtFields.get(i);
					txtField.setStringValue(urls.get(i));
					doc.add(txtField);
				}
			}
			REnvIndexWriter.this.luceneWriter.addDocument(doc);
		}
		
		private void addToLucene(final RdItem item) throws CorruptIndexException, IOException {
			final Document doc= new Document();
			doc.add(this.doctypeField_PAGE);
			this.packageField.setStringValue(item.getPkg());
			doc.add(this.packageField);
			this.pageField.setStringValue(item.getName());
			doc.add(this.pageField);
			this.titleTxtField.setStringValue(item.getTitle());
			doc.add(this.titleTxtField);
			{	final ImList<String> topics= item.getTopics();
				for (int i= 0; i < topics.size(); i++) {
					final NameField nameField= this.aliasFields.get(i);
					nameField.setStringValue(topics.get(i));
					doc.add(nameField);
					final TxtField txtField= this.aliasTxtFields.get(i);
					txtField.setStringValue(topics.get(i));
					doc.add(txtField);
				}
			}
			if (item.keywords != null) {
				final List<String> keywords= item.keywords;
				for (int i= 0; i < keywords.size(); i++) {
					final KeywordField field= this.keywordTxtFields.get(i);
					field.setStringValue(keywords.get(i));
					doc.add(field);
				}
			}
			if (item.concepts != null) {
				final List<String> concepts= item.concepts;
				for (int i= 0; i < concepts.size(); i++) {
					final TxtField txtField= this.conteptTxtFields.get(i);
					txtField.setStringValue(concepts.get(i));
					doc.add(txtField);
				}
			}
			if (item.html != null) {
				createSectionsTxt(item);
				if (item.descrTxt != null) {
					this.descriptionTxtField.setStringValue(item.descrTxt);
					doc.add(this.descriptionTxtField);
				}
				this.docTxtField.setStringValue(item.mainTxt);
				doc.add(this.docTxtField);
				if (item.examplesTxt != null) {
					this.examplesTxtField.setStringValue(item.examplesTxt);
					doc.add(this.examplesTxtField);
				}
				this.docHtmlField.setStringValue(item.html);
				doc.add(this.docHtmlField);
			}
			REnvIndexWriter.this.luceneWriter.addDocument(doc);
		}
		
		private void createSectionsTxt(final RdItem item) throws IOException {
			String html= item.html;
			this.tempBuilder.setLength(0);
			{	final int idx1= html.indexOf("</h2>"); //$NON-NLS-1$
				if (idx1 >= 0) {
					html= html.substring(idx1 + 5);
				}
			}
			{	final int idx1= indexOfLastHr(html);
				if (idx1 >= 0) {
					html= html.substring(0, idx1);
				}
			}
			{	int idxBegin= html.indexOf("<h3 id=\"description\""); //$NON-NLS-1$
				if (idxBegin >= 0) {
					idxBegin= html.indexOf('>', idxBegin + 20);
					if (idxBegin >= 0) {
						idxBegin= html.indexOf("</h3>", idxBegin + 1); //$NON-NLS-1$
						if (idxBegin >= 0) {
							idxBegin+= 5;
							final int idxEnd= indexOfSectionEnd(html, idxBegin);
							if (idxEnd >= 0) {
								item.descrTxt= html2txt(html.substring(idxBegin, idxEnd));
								html= html.substring(idxEnd);
							}
						}
					}
				}
			}
			{	final String[] s= new String[] { html, null };
				if (extract(s, "<h3 id=\"examples\"")) { //$NON-NLS-1$
					item.examplesTxt= html2txt(s[1]);
				}
				item.mainTxt= html2txt(s[0]);
			}
		}
		
		private boolean extract(final String[] s, final String h3) {
			final String html= s[0];
			final int idx0= html.indexOf(h3);
			if (idx0 >= 0) {
				int idxBegin= html.indexOf('>', idx0 + h3.length());
				if (idxBegin >= 0) {
					idxBegin= html.indexOf("</h3>", idxBegin + 1); //$NON-NLS-1$
					if (idxBegin >= 0) {
						idxBegin+= 5;
						final int idxEnd= indexOfSectionEnd(html, idxBegin);
						if (idxEnd >= 0) {
							this.tempBuilder.setLength(0);
							this.tempBuilder.append(html, 0, idx0);
							this.tempBuilder.append(html, idxEnd, html.length());
							s[0]= this.tempBuilder.toString();
							s[1]= html.substring(idxBegin, idxEnd);
						}
						else {
							s[0]= html.substring(0, idx0);
							s[1]= html.substring(idxBegin, html.length());
						}
						return true;
					}
				}
			}
			return false;
		}
		
		private String html2txt(final String html) {
			this.tempBuilder.setLength(0);
			((StringParserInput) this.tempHtmlInput.getSource()).reset(html);
			this.tempHtmlInput.init();
			int c;
			boolean blank= true;
			while ((c= this.tempHtmlInput.get(0)) >= 0) {
				if (c <= 0x20) {
					if (!blank) {
						blank= true;
						this.tempBuilder.append(' ');
					}
				}
				else {
					if (blank) {
						blank= false;
					}
					this.tempBuilder.append((char) c);
				}
				this.tempHtmlInput.consume(1);
			}
			c= this.tempBuilder.length();
			return (c > 0 && this.tempBuilder.charAt(c - 1) == ' ') ?
					this.tempBuilder.substring(0, c - 1) : this.tempBuilder.toString();
		}
		
	}
	
	
	private final RHelpManagerIntern rHelpManager;
	private final REnvHelpConfiguration rEnvConfig;
	
	private long stamp;
	
	private @Nullable String docDir;
	private @Nullable Path docDirPath;
	private final List<DocResource> manuals= new ArrayList<>();
	private final List<DocResource> miscRes= new ArrayList<>();
	
	private Map<String, RPkgHelp> existingPackages;
	private Map<String, @Nullable RPkgHelp> packages;
	private LinkedHashMap<String, RHelpKeywordGroupImpl> keywordGroups;
	
	private final Path indexDirectory;
	private FSDirectory luceneDirectory;
	private IndexWriter luceneWriter;
	
	private @Nullable RPkgHelpImpl currentPkg;
	private final List<RHelpPageImpl> currentPkgPages= new ArrayList<>(1024);
	
	private @Nullable Object indexLock;
	
	private @Nullable Map<String, String> rEnvSharedProperties;
	
	private boolean reset;
	
	// At moment we use a single worker; multi-threading is not worth, because R is the bottleneck.
	// For multi-threading: thread pool / jobs with worker / thread, currentPackage to worker, ...
	private final PackageWorker worker= new PackageWorker();
	
	private @Nullable MultiStatus status;
	
	
	public REnvIndexWriter(final REnvHelpConfiguration rEnvConfig,
			final RHelpManagerIntern rHelpManager) {
		this.rEnvConfig= rEnvConfig;
		this.rHelpManager= rHelpManager;
		
		this.indexDirectory= SerUtil.getIndexDirectoryChecked(rEnvConfig);
	}
	
	
	public void log(final Status status) {
		final MultiStatus multiStatus= this.status;
		if (multiStatus != null) {
			multiStatus.add(status);
		}
		else {
			RHelpCoreInternals.log(status);
		}
	}
	
	public void beginBatch(final boolean reset) throws AbortIndexOperationException, StatusException {
		if (this.luceneWriter != null) {
			throw new IllegalStateException();
		}
		
		this.status= new MultiStatus(RHelpCore.BUNDLE_ID,
				String.format("Indexing: '%1$s'.", this.rEnvConfig.getName()) ); //$NON-NLS-1$
		
		try {
			this.indexLock= this.rHelpManager.beginIndexUpdate(this.rEnvConfig.getREnv());
			if (this.indexLock == null) {
				final Status status= new CancelStatus(RHelpCore.BUNDLE_ID,
						"Indexing is already running in another task." );
				this.status.add(status);
				throw new StatusException(status);
			}
			
			this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID,
					String.format("Beginning batch (index directory= '%1$s').",
							this.indexDirectory.toString() )));
			
			this.luceneDirectory= REnvIndexUtils.getDirectory(this.indexDirectory);
			
			this.reset= true;
			if (!reset) {
				final REnvHelpImpl oldHelp= this.rHelpManager.getHelpIntern(this.rEnvConfig.getREnv());
				try {
					if (oldHelp != null && oldHelp.getVersion() == SerUtil.CURRENT_VERSION) {
						try (final IndexReader dirReader= DirectoryReader.open(this.luceneDirectory)) {
							this.existingPackages= new HashMap<>(64);
							for (final LeafReaderContext leave : dirReader.leaves()) {
								final LeafReader aReader= leave.reader();
								final Terms terms= aReader.terms(PACKAGE_FIELD_NAME);
								if (terms != null) {
									BytesRef term;
									for (final TermsEnum termsEnum= terms.iterator();
											((term= termsEnum.next()) != null); ) {
										final String name= term.utf8ToString();
										final RPkgHelp pkgHelp= (oldHelp != null) ? oldHelp.getPkgHelp(name) : null;
										this.existingPackages.put(name, pkgHelp);
									}
								}
							}
							
							final IndexWriterConfig config= createWriterConfig();
							config.setOpenMode(OpenMode.CREATE_OR_APPEND);
							this.luceneWriter= new IndexWriter(this.luceneDirectory, config);
							this.reset= false;
						}
						catch (final IOException e) {
							assert (this.luceneWriter == null);
							// try again new
						}
					}
				}
				finally {
					if (oldHelp != null) {
						oldHelp.unlock();
					}
				}
			}
			
			if (this.luceneWriter == null) {
				this.existingPackages= new HashMap<>(0);
				
				try {
					final IndexWriterConfig config= createWriterConfig();
					config.setOpenMode(OpenMode.CREATE);
					this.luceneWriter= new IndexWriter(this.luceneDirectory, config);
				}
				catch (final IndexFormatTooOldException e) {
					if (Files.exists(this.indexDirectory)) {
						FileUtils.cleanDirectory(this.indexDirectory);
						final IndexWriterConfig config= createWriterConfig();
						config.setOpenMode(OpenMode.CREATE);
						this.luceneWriter= new IndexWriter(this.luceneDirectory, config);
					}
					else {
						throw e;
					}
				}
			}
			
			this.stamp= REnvHelpImpl.createStamp();
			
			this.packages= new LinkedHashMap<>();
			this.keywordGroups= new LinkedHashMap<>();
		}
		catch (final IOException e) {
			throw new AbortIndexOperationException(e);
		}
		catch (final OutOfMemoryError e) {
			throw new AbortIndexOperationException(e);
		}
	}
	
	private IndexWriterConfig createWriterConfig() {
		final IndexWriterConfig config= new IndexWriterConfig(WRITE_ANALYZER);
		config.setSimilarity(SIMILARITY);
		config.setRAMPerThreadHardLimitMB(512);
		config.setCommitOnClose(false);
		return config;
	}
	
	public String getDocDir() {
		return this.docDir;
	}
	
	public void setDocDir(@Nullable String docDir, final @Nullable Path docDirPath) {
		if (docDir != null && docDir.isEmpty()) {
			docDir= null;
		}
		this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID,
				String.format("Setting doc dir to: %1$s.", //$NON-NLS-1$
						(docDir != null) ? ('\'' + docDir + '\'') : "<missing>" ))); //$NON-NLS-1$
		this.docDir= docDir;
		this.docDirPath= docDirPath;
	}
	
	public void addManual(final DocResource res) {
		this.manuals.add(res);
	}
	
	public void addManuals(final Collection<DocResource> resources) {
		this.manuals.addAll(resources);
	}
	
	public void addMiscResource(final DocResource res) {
		this.miscRes.add(res);
	}
	
	public void addMiscResources(final Collection<DocResource> resources) {
		this.miscRes.addAll(resources);
	}
	
	public void setREnvSharedProperties(final @Nullable Map<String, String> properties) {
		this.rEnvSharedProperties= properties;
	}
	
	
	public void addDefaultKeyword(final String[] path, final String description) {
		if (path == null || path.length == 0) {
			return;
		}
		if (path.length == 1) { // group
			final String key= path[0].trim().intern();
			if (key.length() > 0) {
				this.keywordGroups.put(key, new RHelpKeywordGroupImpl(key, description));
			}
			return;
		}
		else {
			RHelpKeywordNode node= this.keywordGroups.get(path[0]);
			int i= 1;
			while (node != null) {
				if (i == path.length - 1) {
					if (path[i].length() > 0) {
						final String key= path[i].intern();
						node.getNestedKeywords().add(new RHelpKeywordImpl(key, description));
						return;
					}
				}
				else {
					node= node.getNestedKeyword(path[i++]);
					continue;
				}
			}
			return;
		}
	}
	
	/**
	 * 
	 * @param name package name
	 * @param version
	 * @return <code>true</code> if seems OK, otherwise false
	 */
	public boolean checkPackage(final String name, final RNumVersion version, final String built,
			final RLibLocation libLocation) {
		if (IGNORE_PKG_NAMES.contains(name)) {
			return true;
		}
		synchronized (this.packages) {
			if (this.packages.containsKey(name)) {
				return true;
			}
			RPkgHelp pkgHelp= this.existingPackages.remove(name);
			if (!this.reset && pkgHelp != null
					&& version.equals(pkgHelp.getVersion())
					&& built.equals(pkgHelp.getPkgDescription().getBuilt()) ) {
				if (libLocation != pkgHelp.getPkgDescription().getLibLocation()) {
					pkgHelp= new RPkgHelpImpl(pkgHelp, libLocation);
				}
				this.packages.put(name, pkgHelp); // reuse
				return true;
			}
			else {
				this.packages.put(name, null); // placeholder
				return false;
			}
		}
	}
	
	public void beginPackage(final RPkgDescription pkgDesription)
			throws AbortIndexOperationException {
		final String name= pkgDesription.getName();
		if (this.currentPkg != null) {
			throw new IllegalArgumentException();
		}
		try {
			this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID,
					String.format("Beginning package: '%1$s'.", name) )); //$NON-NLS-1$
			
			this.currentPkg= new RPkgHelpImpl(pkgDesription, this.rEnvConfig.getREnv());
			synchronized (this.packages) {
				this.existingPackages.remove(name);
				this.packages.put(name, this.currentPkg);
			}
			this.currentPkgPages.clear();
			this.luceneWriter.deleteDocuments(new Term(PACKAGE_FIELD_NAME, name));
			this.worker.addToLucene(pkgDesription);
		}
		catch (final IOException e) {
			throw new AbortIndexOperationException(e);
		}
		catch (final OutOfMemoryError e) {
			throw new AbortIndexOperationException(e);
		}
	}
	
	public void add(final RdItem item) throws AbortIndexOperationException {
		final RPkgHelpImpl currentPkg= this.currentPkg;
		if (currentPkg == null || !currentPkg.getName().equals(item.getPkg())) {
			throw new IllegalArgumentException();
		}
		try {
			item.finish();
			this.currentPkgPages.add(new RHelpPageImpl(currentPkg, item.getName(),
					RHelpPageImpl.createFlags(item.isInternal()),
					item.getTopics(),
					item.getTitle() ));
			this.worker.addToLucene(item);
		}
		catch (final IOException e) {
			throw new AbortIndexOperationException(e);
		}
		catch (final OutOfMemoryError e) {
			throw new AbortIndexOperationException(e);
		}
	}
	
	public void endPackage() throws AbortIndexOperationException {
		if (DEBUG) {
			this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, "Finishing package.")); //$NON-NLS-1$
			
			final Runtime runtime= Runtime.getRuntime();
			final long maxMemory= runtime.maxMemory();
			final long allocatedMemory= runtime.totalMemory();
			final long freeMemory= runtime.freeMemory();
			final LiveIndexWriterConfig config= this.luceneWriter.getConfig();
			final StringBuilder sb= new StringBuilder("Memory status:\n"); //$NON-NLS-1$
			sb.append("TempBuilder-capycity: ").append(this.worker.tempBuilder.capacity()).append('\n'); //$NON-NLS-1$
			sb.append("Lucene-buffersize: ").append((long) (config.getRAMBufferSizeMB() * 1024.0)).append('\n'); //$NON-NLS-1$
			sb.append("Memory-free: ").append(freeMemory / 1024L).append('\n'); //$NON-NLS-1$
			sb.append("Memory-total: ").append(allocatedMemory / 1024L).append('\n'); //$NON-NLS-1$
			sb.append("Memory-max: ").append(maxMemory / 1024L).append('\n'); //$NON-NLS-1$
			this.status.add(new InfoStatus(RHelpCore.BUNDLE_ID, sb.toString()));
		}
		
		if (this.currentPkg == null) {
			return;
		}
		
		this.currentPkg.setPages(ImCollections.toList(this.currentPkgPages));
		this.currentPkg= null;
	}
	
	public @Nullable Status endBatch() throws AbortIndexOperationException {
		if (this.luceneWriter == null) {
			return null;
		}
		
		final MultiStatus status= this.status;
		this.status= null;
		if (status != null) {
			status.add(new InfoStatus(RHelpCore.BUNDLE_ID, "Finishing batch.")); //$NON-NLS-1$
			RHelpCoreInternals.log(status);
		}
		try {
			for (final String pkgName : this.existingPackages.keySet()) {
				this.luceneWriter.deleteDocuments(new Term(PACKAGE_FIELD_NAME, pkgName));
			}
			this.existingPackages.clear();
			
			final ImList<DocResource> manuals= checkDocResources(this.manuals, this.docDirPath);
			final ImList<DocResource> miscRes= checkDocResources(this.miscRes, this.docDirPath);
			
			final ImList<RHelpKeywordGroup> keywords;
			{	final Collection<RHelpKeywordGroupImpl> values= this.keywordGroups.values();
				for (final RHelpKeywordGroupImpl group : values) {
					group.freeze();
				}
				keywords= ImCollections.<RHelpKeywordGroup>toList(values);
			}
			final ImList<RPkgHelp> packages;
			{	for (final Iterator<RPkgHelp> iter= this.packages.values().iterator(); iter.hasNext(); ) {
					if (iter.next() == null) {
						iter.remove();
					}
				}
				final RPkgHelp[] array= this.packages.values().toArray(new RPkgHelp[this.packages.size()]);
				Arrays.sort(array);
				packages= ImCollections.newList(array);
			}
			
			final REnvHelpImpl help= new REnvHelpImpl(this.rEnvConfig.getREnv(),
					SerUtil.CURRENT_VERSION, this.stamp,
					this.docDir, manuals, miscRes,
					keywords, packages );
			
//			this.luceneWriter.maybeMerge();
			this.luceneWriter.commit();
			
			this.rHelpManager.updateLocalHelp(this.rEnvConfig, this.rEnvSharedProperties, help);
			
			this.luceneWriter.close();
			this.luceneWriter= null;
			
			if (status != null && status.getSeverity() >= Status.WARNING) {
				return status;
			}
			return null;
		}
		catch (final IOException e) {
			cancel();
			throw new AbortIndexOperationException(e);
		}
		catch (final OutOfMemoryError e) {
			throw new AbortIndexOperationException(e);
		}
		finally {
			clear();
		}
	}
	
	public @Nullable Status cancel() {
		final MultiStatus status;
		try {
			if (this.luceneWriter != null) {
				try {
					this.luceneWriter.rollback();
				}
				catch (final Exception e) {
					if (this.status != null) {
						this.status.add(new ErrorStatus(RHelpCore.BUNDLE_ID,
								"Error when rolling back.", null )); //$NON-NLS-1$
					}
				}
				
				this.luceneWriter.close();
				this.luceneWriter= null;
			}
		}
		catch (final Exception close) {
		}
		finally {
			clear();
			
			status= this.status;
			this.status= null;
			if (status != null) {
				status.add(new InfoStatus(RHelpCore.BUNDLE_ID, "Canceling batch.")); //$NON-NLS-1$
				RHelpCoreInternals.log(status);
			}
		}
		if (status != null && status.getSeverity() >= Status.WARNING) {
			return status;
		}
		return null;
	}
	
	private void clear() {
		if (this.luceneWriter != null) {
			try {
				if (this.luceneWriter.isOpen()) {
					this.luceneWriter.close();
				}
			} catch (final Exception ignore) {}
		}
		this.luceneWriter= null;
		
		if (this.indexLock != null) {
			this.rHelpManager.endIndexUpdate(this.indexLock);
			this.indexLock= null;
		}
		
		this.luceneDirectory= null;
		this.currentPkg= null;
		this.currentPkgPages.clear();
		this.indexLock= null;
	}
	
	
	private ImList<DocResource> checkDocResources(final List<DocResource> resources,
			final @Nullable Path basePath) {
		if (basePath == null) {
			ImCollections.emptyList();
		}
		for (final ListIterator<DocResource> iter= resources.listIterator(); iter.hasNext();) {
			final DocResource res= iter.next();
			if (Files.isRegularFile(basePath.resolve(res.getPath()))) {
				if (res.getPdfPath() != null
						&& !Files.isRegularFile(basePath.resolve(res.getPdfPath())) ) {
					iter.set(new DocResource(res.getTitle(), res.getPath(), null));
				}
			}
			else {
				iter.remove();
			}
		}
		return ImCollections.toList(resources);
	}
	
}