plugins/org.eclipse.epf.publishing/src/org/eclipse/epf/publishing/services/search/IndexManager.java

//------------------------------------------------------------------------------
// Copyright (c) 2005, 2006 IBM Corporation and others.
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// which accompanies this distribution, and is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// Contributors:
// IBM Corporation - initial implementation
//------------------------------------------------------------------------------
package org.eclipse.epf.publishing.services.search;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.epf.publishing.PublishingResources;
import org.eclipse.epf.search.analysis.TextAnalyzer;

/**
 * This class is the main class that creates the Index from the file
 * associations in the process layout.
 */
public class IndexManager {

	public static final String VERSION_FILE_NAME = "version.txt"; //$NON-NLS-1$

	public static final String VERSION_DELIMITER = "*"; //$NON-NLS-1$

	public static final String STOPWORDS_FILE = "search_stopwords.cfg"; //$NON-NLS-1$

	public static boolean createIndex(String publishDir,
			IProgressMonitor monitor) throws ClassNotFoundException,
			java.io.IOException {
		if (monitor != null && monitor.isCanceled()) {
			return false;
		}

		String siteName = publishDir.replace(File.separatorChar, '/');
		int index = siteName.length();
		if (siteName.endsWith("/")) //$NON-NLS-1$
		{
			index = index - 1;
		}

		int index2 = siteName.lastIndexOf("/", index - 1); //$NON-NLS-1$
		siteName = siteName.substring(index2 + 1, index);

		// create the index
		String searchFolder = publishDir;
		if (!searchFolder.endsWith(File.separator)) {
			searchFolder += File.separator;
		}

		searchFolder += "applet" + File.separator + "search"; //$NON-NLS-1$ //$NON-NLS-2$

		String indexFolder = searchFolder + File.separator + "index"; //$NON-NLS-1$

		if (monitor != null) {
			monitor.worked(1);
		}

		List activeFiles = new ArrayList();
		iterateFiles(new File(publishDir), activeFiles, monitor);

		if (monitor != null && monitor.isCanceled()) {
			return false;
		}

		index(publishDir, activeFiles, indexFolder, monitor, true);

		if (monitor != null && monitor.isCanceled()) {
			return false;
		}

		File stopwordsFile = new File(searchFolder + File.separator
				+ STOPWORDS_FILE);
		FileReader reader = new FileReader(stopwordsFile);
		BufferedReader bReader = new BufferedReader(reader);

		File newStFile = new File(indexFolder + File.separator + STOPWORDS_FILE);
		FileWriter writer = new FileWriter(newStFile);
		BufferedWriter bWriter = new BufferedWriter(writer);

		String temp = bReader.readLine();
		while (temp != null) {
			bWriter.write(temp + "\n"); //$NON-NLS-1$
			temp = bReader.readLine();
		}

		bWriter.close();
		writer.close();
		bReader.close();
		reader.close();

		System.out.println("index created successfully"); //$NON-NLS-1$

		if (monitor != null && monitor.isCanceled()) {
			return false;
		}

		// jar up the created index.
		JarCreator.jarFolder(indexFolder);

		System.out.println("index Jarred successfully"); //$NON-NLS-1$

		// delete the files now that they've been jarred.
		File indexDir = new File(indexFolder);
		File[] files = indexDir.listFiles();
		for (int i = 0; i < files.length; i++) {
			File tempFile = files[i];
			if (!tempFile.getName().equals(JarCreator.INDEX_JAR)) {
				tempFile.delete();
			}
		}

		// create the version file.
		Date today = new Date();
		long milliseconds = today.getTime();
		File newIndexJar = new File(indexFolder + File.separator
				+ JarCreator.INDEX_JAR);
		if (newIndexJar.exists()) {
			String fileSize = "" + newIndexJar.length(); //$NON-NLS-1$
			FileWriter fw = new FileWriter(indexFolder + File.separator
					+ VERSION_FILE_NAME);
			BufferedWriter bw = new BufferedWriter(fw);
			bw.write(siteName + VERSION_DELIMITER + milliseconds
					+ VERSION_DELIMITER + fileSize + "\n"); //$NON-NLS-1$
			bw.close();
			fw.close();
		} else {
			throw new IOException(PublishingResources
					.getString("Publishing.createSearchIndexError.msg")); //$NON-NLS-1$
		}
		return true;
	}

	public static void index(String fileDirectory, List activeFiles,
			String indexName, IProgressMonitor monitor, boolean isNewIndex) {
		// first get the file list from the process layout
		Iterator fileList = activeFiles.iterator();

		// gets the stop words
		String[] stopwords = StopWordReader.getStopwords(fileDirectory
				+ File.separator + "applet" + File.separator + "search", //$NON-NLS-1$ //$NON-NLS-2$
				STOPWORDS_FILE);
		if (isNewIndex) {
			try {
				// System.out.println("index name = " + indexName);
				// create the Lucene index writer
				// the chinese analyzer here will work with japanese, korean as
				// well as
				// latin based languages such as English
				IndexWriter writer = new IndexWriter(indexName,
				// new StopAnalyzer( stopwords ),
						new TextAnalyzer(), isNewIndex);
				// index the files
				if ((writer != null) && (fileList != null)) {
					indexDocs(writer, fileList, monitor);
				}

				writer.optimize();
				writer.close();
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

	}

	/**
	 * Index the actual documents specified by the files.
	 */
	protected static void indexDocs(IndexWriter writer, Iterator fileList,
			IProgressMonitor monitor) throws IOException {
		// System.out.println("Start the creation of index");
		// java.util.Date now = new java.util.Date();
		System.gc();
		while (fileList.hasNext()) {
			if (monitor != null && monitor.isCanceled()) {
				return;
			}

			File file = (File) fileList.next();

			try {
				Document doc = DocumentFactory.document(file);

				if (doc != null) {
					writer.addDocument(doc);
				}

				if (monitor != null) {
					monitor.worked(1);
				}
			} catch (Exception e1) {
				System.out.println(file.getName());
				e1.printStackTrace();
			}
		}
		// java.util.Date later = new java.util.Date();
		// double time = (later.getTime() - now.getTime())/1000.00;
		// System.out.println( "took " + time + " to index..............." );
		// System.out.println("Finished creation of index");
	}

	/**
	 * A recursively get all file in the specified folder file
	 */
	private static void iterateFiles(File f, List files,
			IProgressMonitor monitor) {
		if (monitor != null && monitor.isCanceled()) {
			return;
		}

		if (f.isDirectory()) {
			if (f.getName().equalsIgnoreCase("applet")) //$NON-NLS-1$
				return;

			String[] kids = f.list();
			for (int i = 0; i < kids.length; i++) {
				if (monitor != null && monitor.isCanceled()) {
					return;
				}
				iterateFiles(new File(f, kids[i]), files, monitor);
			}
		} else {
			files.add(f);
		}
	}
	
}