plugins/org.eclipse.epf.publishing/src/org/eclipse/epf/publishing/services/search/IndexManager.java - gerrit/epf/org.eclipse.epf.archive - Git at Google

 //------------------------------------------------------------------------------
 // Copyright (c) 2005, 2006 IBM Corporation and others.
 // All rights reserved. This program and the accompanying materials
 // are made available under the terms of the Eclipse Public License v1.0
 // which accompanies this distribution, and is available at
 // http://www.eclipse.org/legal/epl-v10.html
 //
 // Contributors:
 // IBM Corporation - initial implementation
 //------------------------------------------------------------------------------
 package org.eclipse.epf.publishing.services.search;

 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.List;

 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriter;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.epf.publishing.PublishingResources;
 import org.eclipse.epf.search.analysis.TextAnalyzer;

 /**
  * This class is the main class that creates the Index from the file
  * associations in the process layout.
  */
 public class IndexManager
 {
 	public static final String VERSION_FILE_NAME = "version.txt"; //$NON-NLS-1$
 	public static final String VERSION_DELIMITER = "*"; //$NON-NLS-1$
 	public static final String STOPWORDS_FILE = "search_stopwords.cfg"; //$NON-NLS-1$

 	public static boolean createIndex(String publishDir, IProgressMonitor monitor)
 		throws ClassNotFoundException, java.io.IOException
 	{
 		if ( monitor != null && monitor.isCanceled() )
 		{
 			return false;
 		}

 		String siteName = publishDir.replace(File.separatorChar, '/');
 		int index = siteName.length();
 		if ( siteName.endsWith("/") ) //$NON-NLS-1$
 		{
 			index = index -1;
 		}

 		int index2 = siteName.lastIndexOf("/", index-1); //$NON-NLS-1$
 		siteName = siteName.substring(index2+1, index);

 		// create the index
 		String searchFolder = publishDir;
 		if ( !searchFolder.endsWith(File.separator) )
 		{
 			searchFolder += File.separator;
 		}

 		searchFolder += "applet" + File.separator + "search"; //$NON-NLS-1$ //$NON-NLS-2$

 		String indexFolder =  searchFolder + File.separator + "index"; //$NON-NLS-1$

 		// this does not match with the search applet format
 		//TNGSearcher searcher = new TNGSearcher(publishDir, indexFolder);
 		//searcher.index(monitor, true);
 		if(monitor!=null)
 		{
 			monitor.worked(1);
 		}

 		// using the old rup way to generate search index, or fix the search applet
 		// to do later
 		List activeFiles = new ArrayList();
 		iterateFiles(new File(publishDir), activeFiles, monitor);

 		if ( monitor != null && monitor.isCanceled() )
 		{
 			return false;
 		}

 		index( publishDir, activeFiles, indexFolder, monitor, true);

 		if ( monitor != null && monitor.isCanceled() )
 		{
 			return false;
 		}

 	    File stopwordsFile = new File(searchFolder + File.separator + STOPWORDS_FILE);
 	    FileReader reader = new FileReader(stopwordsFile);
 	    BufferedReader bReader = new BufferedReader(reader);

 	    File newStFile = new File(indexFolder + File.separator + STOPWORDS_FILE);
 	    FileWriter writer = new FileWriter(newStFile);
 	    BufferedWriter bWriter = new BufferedWriter(writer);

 		String temp = bReader.readLine();
 		while (temp != null)
 		{
 			bWriter.write(temp + "\n"); //$NON-NLS-1$
 			temp = bReader.readLine();
 		}

 		bWriter.close();
 		writer.close();
 		bReader.close();
 		reader.close();

 	    System.out.println("index created successfully"); //$NON-NLS-1$

 		if ( monitor != null && monitor.isCanceled() )
 		{
 			return false;
 		}

 	    // jar up the created index.
 	    JarCreator.jarFolder(indexFolder);

 	    System.out.println("index Jarred successfully"); //$NON-NLS-1$

 	    // delete the files now that they've been jarred.
 	    File indexDir = new File(indexFolder);
 	    File[] files = indexDir.listFiles();
 	    for (int i=0; i<files.length; i++)
 	    {
 	    	File tempFile = files[i];
 	    	if (!tempFile.getName().equals(JarCreator.INDEX_JAR))
 	    	{
 	    		tempFile.delete();
 	    	}
 	    }

 		// create the version file.
 		Date today = new Date();
 		long milliseconds = today.getTime();
 		//int index = publishDir.lastIndexOf(File.separator) + 1;
 		//if (index > 0)
 		{
 			//String rupName = publishDir.substring(index);
 			File newIndexJar = new File(indexFolder + File.separator + JarCreator.INDEX_JAR);
 			if (newIndexJar.exists())
 			{
 				String fileSize = "" + newIndexJar.length(); //$NON-NLS-1$
 				FileWriter fw = new FileWriter(indexFolder + File.separator + VERSION_FILE_NAME);
 				BufferedWriter bw = new BufferedWriter(fw);
 				bw.write(siteName + VERSION_DELIMITER + milliseconds + VERSION_DELIMITER +  fileSize + "\n"); //$NON-NLS-1$
 				bw.close();
 				fw.close();
 			}
 			else
 			{
 				throw new IOException(PublishingResources.getString("Publishing.createSearchIndexError.msg")); //$NON-NLS-1$
 			}
 		}
 		/*
 		else
 		{
 			throw new IOException("Error creating index.  Path to rup doesn't have recognized folder structure.");
 		}
 		*/
 		return true;
 	}


 	public static void index( String fileDirectory,
 			   List activeFiles,
 			   String indexName,
 			   IProgressMonitor monitor,
 			   boolean isNewIndex )
 	{
 		// first get the file list from the process layout
 		Iterator fileList = activeFiles.iterator();

 		// gets the stop words
 		String[] stopwords = StopWordReader.getStopwords( fileDirectory + File.separator + "applet" + File.separator + "search", //$NON-NLS-1$ //$NON-NLS-2$
 		                                               STOPWORDS_FILE );
 		if( isNewIndex )
 		{
 			try
 			{
 				//System.out.println("index name = " + indexName);
 				// create the Lucene index writer
 				// the chinese analyzer here will work with japanese, korean as well as
 				// latin based languages such as English
 				IndexWriter writer = new IndexWriter( indexName,
 				                                      //new StopAnalyzer( stopwords ),
 				                                      new TextAnalyzer(),
 				                                      isNewIndex );
 				// index the files
 				if ((writer != null) && (fileList != null))
 				{
 					indexDocs( writer, fileList, monitor);
 				}

 				writer.optimize();
 				writer.close();
 			}
 			catch( Exception e )
 			{
 				e.printStackTrace();
 			}
 		}

 	}

 	/**
 	 * Index the actual documents specified by the files.
 	 */
   	protected static void indexDocs( IndexWriter writer,
   	                          Iterator fileList,
   	                        IProgressMonitor monitor)
   		throws IOException
     {
     	//System.out.println("Start the creation of index");
     	//java.util.Date now = new java.util.Date();
     	System.gc();
     	while( fileList.hasNext())
     	{
     		if ( monitor != null && monitor.isCanceled() )
     		{
     			return;
     		}

     		//System.gc();
 			File file = (File)fileList.next();

     		try
     		{
 	    		//le file = (File)fileList.next();
     			Document doc = DocumentFactory.document(file);

     			if (doc != null)
 	    		{
 		    		//writer.addDocument( DocumentFactory.document( file ) );
 		    		writer.addDocument( doc );
     			}

 				if(monitor != null)
 				{
 					monitor.worked(1);
 				}
     		}
     		catch (Exception e1)
     		{
 				System.out.println( file.getName());
     			e1.printStackTrace();
     		}
   		}
     	//java.util.Date later = new java.util.Date();
     	//double time = (later.getTime() - now.getTime())/1000.00;
     	//System.out.println( "took " + time + " to index..............." );
   		//System.out.println("Finished creation of index");
     }

 	/**
 	 * A recursively get all file in the specified folder file
 	 */
 	private static void iterateFiles(File f, List files, IProgressMonitor monitor)
 	{
 		if ( monitor != null && monitor.isCanceled() )
 		{
 			return;
 		}

 		if (f.isDirectory())
 		{
 			if(f.getName().equalsIgnoreCase("applet")) //$NON-NLS-1$
 				return;

 			String[] kids = f.list();
 			for (int i = 0; i < kids.length; i++)
 			{
 				if ( monitor != null && monitor.isCanceled() )
 				{
 					return;
 				}
 				iterateFiles(new File(f, kids[i]), files, monitor);
 			}
 		}
 		else
 		{
 			files.add(f);
 		}
 	}
 }
	//------------------------------------------------------------------------------
	// Copyright (c) 2005, 2006 IBM Corporation and others.
	// All rights reserved. This program and the accompanying materials
	// are made available under the terms of the Eclipse Public License v1.0
	// which accompanies this distribution, and is available at
	// http://www.eclipse.org/legal/epl-v10.html
	//
	// Contributors:
	// IBM Corporation - initial implementation
	//------------------------------------------------------------------------------
	package org.eclipse.epf.publishing.services.search;

	import java.io.BufferedReader;
	import java.io.BufferedWriter;
	import java.io.File;
	import java.io.FileReader;
	import java.io.FileWriter;
	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.Date;
	import java.util.Iterator;
	import java.util.List;

	import org.apache.lucene.document.Document;
	import org.apache.lucene.index.IndexWriter;
	import org.eclipse.core.runtime.IProgressMonitor;
	import org.eclipse.epf.publishing.PublishingResources;
	import org.eclipse.epf.search.analysis.TextAnalyzer;

	/**
	* This class is the main class that creates the Index from the file
	* associations in the process layout.
	*/
	public class IndexManager
	{
	public static final String VERSION_FILE_NAME = "version.txt"; //$NON-NLS-1$
	public static final String VERSION_DELIMITER = "*"; //$NON-NLS-1$
	public static final String STOPWORDS_FILE = "search_stopwords.cfg"; //$NON-NLS-1$

	public static boolean createIndex(String publishDir, IProgressMonitor monitor)
	throws ClassNotFoundException, java.io.IOException
	{
	if ( monitor != null && monitor.isCanceled() )
	{
	return false;
	}

	String siteName = publishDir.replace(File.separatorChar, '/');
	int index = siteName.length();
	if ( siteName.endsWith("/") ) //$NON-NLS-1$
	{
	index = index -1;
	}

	int index2 = siteName.lastIndexOf("/", index-1); //$NON-NLS-1$
	siteName = siteName.substring(index2+1, index);

	// create the index
	String searchFolder = publishDir;
	if ( !searchFolder.endsWith(File.separator) )
	{
	searchFolder += File.separator;
	}

	searchFolder += "applet" + File.separator + "search"; //$NON-NLS-1$ //$NON-NLS-2$

	String indexFolder = searchFolder + File.separator + "index"; //$NON-NLS-1$

	// this does not match with the search applet format
	//TNGSearcher searcher = new TNGSearcher(publishDir, indexFolder);
	//searcher.index(monitor, true);
	if(monitor!=null)
	{
	monitor.worked(1);
	}

	// using the old rup way to generate search index, or fix the search applet
	// to do later
	List activeFiles = new ArrayList();
	iterateFiles(new File(publishDir), activeFiles, monitor);

	if ( monitor != null && monitor.isCanceled() )
	{
	return false;
	}

	index( publishDir, activeFiles, indexFolder, monitor, true);

	if ( monitor != null && monitor.isCanceled() )
	{
	return false;
	}

	File stopwordsFile = new File(searchFolder + File.separator + STOPWORDS_FILE);
	FileReader reader = new FileReader(stopwordsFile);
	BufferedReader bReader = new BufferedReader(reader);

	File newStFile = new File(indexFolder + File.separator + STOPWORDS_FILE);
	FileWriter writer = new FileWriter(newStFile);
	BufferedWriter bWriter = new BufferedWriter(writer);

	String temp = bReader.readLine();
	while (temp != null)
	{
	bWriter.write(temp + "\n"); //$NON-NLS-1$
	temp = bReader.readLine();
	}

	bWriter.close();
	writer.close();
	bReader.close();
	reader.close();

	System.out.println("index created successfully"); //$NON-NLS-1$

	if ( monitor != null && monitor.isCanceled() )
	{
	return false;
	}

	// jar up the created index.
	JarCreator.jarFolder(indexFolder);

	System.out.println("index Jarred successfully"); //$NON-NLS-1$

	// delete the files now that they've been jarred.
	File indexDir = new File(indexFolder);
	File[] files = indexDir.listFiles();
	for (int i=0; i<files.length; i++)
	{
	File tempFile = files[i];
	if (!tempFile.getName().equals(JarCreator.INDEX_JAR))
	{
	tempFile.delete();
	}
	}

	// create the version file.
	Date today = new Date();
	long milliseconds = today.getTime();
	//int index = publishDir.lastIndexOf(File.separator) + 1;
	//if (index > 0)
	{
	//String rupName = publishDir.substring(index);
	File newIndexJar = new File(indexFolder + File.separator + JarCreator.INDEX_JAR);
	if (newIndexJar.exists())
	{
	String fileSize = "" + newIndexJar.length(); //$NON-NLS-1$
	FileWriter fw = new FileWriter(indexFolder + File.separator + VERSION_FILE_NAME);
	BufferedWriter bw = new BufferedWriter(fw);
	bw.write(siteName + VERSION_DELIMITER + milliseconds + VERSION_DELIMITER + fileSize + "\n"); //$NON-NLS-1$
	bw.close();
	fw.close();
	}
	else
	{
	throw new IOException(PublishingResources.getString("Publishing.createSearchIndexError.msg")); //$NON-NLS-1$
	}
	}
	/*
	else
	{
	throw new IOException("Error creating index. Path to rup doesn't have recognized folder structure.");
	}
	*/
	return true;
	}


	public static void index( String fileDirectory,
	List activeFiles,
	String indexName,
	IProgressMonitor monitor,
	boolean isNewIndex )
	{
	// first get the file list from the process layout
	Iterator fileList = activeFiles.iterator();

	// gets the stop words
	String[] stopwords = StopWordReader.getStopwords( fileDirectory + File.separator + "applet" + File.separator + "search", //$NON-NLS-1$ //$NON-NLS-2$
	STOPWORDS_FILE );
	if( isNewIndex )
	{
	try
	{
	//System.out.println("index name = " + indexName);
	// create the Lucene index writer
	// the chinese analyzer here will work with japanese, korean as well as
	// latin based languages such as English
	IndexWriter writer = new IndexWriter( indexName,
	//new StopAnalyzer( stopwords ),
	new TextAnalyzer(),
	isNewIndex );
	// index the files
	if ((writer != null) && (fileList != null))
	{
	indexDocs( writer, fileList, monitor);
	}

	writer.optimize();
	writer.close();
	}
	catch( Exception e )
	{
	e.printStackTrace();
	}
	}

	}

	/**
	* Index the actual documents specified by the files.
	*/
	protected static void indexDocs( IndexWriter writer,
	Iterator fileList,
	IProgressMonitor monitor)
	throws IOException
	{
	//System.out.println("Start the creation of index");
	//java.util.Date now = new java.util.Date();
	System.gc();
	while( fileList.hasNext())
	{
	if ( monitor != null && monitor.isCanceled() )
	{
	return;
	}

	//System.gc();
	File file = (File)fileList.next();

	try
	{
	//le file = (File)fileList.next();
	Document doc = DocumentFactory.document(file);

	if (doc != null)
	{
	//writer.addDocument( DocumentFactory.document( file ) );
	writer.addDocument( doc );
	}

	if(monitor != null)
	{
	monitor.worked(1);
	}
	}
	catch (Exception e1)
	{
	System.out.println( file.getName());
	e1.printStackTrace();
	}
	}
	//java.util.Date later = new java.util.Date();
	//double time = (later.getTime() - now.getTime())/1000.00;
	//System.out.println( "took " + time + " to index..............." );
	//System.out.println("Finished creation of index");
	}

	/**
	* A recursively get all file in the specified folder file
	*/
	private static void iterateFiles(File f, List files, IProgressMonitor monitor)
	{
	if ( monitor != null && monitor.isCanceled() )
	{
	return;
	}

	if (f.isDirectory())
	{
	if(f.getName().equalsIgnoreCase("applet")) //$NON-NLS-1$
	return;

	String[] kids = f.list();
	for (int i = 0; i < kids.length; i++)
	{
	if ( monitor != null && monitor.isCanceled() )
	{
	return;
	}
	iterateFiles(new File(f, kids[i]), files, monitor);
	}
	}
	else
	{
	files.add(f);
	}
	}
	}