| //------------------------------------------------------------------------------ |
| // Copyright (c) 2005, 2007 IBM Corporation and others. |
| // All rights reserved. This program and the accompanying materials |
| // are made available under the terms of the Eclipse Public License v1.0 |
| // which accompanies this distribution, and is available at |
| // http://www.eclipse.org/legal/epl-v10.html |
| // |
| // Contributors: |
| // IBM Corporation - initial implementation |
| //------------------------------------------------------------------------------ |
| package org.eclipse.epf.search; |
| |
| import java.io.BufferedReader; |
| import java.io.BufferedWriter; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileWriter; |
| import java.io.IOException; |
| import java.io.InputStreamReader; |
| import java.io.Reader; |
| import java.util.ArrayList; |
| import java.util.Date; |
| import java.util.Enumeration; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Properties; |
| |
| import org.apache.lucene.analysis.Analyzer; |
| import org.apache.lucene.analysis.cjk.CJKAnalyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.index.IndexWriter; |
| import org.apache.lucene.store.FSDirectory; |
| import org.eclipse.epf.search.analysis.TextAnalyzer; |
| import org.eclipse.epf.search.utils.JarCreator; |
| import org.eclipse.epf.search.utils.LHTMLParser; |
| import org.eclipse.epf.search.utils.UNCUtil; |
| |
| /** |
| * This class is the main class that creates the Index from the file |
| * associations in the process layout. |
| */ |
| public class IndexBuilder { |
| |
| static final String VERSION_FILE_NAME = "version.txt"; //$NON-NLS-1$ |
| static final String VERSION_DELIMITER = "*"; //$NON-NLS-1$ |
| |
| /** |
| * Document fields. |
| */ |
| public static final String BRIEF_DESCRIPTION_FIELD = "briefDescription"; //$NON-NLS-1$ |
| public static final String CONTENT_FIELD = "contents"; //$NON-NLS-1$ |
| public static final String ID_FIELD = "id"; //$NON-NLS-1$ |
| public static final String MODIFIED_FIELD = "modified"; //$NON-NLS-1$ |
| public static final String NAME_FIELD = "name"; //$NON-NLS-1$ |
| public static final String ROLE_FIELD = "role"; //$NON-NLS-1$ |
| public static final String SUMMARY_FIELD = "summary"; //$NON-NLS-1$ |
| public static final String TYPE_FIELD = "type"; //$NON-NLS-1$ |
| public static final String URL_FIELD = "url"; //$NON-NLS-1$ |
| public static final String TITLE_FIELD = "title"; //$NON-NLS-1$ |
| public static final String UMA_ELEMENT_TYPE_FIELD = "uma.type"; //$NON-NLS-1$ |
| public static final String GENERAL_CONTENT = "general_content"; //$NON-NLS-1$ |
| |
| // List of UMA elements that should be included in the search index. |
| private static List NO_SEARCHEABLE_UMA_ELEMENTS = new ArrayList(); |
| static { |
| NO_SEARCHEABLE_UMA_ELEMENTS.add("summary"); //$NON-NLS-1$ |
| NO_SEARCHEABLE_UMA_ELEMENTS.add("workproductdescriptor"); //$NON-NLS-1$ |
| NO_SEARCHEABLE_UMA_ELEMENTS.add("taskdescriptor"); //$NON-NLS-1$ |
| NO_SEARCHEABLE_UMA_ELEMENTS.add("roledescriptor"); //$NON-NLS-1$ |
| } |
| |
| // A list of top level directories that should be excluded from the search |
| // index. |
| public static List dirsToSkip = new ArrayList(); |
| public static String pDirectory = null; |
| private StringBuffer indexFolder = null; |
| private String productName = null; |
| private List filesToSkip = new ArrayList(); |
| private File parentFolder = null; |
| |
| public IndexBuilder(String publishDir) { |
| int appletIndex = -1; |
| if (publishDir == null) |
| return; |
| |
| appletIndex = publishDir.indexOf(File.separator + "applet"); //$NON-NLS-1$ |
| |
| pDirectory = UNCUtil.convertFilename((appletIndex > -1) ? publishDir |
| .substring(0, appletIndex + 1) : publishDir); |
| String siteName = pDirectory.replace(File.separatorChar, '/'); |
| parentFolder = new File(pDirectory); |
| int index = siteName.length(); |
| if (siteName.endsWith("/")) { //$NON-NLS-1$ |
| index = index - 1; |
| } |
| |
| int index2 = siteName.lastIndexOf("/", index - 1); //$NON-NLS-1$ |
| |
| productName = siteName.substring(index2 + 1, index); |
| |
| // create the index |
| StringBuffer searchFolder = new StringBuffer(pDirectory); |
| if (!searchFolder.toString().endsWith(File.separator)) { |
| searchFolder.append(File.separator); |
| } |
| searchFolder.append("search"); //$NON-NLS-1$ |
| |
| indexFolder = new StringBuffer(searchFolder.toString()); |
| indexFolder.append(File.separator).append("index"); //$NON-NLS-1$ |
| |
| dirsToSkip.add(pDirectory + "applet"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "css"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "ext_help"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "icons"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "images"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "index"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "logs"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "manuals"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "noapplet"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "pages_not_installed"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "process"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "scripts"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "stylesheets"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "xml"); //$NON-NLS-1$ |
| dirsToSkip.add(pDirectory + "search"); //$NON-NLS-1$ |
| |
| filesToSkip.add("_desc.htm"); //$NON-NLS-1$ |
| filesToSkip.add("_wbs.htm"); //$NON-NLS-1$ |
| filesToSkip.add("_tbs.htm"); //$NON-NLS-1$ |
| filesToSkip.add("_wpbs.htm"); //$NON-NLS-1$ |
| } |
| |
| public boolean createIndex(boolean jarIt) throws SearchServiceException { |
| synchronized (IndexBuilder.class) { |
| |
| if (indexFolder == null || pDirectory == null) { |
| throw new IllegalStateException( |
| "Invalid indexFolder or pDirectory"); //$NON-NLS-1$ |
| } |
| |
| |
| boolean jako = false; |
| Locale locale = Locale.getDefault(); |
| String lang = locale.getLanguage(); |
| if (lang.equals(Locale.JAPANESE.getLanguage()) || |
| lang.equals(Locale.KOREA.getLanguage())) { |
| jako = true; |
| } |
| Analyzer analyzer = jako ? new CJKAnalyzer() : new TextAnalyzer(); |
| |
| try { |
| // RAMDirectory ramDir = new RAMDirectory(); |
| IndexWriter fsWriter = new IndexWriter(FSDirectory |
| .getDirectory(indexFolder.toString(), true), |
| analyzer, true); |
| |
| // IndexWriter ramWriter = new IndexWriter(ramDir, |
| // new TextAnalyzer(), true); |
| |
| if ((fsWriter != null)) { |
| // fsWriter.mergeFactor = 1000; |
| // fsWriter.maxMergeDocs = 10000; |
| fsWriter.setMaxFieldLength(1000000); |
| |
| indexDocs(new File(pDirectory), fsWriter); |
| |
| // fsWriter.addIndexes(new Directory[] { ramDir }); |
| fsWriter.optimize(); |
| // ramWriter.close(); |
| fsWriter.close(); |
| } |
| } catch (Exception e) { |
| e.printStackTrace(); |
| } |
| |
| // create the version file. |
| Date today = new Date(); |
| long milliseconds = today.getTime(); |
| |
| if (!jarIt) { |
| try { |
| FileWriter fw = new FileWriter(indexFolder + File.separator |
| + VERSION_FILE_NAME); |
| BufferedWriter bw = new BufferedWriter(fw); |
| bw.write(productName + VERSION_DELIMITER + milliseconds |
| + "\n"); //$NON-NLS-1$ |
| if (analyzer instanceof CJKAnalyzer) { |
| bw.write("CJKAnalyzer" + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ |
| } |
| bw.close(); |
| fw.close(); |
| } catch (IOException ioe) { |
| throw new SearchServiceException( |
| SearchResources.createSearchIndexError); |
| } |
| |
| return true; |
| } |
| |
| // jar up the created index. |
| JarCreator.jarFolder(indexFolder.toString()); |
| |
| System.out.println("index Jarred successfully"); //$NON-NLS-1$ |
| |
| try { |
| // delete the files now that they've been jarred. |
| File indexDir = new File(indexFolder.toString()); |
| File[] files = indexDir.listFiles(); |
| for (int i = 0; i < files.length; i++) { |
| File tempFile = files[i]; |
| if (!tempFile.getName().equals(JarCreator.INDEX_JAR)) { |
| tempFile.delete(); |
| } |
| } |
| |
| // String rupName = publishDir.substring(index); |
| File newIndexJar = new File(indexFolder + File.separator |
| + JarCreator.INDEX_JAR); |
| if (newIndexJar.exists()) { |
| String fileSize = "" + newIndexJar.length(); //$NON-NLS-1$ |
| FileWriter fw = new FileWriter(indexFolder + File.separator |
| + VERSION_FILE_NAME); |
| BufferedWriter bw = new BufferedWriter(fw); |
| bw.write(productName + VERSION_DELIMITER + milliseconds |
| + VERSION_DELIMITER + fileSize + "\n"); //$NON-NLS-1$ |
| if (analyzer instanceof CJKAnalyzer) { |
| bw.write("CJKAnalyzer" + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ |
| } |
| bw.close(); |
| fw.close(); |
| } else { |
| throw new SearchServiceException( |
| SearchResources.createSearchIndexError); |
| } |
| } catch (IOException ioe) { |
| throw new SearchServiceException( |
| SearchResources.createSearchIndexError); |
| } |
| |
| return true; |
| } |
| } |
| |
| /** |
| * Index the actual documents specified by the files and recursively get all |
| * file in the specified folder file |
| * |
| */ |
| private void indexDocs(File file, IndexWriter writer) throws Exception { |
| if (dirsToSkip.contains(file.getAbsolutePath())) { |
| return; |
| } |
| if (file.isFile()) { |
| for (Iterator iter = filesToSkip.iterator(); iter.hasNext();) { |
| String fileToSkip = (String) iter.next(); |
| if (file.getName().indexOf(fileToSkip) > -1) { |
| return; |
| } |
| } |
| } |
| |
| if (file.isDirectory()) { |
| String[] files = file.list(); |
| for (int i = 0; i < files.length; i++) { |
| indexDocs(new File(file, files[i]), writer); |
| } |
| } else if (isHtmlDoc(file)) { |
| if (shouldBeExcluded(file)) { |
| return; |
| } |
| try { |
| |
| Document doc = getHTMLDocument(file); |
| |
| if (doc != null) { |
| writer.addDocument(doc); |
| } |
| |
| } catch (Exception e1) { |
| System.out.println(file.getName()); |
| System.out.println("indexDocs"); //$NON-NLS-1$ |
| e1.printStackTrace(); |
| } |
| } |
| } |
| |
| /** |
| * Checks whether the given file should be excluded from the search index. |
| * |
| * @param file |
| * The file to be verified. |
| * @return <code>true</code> if the given file should be excluded from the |
| * search index. |
| */ |
| private boolean shouldBeExcluded(File file) { |
| String path = file.getParentFile().getAbsolutePath(); |
| if (pDirectory.startsWith(path)) { |
| return true; |
| } |
| |
| return false; |
| } |
| |
| private static boolean isHtmlDoc(File file) { |
| String path = file.getPath(); |
| return path.endsWith(".html") || path.endsWith(".htm"); //$NON-NLS-1$ //$NON-NLS-2$ |
| } |
| |
| private boolean isNoSearchableDocument(Properties metaTags) { |
| String value = metaTags.getProperty(UMA_ELEMENT_TYPE_FIELD); |
| |
| // value == null is treated as general document |
| return (value != null) && NO_SEARCHEABLE_UMA_ELEMENTS.contains(value); |
| } |
| |
| char[] cbuf = new char[1024]; |
| int skipCount = 0; |
| |
| private Document getHTMLDocument(File file) { |
| Document luceneDocument = null; |
| InputStreamReader input = null; |
| Reader reader = null; |
| BufferedReader bufferedReader = null; |
| try { |
| |
| input = new InputStreamReader(new FileInputStream(file), "UTF-8"); //$NON-NLS-1$ |
| |
| LHTMLParser parser = new LHTMLParser(input); |
| |
| reader = parser.getReader(); |
| if ( reader == null ) { |
| return null; |
| } |
| |
| StringBuffer htmlContent = new StringBuffer(""); |
| String line = ""; |
| bufferedReader = new BufferedReader(reader); |
| while((line = bufferedReader.readLine()) != null) |
| { |
| htmlContent.append(line + "\n"); |
| } |
| |
| Properties metaTags = parser.getMetaTags(); |
| if ( isNoSearchableDocument(metaTags) ) { |
| |
| // the LHTMLParser thread will not end if the reader is not processed |
| // causing major resource leak |
| // while ( reader.read(cbuf) > 0 ) { |
| // ; |
| // } |
| //System.out.println( ++skipCount + " file skipped: " + file.getAbsolutePath()); |
| parser = null; |
| return null; |
| } |
| |
| luceneDocument = new Document(); |
| |
| String url = productName |
| + file.getPath().substring(parentFolder.getPath().length()) |
| .replace(File.separatorChar, '/'); //$NON-NLS-1$ |
| luceneDocument.add(Field.UnIndexed(URL_FIELD, url)); |
| |
| // luceneDocument.add(Field.Text(CONTENT_FIELD, reader)); |
| luceneDocument.add(Field.UnStored(CONTENT_FIELD, htmlContent.toString())); |
| |
| String title = parser.getTitle(); |
| if (title != null && title.length() > 0) { |
| // Workaround a Linux specific issue. |
| title = title.replaceAll("\\xa0", " "); //$NON-NLS-1$ //$NON-NLS-2$ |
| luceneDocument.add(Field.Keyword(TITLE_FIELD, title)); |
| } else { |
| return null; |
| } |
| |
| String summary = parser.getSummary(); |
| if (summary.startsWith(title) && summary.length() > title.length()) { |
| luceneDocument.add(Field.Keyword(SUMMARY_FIELD, summary |
| .substring(title.length() + 1))); |
| } else |
| luceneDocument.add(Field.Keyword(SUMMARY_FIELD, parser |
| .getSummary())); |
| |
| for (Enumeration names = metaTags.propertyNames(); names |
| .hasMoreElements();) { |
| String tagName = (String) names.nextElement(); |
| if (tagName != null) { |
| if (tagName.equals(ROLE_FIELD)) { |
| String roleName = metaTags.getProperty(tagName); |
| if (roleName != null) { |
| luceneDocument.add(Field.Text(tagName, roleName)); |
| } |
| } else { |
| String tagValue = metaTags.getProperty(tagName); |
| if (tagValue != null) { |
| luceneDocument.add(Field.Text(tagName, tagValue)); |
| } |
| } |
| } |
| } |
| |
| if (luceneDocument.getField(ROLE_FIELD) == null) { |
| // Default to "na" to support searching for files without |
| // role meta tags. |
| luceneDocument.add(Field.Text(ROLE_FIELD, "NORUPROLE")); //$NON-NLS-1$ |
| } |
| |
| Field umaTypeField = luceneDocument |
| .getField(UMA_ELEMENT_TYPE_FIELD); |
| if (umaTypeField == null) { |
| // Default to general content. |
| luceneDocument.add(Field.Text(UMA_ELEMENT_TYPE_FIELD, |
| GENERAL_CONTENT)); |
| } |
| |
| parser = null; |
| } catch (Exception e) { |
| luceneDocument = null; |
| SearchPlugin.getDefault().getLogger().logError(e); |
| } finally { |
| if (bufferedReader != null) { |
| try { |
| bufferedReader.close(); |
| } catch (Exception e) { |
| } |
| } |
| if (input != null) { |
| try { |
| input.close(); |
| } catch (Exception e) { |
| } |
| } |
| } |
| |
| return luceneDocument; |
| } |
| |
| } |
