Changes related to advance search used in Rational Method Composer 7.5.3.
diff --git a/org.eclipse.epf.web.search/.classpath b/org.eclipse.epf.web.search/.classpath index 11c9c2b..e0ed5de 100644 --- a/org.eclipse.epf.web.search/.classpath +++ b/org.eclipse.epf.web.search/.classpath
@@ -1,8 +1,8 @@ -<?xml version="1.0" encoding="UTF-8"?> -<classpath> - <classpathentry kind="src" path="src"/> - <classpathentry kind="lib" path="lib/icu4j_3_4.jar"/> - <classpathentry kind="lib" path="lib/lucene-core-1.9.1.jar"/> - <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> - <classpathentry kind="output" path="bin"/> -</classpath> +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src"/> + <classpathentry kind="lib" path="lib/icu4j_3_4.jar"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/> + <classpathentry kind="lib" path="lib/lucene-core-2.9.1.jar"/> + <classpathentry kind="output" path="bin"/> +</classpath>
diff --git a/org.eclipse.epf.web.search/lib/lucene-core-2.9.1.jar b/org.eclipse.epf.web.search/lib/lucene-core-2.9.1.jar new file mode 100644 index 0000000..a5cfb35 --- /dev/null +++ b/org.eclipse.epf.web.search/lib/lucene-core-2.9.1.jar Binary files differ
diff --git a/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/IndexSearch.java b/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/IndexSearch.java index d25f5c9..9182f81 100644 --- a/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/IndexSearch.java +++ b/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/IndexSearch.java
@@ -1,16 +1,22 @@ package org.eclipse.epf.web.search; +import java.io.File; +import java.util.Arrays; +import java.util.List; import java.util.StringTokenizer; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.Hits; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Searcher; import org.apache.lucene.search.Sort; -import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Version; import org.eclipse.epf.web.search.analysis.CJKAnalyzer; import org.eclipse.epf.web.search.analysis.ChineseAnalyzer; import org.eclipse.epf.web.search.analysis.TextAnalyzer; @@ -21,24 +27,25 @@ public class IndexSearch { private static Searcher searcher; - + /** * @param indexLocation * @param inputQuery * @param searchField - * @param analyzerName: name of analyzer class + * @param analyzerName + * : name of analyzer class * @return */ - public static Hits search(String indexLocation, String inputQuery, - String searchField, String analyzerName) { + public static TopDocs search(String indexLocation, String inputQuery, + String searchField, String analyzerName) { - Hits hits = null; + TopDocs hits = null; Query q = null; String queryString = null; DecimalFormat scoreFormatter = new DecimalFormat("0.######"); if (indexLocation == null || inputQuery == null || searchField == null - || searchField.length() == 0 || inputQuery.length() == 0) + || searchField.length() == 0 || inputQuery.length() == 0) return null; try { @@ -48,58 +55,60 @@ return null; if (searcher == null) { - searcher = new IndexSearcher(new RAMDirectory(indexLocation)); -// searcher = new IndexSearcher(FSDirectory.getDirectory(indexLocation, -// false)); - } - - if ( searcher == null ) { + //searcher = new IndexSearcher(new RAMDirectory(indexLocation)); + // searcher = new + //searcher= new IndexSearcher(FSDirectory.getDirectory(indexLocation, false)); + Directory dir = FSDirectory.open(new File(indexLocation)); + searcher = new IndexSearcher(IndexReader.open(dir)); + } + + if (searcher == null) { return hits; } - -// Sort sort = new Sort(new SortField[]{ -// new SortField("title"), -// new SortField("summary") -// }); - - // per user's request, the sorting should be based on the hit rate, + + // Sort sort = new Sort(new SortField[]{ + // new SortField("title"), + // new SortField("summary") + // }); + + // per user's request, the sorting should be based on the hit rate, // so don't pass in any sorting criteria Sort sort = null; - - try { + try { Analyzer analyzer = null; if (analyzerName != null) { if (analyzerName.equals("CJKAnalyzer")) {//$NON-NLS-1$) analyzer = new CJKAnalyzer(); } } - if (analyzer == null) { - analyzer = new TextAnalyzer(); - } + if (analyzer == null) { + analyzer = new TextAnalyzer(); + } if (DebugUtil.DEBUG) { System.out.println("LD> analyzer: " + analyzer); } - q = QueryParser.parse(queryString, searchField, - analyzer); - if ( q != null ) { - hits = searcher.search(q, (Sort)sort); + //q = QueryParser.parse(queryString, searchField, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_29, searchField, analyzer); + q = parser.parse(queryString); + if (q != null) { + hits = searcher.search(q, 1000); } - } catch (Throwable th ) { + } catch (Throwable th) { hits = null; } - - if ( hits == null ) { + + if (hits == null) { // note: icu4j failed with jre 1.5 in server mode. // in such a case, try the old one - q = QueryParser.parse(queryString, searchField, - new ChineseAnalyzer()); - if ( q != null ) { - hits = searcher.search(q, (Sort)sort); + //q = QueryParser.parse(queryString, searchField, new ChineseAnalyzer()); + QueryParser parser = new QueryParser(Version.LUCENE_29, searchField, new ChineseAnalyzer()); + q = parser.parse(queryString); + if (q != null) { + hits = searcher.search(q, 1000); } } - + return hits; - } catch (ParseException pe) { pe.printStackTrace(); @@ -112,55 +121,244 @@ return hits; } - /* - * detect hyphenated words and if hyphenate, put quotes around so that they are - * considered one word for example,"use-case" as opposed to use-case, which Lucene - * interprets as "use" NOT "case" which is not what we want. algorithm: first tokenize - * by space to isolate words next, if words have hyphens, put quotes at the beginning - * and end append to buffer + /** + * This method enhances search functionality so that search now works with + * type, title and tag. + * + * @param indexLocation + * @param inputQuery + * @param searchField + * @param searchType + * @param cbValue + * : boolean value of searchTitleOnly checkbox + * @param txtTags + * @param analyzerName + * : name of analyzer class + * @return */ - public static String detectHyphenated(String queryString) { + public static TopDocs searchNew(String indexLocation, String inputQuery, + String searchField, String searchType, boolean cbValue, + String txtTags, String analyzerName) { + + TopDocs hits = null; + Query q = null; + String queryString = null; + DecimalFormat scoreFormatter = new DecimalFormat("0.######"); + + if (indexLocation == null || inputQuery == null || searchField == null + || searchField.length() == 0 || searchType == null + || searchType.length() == 0 || inputQuery.length() == 0) + return null; + + try { + queryString = detectHyphenated(inputQuery); + + if (queryString == null || queryString.length() == 0) + return null; + + if (searcher == null) { + Directory dir = FSDirectory.open(new File(indexLocation)); + searcher = new IndexSearcher(IndexReader.open(dir)); + } + + if (searcher == null) { + return hits; + } + + // Sort sort = new Sort(new SortField[]{ + // new SortField("title"), + // new SortField("summary") + // }); + + // per user's request, the sorting should be based on the hit rate, + // so don't pass in any sorting criteria + Sort sort = null; + + try { + Analyzer analyzer = null; + if (analyzerName != null) { + if (analyzerName.equals("CJKAnalyzer")) {//$NON-NLS-1$) + analyzer = new CJKAnalyzer(); + } + } + if (analyzer == null) { + analyzer = new TextAnalyzer(); + } + if (DebugUtil.DEBUG) { + System.out.println("LD> analyzer: " + analyzer); + } + String str = txtTags; + String queryForList = ""; + String temp = ""; + + List<String> process = Arrays.asList("Activity", + "CapabilityPattern", "DeliveryProcess", "Iteration", + "Milestone", "Phase", "RoleDescriptor", + "TaskDescriptor", "WorkProductDescriptor"); + List<String> guidance = Arrays.asList("Checklist", "Concept", + "Example", "Guideline", "EstimationConsideration", + "Practice", "Report", "ReusableAsset", "RoadMap", + "SupportingMaterial", "TermDefinition", "Template", + "ToolMentor", "Whitepaper"); + List<String> category = Arrays.asList("CustomCategory", + "Domain", "Discipline", "RoleSet", "RoleSetGrouping", + "Tool"); + List<String> workProductType = Arrays.asList("Artifact", + "Deliverable", "Outcome"); + + if ((!"<All>".equals(searchType))) { + // check for (lists->Process,guidance,category,work_product) + switch (searchType.replaceAll(" ", "").toLowerCase()) { + case "category": + queryForList = createQueryForList(category, inputQuery); + break; + case "guidance": + queryForList = createQueryForList(guidance, inputQuery); + break; + case "process": + queryForList = createQueryForList(process, inputQuery); + break; + case "workproducttype": + queryForList = createQueryForList(workProductType, + inputQuery); + break; + default: // applicable for all cases except list and <All> + searchType = searchType.replaceAll("[-]", ""); + inputQuery = "uma.type:" + searchType + " AND " + + "contents:" + inputQuery; + break; + } + if (!queryForList.isEmpty()) { + inputQuery = queryForList; + } + } else { // if <All> + inputQuery = "contents:" + inputQuery; + } + // check for title-only values + if (cbValue) { + inputQuery = inputQuery + " AND " + "title:" + queryString; + } + // check for tags + if (str != null && !str.isEmpty()) { + List<String> tagList = Arrays.asList(str.split(",[ ]*")); + for (String tag : tagList) { + tag = detectHyphenated(tag); + inputQuery = inputQuery + " AND " + "tags:" + tag; + } + } + /*QueryParser parser = new QueryParser(searchField, analyzer); + q = parser.parse(inputQuery);*/ + QueryParser parser = new QueryParser(Version.LUCENE_29, searchField, analyzer); + q = parser.parse(inputQuery); + if (q != null) { + hits = searcher.search(q, 1000); + } + + } catch (Throwable th) { + hits = null; + } + + if (hits == null) { + // note: icu4j failed with jre 1.5 in server mode. + // in such a case, try the old one + //q = QueryParser.parse(queryString, searchField, new ChineseAnalyzer()); + QueryParser parser = new QueryParser(Version.LUCENE_29, searchField, new ChineseAnalyzer()); + q = parser.parse(inputQuery); + if (q != null) { + hits = searcher.search(q, 1000); + } + } + return hits; + + } catch (ParseException pe) { + pe.printStackTrace(); + } catch (java.io.IOException io) { + io.printStackTrace(); + } catch (Throwable p1) { + p1.printStackTrace(); + } + + return hits; + } + + /** + * This method is used to iterate through and create query for the 4 lists - + * Process, Guidance, Category, Work Product Type. + * + * @param List + * list - based on user input passed as type field. String + * inputQuery - String text to be searched through the list. + * + * @return String This returns the query to be parsed through the + * queryparser. + */ + private static String createQueryForList(List<String> list, + String inputQuery) { + // TODO Auto-generated method stub + String queryForList = ""; + String temp = ""; + for (String values : list) { + temp = "(uma.type:" + values + " AND " + "contents:" + inputQuery + + ")"; + if (queryForList.isEmpty()) { + queryForList = temp; + } else { + queryForList = queryForList + " OR " + temp; + } + } + return queryForList; + } + + /* + * detect hyphenated words and if hyphenate, put quotes around so that they + * are considered one word for example,"use-case" as opposed to use-case, + * which Lucene interprets as "use" NOT "case" which is not what we want. + * algorithm: first tokenize by space to isolate words next, if words have + * hyphens, put quotes at the beginning and end append to buffer + */ + public static String detectHyphenated(String queryString) { StringTokenizer spaceTokenizer = new StringTokenizer(queryString); StringBuffer wordBuffer = new StringBuffer(); - + while (spaceTokenizer.hasMoreTokens()) { // isolate into words String word = spaceTokenizer.nextToken(); - if ((word.indexOf('-') != -1 ) && !word.startsWith("\"")) //$NON-NLS-1$ + if ((word.indexOf('-') != -1) && !word.startsWith("\"")) //$NON-NLS-1$ { // hyphen found, and word is not already an exact phrase, // so add quotes to beginning and end of word wordBuffer.append("\""); //$NON-NLS-1$ wordBuffer.append(word); wordBuffer.append("\""); //$NON-NLS-1$ - //System.out.println("if: " + wordBuffer.toString()); + // System.out.println("if: " + wordBuffer.toString()); } else { - //System.out.println("else: " + queryString); - return queryString; //RATLC00251031 - //wordBuffer.append(word); + // System.out.println("else: " + queryString); + return queryString; // RATLC00251031 + // wordBuffer.append(word); } if (spaceTokenizer.hasMoreTokens()) { wordBuffer.append(" "); //$NON-NLS-1$ } } - - + return wordBuffer.toString(); } /** * @param args */ -// public static void main(String[] args) { -// if (args.length > 0) { -// // String rupPath = System.getProperty( "user.home" ) + java.io.File.separator -// // + "rup"; -// String rupPath = args[0]; -// SortedHits sortedHits = IndexSearch.search(rupPath + "\\index", "Analyst", -// "contents"); -// System.out -// .println("There are " + sortedHits.length() + " hits for 'analyst'"); -// } -// -// } + // public static void main(String[] args) { + // if (args.length > 0) { + // // String rupPath = System.getProperty( "user.home" ) + + // java.io.File.separator + // // + "rup"; + // String rupPath = args[0]; + // SortedHits sortedHits = IndexSearch.search(rupPath + "\\index", + // "Analyst", + // "contents"); + // System.out + // .println("There are " + sortedHits.length() + " hits for 'analyst'"); + // } + // + // } } \ No newline at end of file
diff --git a/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/analysis/ChineseFilter.java b/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/analysis/ChineseFilter.java index 3cf281f..696d295 100644 --- a/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/analysis/ChineseFilter.java +++ b/org.eclipse.epf.web.search/src/org/eclipse/epf/web/search/analysis/ChineseFilter.java
@@ -5,6 +5,7 @@ import java.util.Hashtable; import java.util.ResourceBundle; +import java.util.Set; import java.util.StringTokenizer; import org.apache.lucene.analysis.StopAnalyzer; @@ -22,7 +23,7 @@ { super(in); - input = in; + // input = in; if (_stopTable == null) { loadStopWords(); @@ -74,9 +75,13 @@ String property = bundle.getString("Search.stopWords"); //$NON-NLS-1$ words = split(property, " ,", -1); //$NON-NLS-1$ } catch (Exception e) { - words = StopAnalyzer.ENGLISH_STOP_WORDS; + //words = StopAnalyzer.ENGLISH_STOP_WORDS; + Set<?> stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; + words = stopWords.toArray(new String[stopWords.size()]); } - _stopTable = StopFilter.makeStopTable(words); + + + _stopTable = (Hashtable) StopFilter.makeStopSet(words); } /**
diff --git a/org.eclipse.epf.web/.classpath b/org.eclipse.epf.web/.classpath index 3bde06c..f9ddb93 100644 --- a/org.eclipse.epf.web/.classpath +++ b/org.eclipse.epf.web/.classpath
@@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="UTF-8"?> -<classpath> - <classpathentry kind="src" path="src"/> - <classpathentry kind="lib" path="lib/servlet.jar"/> - <classpathentry kind="lib" path="/org.eclipse.epf.web.search/lib/icu4j_3_4.jar"/> - <classpathentry kind="lib" path="/org.eclipse.epf.web.search/lib/lucene-core-1.9.1.jar"/> - <classpathentry combineaccessrules="false" kind="src" path="/org.eclipse.epf.web.search"/> - <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> - <classpathentry kind="output" path="bin"/> -</classpath> +<?xml version="1.0" encoding="UTF-8"?> +<classpath> + <classpathentry kind="src" path="src"/> + <classpathentry kind="lib" path="lib/servlet.jar"/> + <classpathentry kind="lib" path="/org.eclipse.epf.web.search/lib/icu4j_3_4.jar"/> + <classpathentry combineaccessrules="false" kind="src" path="/org.eclipse.epf.web.search"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> + <classpathentry kind="lib" path="/org.eclipse.epf.web.search/lib/lucene-core-2.9.1.jar"/> + <classpathentry kind="output" path="bin"/> +</classpath>
diff --git a/org.eclipse.epf.web/src/org/eclipse/epf/web/servlet/SearchServlet.java b/org.eclipse.epf.web/src/org/eclipse/epf/web/servlet/SearchServlet.java index 1e27c74..fe6fab0 100644 --- a/org.eclipse.epf.web/src/org/eclipse/epf/web/servlet/SearchServlet.java +++ b/org.eclipse.epf.web/src/org/eclipse/epf/web/servlet/SearchServlet.java
@@ -3,6 +3,7 @@ //------------------------------------------------------------------------------ package org.eclipse.epf.web.servlet; +import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; @@ -19,7 +20,13 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.eclipse.epf.web.search.IndexLoader; import org.eclipse.epf.web.search.IndexSearch; import org.eclipse.epf.web.search.SearchResources; @@ -43,10 +50,11 @@ private String _currentSearchString; private String _queryString; private String _webAppName; - private Hits _hits; + private TopDocs _hits; private int _numHits; private int _currentPage; private String analyzerName; + private static Searcher searcher; private HttpServletResponse _response; @@ -63,6 +71,11 @@ config.getInitParameter("searchIndex")); _xslURL = config.getServletContext().getRealPath( config.getInitParameter("xslFile")); + if (searcher == null) { + Directory dir = FSDirectory.open(new File(_indexPath)); + searcher = new IndexSearcher(IndexReader.open(dir)); + + } } catch (Exception e) { e.printStackTrace(); } @@ -87,12 +100,13 @@ synchronized(session) { String results; + String newResults; String action = request.getParameter("action"); _webAppName = request.getContextPath(); if (action != null && action.length() > 0) { - _hits = (Hits) session.getAttribute("hits"); + _hits = (TopDocs) session.getAttribute("hits"); _queryString = (String) session.getAttribute("queryString"); _numHits = ((Integer) session.getAttribute("numHits")).intValue(); _currentPage = ((Integer) session.getAttribute("currentPage")).intValue(); @@ -111,6 +125,11 @@ } else { String searchString = request.getParameter("searchString"); + String searchType = request.getParameter("searchType"); + boolean cbValue = Boolean.parseBoolean(request.getParameter("cbValue")); + String txtTags = request.getParameter("txtTags"); + int hitsPerPage = Integer.parseInt(request.getParameter("hitsPerPage")); + if (DebugUtil.DEBUG) { System.out.println("LD> searchString: " + searchString); System.out.println("LD> documentBasePath: " + _pathDocumentBase); @@ -130,12 +149,14 @@ if (resultsPerPage != null && resultsPerPage.length() > 0) { _hitsPerPage = Integer.parseInt(resultsPerPage); } + results = runSearch(searchString); + newResults = runSearchNew(searchString, searchType, cbValue, txtTags, hitsPerPage); session.setAttribute("hits", _hits); session.setAttribute("queryString", _queryString); session.setAttribute("numHits", new Integer(_numHits)); session.setAttribute("currentPage", new Integer(_currentPage)); - out.println(results); + out.println(newResults); } } @@ -210,12 +231,12 @@ _hits = IndexSearch.search(_indexPath, _currentSearchString, searchField, analyzerName);// new if (DebugUtil.DEBUG) { System.out.println("LD> _hits: " + - (_hits == null ? "null" : Integer.toString(_hits.length())));//$NON-NLS-1$ //$NON-NLS-2$ + (_hits == null ? "null" : Integer.toString(_hits.totalHits)));//$NON-NLS-1$ //$NON-NLS-2$ } // SortedHits(hits); - _numHits = _hits.length(); + _numHits = _hits.totalHits; result = true; } catch (Exception e1) { @@ -229,6 +250,77 @@ } /** + * Performs the advanced search with new search dialog. + */ + private String runSearchNew(String searchString, String searchType, boolean cbValue, String txtTags, int _hitsPerPage2) + { + + try { + + _queryString = searchString;//.replace(':', ' '); // RATLC00251228 enable to use tag to filte search results + + _currentSearchString = IndexSearch.detectHyphenated(_queryString); // wordBuffer.toString(); + + if (_currentSearchString.length() == 0) { + return EMPTY_HTML; + } + + boolean safe = runQueryNew("contents", searchType, cbValue, txtTags); + + String result = updateOutput(safe); + + return (result == null) ? EMPTY_HTML : result; + + } catch (Exception e) { + e.printStackTrace(); + } + + return EMPTY_HTML; + } + + private boolean runQueryNew(String searchField, String searchType, boolean cbValue, String txtTags) + { + boolean result = false; + //_currentSearchString = "\u57FA\u672C\u539F\u5247"; + try { + if (DebugUtil.DEBUG) { + System.out.println("LD> _indexPath: " + _indexPath);//$NON-NLS-1$ + System.out.println("LD> _currentSearchString: " + _currentSearchString);//$NON-NLS-1$ + } + if (_currentSearchString != null) { + if (DebugUtil.DEBUG) { + System.out.println("LD> unicodes: " + + escapeUnicodeString(_currentSearchString, false));//$NON-NLS-1$ + } + } + + if (DebugUtil.DEBUG) { + System.out.println("LD> analyzerName: " + analyzerName);//$NON-NLS-1$ + } + + _hits = IndexSearch.searchNew(_indexPath, this._currentSearchString, + searchField, searchType, cbValue, txtTags, this.analyzerName);// new + if (DebugUtil.DEBUG) { + System.out.println("LD> _hits: " + + (_hits == null ? "null" : Integer.toString(_hits.totalHits)));//$NON-NLS-1$ //$NON-NLS-2$ + } + + // SortedHits(hits); + + _numHits = _hits.totalHits; + + result = true; + } catch (Exception e1) { + e1.printStackTrace(); + _numHits = 0; + _hits = null; + } + _currentPage = 0; + + return result; + } + + /** * Updates the results panel with the search output. */ private String updateOutput(boolean safe) { @@ -281,7 +373,9 @@ .append("\">"); //$NON-NLS-1$ for (int i = init; i < max; i++) { - Document doc = _hits.doc(i); + + //Document doc = _hits.doc(i); + Document doc = searcher.doc(_hits.scoreDocs[i].doc); Field url = doc.getField("url"); //$NON-NLS-1$ Field title = doc.getField("title"); //$NON-NLS-1$ Field summary = doc.getField("summary"); //$NON-NLS-1$