Use ID hashes for Elastic, improved test coverage
diff --git a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/EObjectToJsonTest.java b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/EObjectToJsonTest.java
index 81771a9..5c6408a 100644
--- a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/EObjectToJsonTest.java
+++ b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/EObjectToJsonTest.java
@@ -12,6 +12,8 @@
******************************************************************************/
package org.eclipse.opencert.elastic;
+import java.util.Iterator;
+
import org.eclipse.emf.ecore.EObject;
import org.junit.Assert;
import org.junit.Test;
@@ -36,7 +38,8 @@
@Test
public void testUML() {
- EObject elem = TestData.uml().next();
+ Iterator<EObject> it = TestData.uml();
+ EObject elem = it.next();
JsonObject json = EObjectToJson.INSTANCE.convert(elem);
Assert.assertNotNull(json);
System.out.println(json.toString());
diff --git a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticClientTest.java b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticClientTest.java
index 4c77f1b..6da9035 100644
--- a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticClientTest.java
+++ b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticClientTest.java
@@ -36,8 +36,13 @@
}
@Test
+ public void testDelete() throws Exception {
+ ElasticClientImpl.on("localhost", 9200, "http").delete("amass-test").close();
+ }
+
+ @Test
public void testSendMany() throws Exception {
- List<ElasticDocument> documents = EObjectToDocument.INSTANCE.convert(TestData.uml(), "amass-test");
+ List<ElasticDocument> documents = EObjectToDocument.INSTANCE.convert(TestData.uml(50), "amass-test");
ElasticClientImpl.on("localhost", 9200, "http").storeAll(documents.iterator()).close();
}
@@ -45,7 +50,7 @@
public void testSendAndSearch() throws Exception {
Component object = TestData.createComponent();
ElasticDocument document = EObjectToDocument.INSTANCE.convert(object, "amass-test");
- ElasticClient client = ElasticClientImpl.on("localhost", 9200, "http")/*.delete("amass-test")*/.store(document);
+ ElasticClient client = ElasticClientImpl.on("localhost", 9200, "http").delete("amass-test").store(document);
Set<ElasticDocument> hits = client.search("amass-test", null, null);
Assert.assertEquals(1, hits.size());
//
diff --git a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticFinderTest.java b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticFinderTest.java
index 6a29336..ce3291f 100644
--- a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticFinderTest.java
+++ b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/ElasticFinderTest.java
@@ -13,16 +13,26 @@
package org.eclipse.opencert.elastic;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.eclipse.opencert.elastic.search.ElasticFinder;
import org.eclipse.opencert.elastic.search.ElasticFinderImpl;
import org.eclipse.opencert.elastic.search.Hit;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
public class ElasticFinderTest {
+ @BeforeClass
+ public static void setup() throws Exception {
+ System.out.println("Indexing UML classes...");
+ List<ElasticDocument> documents = EObjectToDocument.INSTANCE.convert(TestData.umlClasses(), "amass-test");
+ ElasticClientImpl.on("localhost", 9200, "http").storeAll(documents.iterator()).close();
+ }
+
@Test
public void testSearch() throws Exception {
ElasticClient client = ElasticClientImpl.on("localhost", 9200, "http");
@@ -40,53 +50,77 @@
@Test
public void testNoHit() throws Exception {
+ testNoHit(ElasticFinderImpl.onDummy(TestData.uml()));
+ // we use indexed UML meta model for testing
+ testNoHit(ElasticFinderImpl.onDefaultClient());
+ }
+
+ private void testNoHit(ElasticFinder finder) throws Exception {
// there is no "Requirement" in UML
- Set<Hit> hits = ElasticFinderImpl.onDummy(TestData.uml()).search("Requirement", null);
+ Set<Hit> hits = finder.search("Requirement", null);
Assert.assertNotNull("set expected", hits);
Assert.assertTrue("empty set expected", hits.isEmpty());
}
@Test
public void testHit() throws Exception {
+ // we use UML dummy meta model for testing
+ testHit(ElasticFinderImpl.onDummy(TestData.uml()));
+ // we use indexed UML meta model for testing
+ testHit(ElasticFinderImpl.onDefaultClient());
+ }
+
+ private void testHit(ElasticFinder finder) throws Exception {
// we should find the "Component" class
- Set<Hit> hits = ElasticFinderImpl.onDummy(TestData.uml()).search("Component", null);
+ Set<Hit> hits = finder.search("Component", null);
Assert.assertNotNull("set expected", hits);
Assert.assertFalse("filled set expected", hits.isEmpty());
Hit firstHit = hits.iterator().next();
Assert.assertTrue("score must be > 0", firstHit.score > 0);
}
-
+
@Test
public void testHitWithExpression() throws Exception {
- Set<Hit> hits = ElasticFinderImpl.onDummy(TestData.uml()).search(".*ompon.*", null);
+ testHitWithExpression(ElasticFinderImpl.onDummy(TestData.uml()));
+ testHitWithExpression(ElasticFinderImpl.onDefaultClient());
+ }
+
+ private void testHitWithExpression(ElasticFinder finder) throws Exception {
+ Set<Hit> hits = finder.search(".*ompon.*", null);
Assert.assertNotNull("set expected", hits);
Assert.assertFalse("filled set expected", hits.isEmpty());
}
@Test
public void testHitWithNegativeFilter() throws Exception {
- // we use UML meta model for testing
+ testHitWithNegativeFilter(ElasticFinderImpl.onDummy(TestData.uml()));
+ testHitWithNegativeFilter(ElasticFinderImpl.onDefaultClient());
+ }
+ private void testHitWithNegativeFilter(ElasticFinder finder) throws Exception {
Map<String, Object> filters = new HashMap<>();
filters.put("abstract", Boolean.TRUE);
- Set<Hit> hits = ElasticFinderImpl.onDummy(TestData.uml()).search("Component", filters);
+ Set<Hit> hits = finder.search("Component", filters);
Assert.assertNotNull("set expected", hits);
Assert.assertTrue("empty set expected - Component is not abstract", hits.isEmpty());
}
@Test
public void testHitWithPositiveFilter() throws Exception {
+ testHitWithPositiveFilter(ElasticFinderImpl.onDummy(TestData.uml()));
+ testHitWithPositiveFilter(ElasticFinderImpl.onDefaultClient());
+ }
+
+ private void testHitWithPositiveFilter(ElasticFinder finder) throws Exception {
Map<String, Object> filters = new HashMap<>();
filters.put("abstract", Boolean.FALSE);
filters.put("interface", Boolean.FALSE);
- Set<Hit> hits = ElasticFinderImpl.onDummy(TestData.uml()).search("Component", filters);
+ Set<Hit> hits = finder.search("Component", filters);
Assert.assertNotNull("set expected", hits);
Assert.assertFalse("filled set expected - Component is not abstract", hits.isEmpty());
Hit firstHit = hits.iterator().next();
Assert.assertTrue("score must be > 1 (due to filters matched)", firstHit.score > 1);
-
- System.out.println(hits);
}
}
diff --git a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/TestData.java b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/TestData.java
index a132260..d6619fb 100644
--- a/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/TestData.java
+++ b/org.eclipse.opencert.elastic.test/src/org/eclipse/opencert/elastic/TestData.java
@@ -12,9 +12,12 @@
******************************************************************************/
package org.eclipse.opencert.elastic;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import org.eclipse.emf.common.util.URI;
+import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.emf.ecore.resource.impl.ResourceImpl;
import org.eclipse.uml2.uml.Component;
@@ -32,21 +35,53 @@
static Component createComponent() {
return createComponent(false);
}
-
+
static Component createComponent(boolean inResource) {
Component comp = UMLFactory.eINSTANCE.createComponent();
comp.setName("Component A");
comp.setVisibility(VisibilityKind.PUBLIC_LITERAL);
-
+
if (inResource) {
ResourceImpl res = new ResourceImpl(URI.createURI("custom:somewhere"));
res.getContents().add(comp);
}
-
+
return comp;
}
-
+
static Iterator<EObject> uml() {
return UMLPackage.eINSTANCE.eAllContents();
}
+
+ static Iterator<EObject> umlClasses() {
+ List<EObject> classes = new ArrayList<>();
+ Iterator<EObject> it = uml();
+ while (it.hasNext()) {
+ EObject object = (EObject) it.next();
+ if (object instanceof EClass) {
+ classes.add(object);
+ }
+ }
+
+ System.out.println("Classes: " + classes.size());
+ return classes.iterator();
+ }
+
+ static Iterator<EObject> uml(final int max) {
+ return new Iterator<EObject>() {
+ private Iterator<EObject> delegate = UMLPackage.eINSTANCE.eAllContents();
+ private long i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < max && delegate.hasNext();
+ }
+
+ @Override
+ public EObject next() {
+ i++;
+ return delegate.next();
+ }
+ };
+ }
}
diff --git a/org.eclipse.opencert.elastic/build.properties b/org.eclipse.opencert.elastic/build.properties
index 6165ef1..ef66ef1 100644
--- a/org.eclipse.opencert.elastic/build.properties
+++ b/org.eclipse.opencert.elastic/build.properties
@@ -2,15 +2,4 @@
output.. = bin/
bin.includes = META-INF/,\
.,\
- lib/,\
- lib/elastic/elasticsearch-rest-client-5.6.1.jar,\
- lib/elastic/elasticsearch-rest-high-level-client-5.6.1.jar,\
- lib/elastic/elasticsearch-5.6.1.jar,\
- lib/elastic/lucene-core-6.6.1.jar,\
- lib/elastic/log4j-api-2.11.1.jar,\
- lib/elastic/jackson-core-2.8.6.jar,\
- lib/elastic/joda-time-2.9.5.jar,\
- lib/elastic/hppc-0.7.1.jar,\
- lib/elastic/log4j-core-2.11.1.jar,\
- lib/elastic/lucene-queryparser-6.6.1.jar,\
- plugin.xml
+ lib/
diff --git a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToDocument.java b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToDocument.java
index 7949f56..aee7a62 100644
--- a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToDocument.java
+++ b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToDocument.java
@@ -16,8 +16,11 @@
import java.util.Iterator;
import java.util.List;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.eclipse.emf.ecore.EAnnotation;
import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EObject;
+import org.eclipse.emf.ecore.impl.EStringToStringMapEntryImpl;
import org.eclipse.emf.ecore.resource.Resource;
import com.google.gson.JsonObject;
@@ -50,7 +53,7 @@
JsonObject source = EObjectToJson.INSTANCE.convert(object);
ElasticDocument doc = new ElasticDocument(source, index, type != null ? type.getName() : "EObject",
- resource != null ? resource.getURIFragment(object) : "null");
+ resource != null ? DigestUtils.md5Hex(resource.getURIFragment(object)) : "null");
return doc;
}
diff --git a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToJson.java b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToJson.java
index 631e6e5..24775ad 100644
--- a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToJson.java
+++ b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/EObjectToJson.java
@@ -14,9 +14,11 @@
import org.eclipse.emf.common.util.EList;
import org.eclipse.emf.common.util.URI;
+import org.eclipse.emf.ecore.EAnnotation;
import org.eclipse.emf.ecore.EAttribute;
import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EObject;
+import org.eclipse.emf.ecore.impl.EStringToStringMapEntryImpl;
import org.eclipse.emf.ecore.resource.Resource;
import com.google.gson.JsonObject;
@@ -84,6 +86,15 @@
Resource resource = object.eResource();
if (resource != null) {
String fragment = resource.getURIFragment(object);
+ if (object instanceof EAnnotation) {
+ // TODO We have problems with annotations
+ fragment = "/#";
+ }
+ else if (object instanceof EStringToStringMapEntryImpl) {
+ // TODO We have problems with these maps
+ fragment = "/#";
+ }
+
URI uri = resource.getURI().appendFragment(fragment);
String uriString = uri.toString();
jsonObject.addProperty("uri", uriString);
diff --git a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/ElasticClientImpl.java b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/ElasticClientImpl.java
index f743efe..b363579 100644
--- a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/ElasticClientImpl.java
+++ b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/ElasticClientImpl.java
@@ -20,6 +20,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.http.HttpEntity;
@@ -34,6 +35,7 @@
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -104,7 +106,10 @@
// find the best query builder depending on the arguments
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
if (queryString != null) {
- queryBuilder = QueryBuilders.queryStringQuery(queryString);
+ queryString = queryString.replaceAll(".*", "*");
+ QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery(queryString);
+ stringQuery.allowLeadingWildcard(true);
+ queryBuilder = stringQuery;
}
sourceBuilder.query(queryBuilder);
@@ -114,8 +119,8 @@
Set<ElasticDocument> result = new HashSet<>();
SearchHits hits = response.getHits();
- for (int i=0; i< hits.totalHits; i++) {
- SearchHit hit = hits.getAt(i);
+ // Note: total hits might be much bigger
+ for (SearchHit hit : hits.getHits()) {
// convert Hit to Json
JsonObject json = new JsonObject();
diff --git a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/search/ElasticFinderImpl.java b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/search/ElasticFinderImpl.java
index fe40dff..111ff6d 100644
--- a/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/search/ElasticFinderImpl.java
+++ b/org.eclipse.opencert.elastic/src/org/eclipse/opencert/elastic/search/ElasticFinderImpl.java
@@ -20,6 +20,7 @@
import org.eclipse.opencert.elastic.ElasticClient;
import org.eclipse.opencert.elastic.ElasticClientImpl;
+import org.eclipse.opencert.elastic.ElasticDocument;
/**
* Factory for the {@link ElasticFinder}.
@@ -71,13 +72,19 @@
}
try {
- this.client.search("_all", query, filters);
+ Set<Hit> hits = new HashSet<>();
+ Set<ElasticDocument> documents = this.client.search("_all", query, filters);
+ for (ElasticDocument document : documents) {
+ Hit hit = new Hit();
+ hit.score = 1;
+ hit.objectId = document.id;
+ hits.add(hit);
+ }
+ return hits;
} catch (Exception e) {
- // TODO Auto-generated catch block
e.printStackTrace();
+ return Collections.emptySet();
}
-
- return Collections.emptySet();
}
/*
@@ -88,7 +95,6 @@
for (Object data : dummyData) {
String document = DummyData.INSTANCE.asDocument(data);
if ((document.contains(query) || document.matches(query)) && matchFilters(document, filters)) {
- matchFilters(document, filters);
Hit hit = new Hit();
// we count 1 for string match + number of filters
hit.score = 1 + (filters != null ? filters.size() : 0);
