Update to elasticsearch 7
diff --git a/build.gradle b/build.gradle
index b89a343..927f374 100644
--- a/build.gradle
+++ b/build.gradle
@@ -94,15 +94,16 @@
// querying es
compile 'commons-httpclient:commons-httpclient:3.1'
- compile 'org.apache.commons:commons-lang3:3.8.1'
+ compile 'org.apache.commons:commons-text:1.6'
// testing
testCompile 'junit:junit:4.12'
testRuntime 'org.slf4j:slf4j-simple:1.7.25'
testCompile 'org.mockito:mockito-core:2.13.0'
testCompile 'org.assertj:assertj-core:3.6.2'
- testCompile(group: 'org.elasticsearch', name: 'elasticsearch', version: '2.3.4')
- testCompile(group: 'org.elasticsearch', name: 'elasticsearch', version: '2.3.4', classifier: 'tests')
+
+ testCompile "org.elasticsearch.client:elasticsearch-rest-client:6.4.1"
+ testCompile "org.testcontainers:elasticsearch:1.12.5"
}
test {
diff --git a/src/main/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearch.java b/src/main/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearch.java
index 77f7c11..212a4f9 100644
--- a/src/main/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearch.java
+++ b/src/main/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearch.java
@@ -17,6 +17,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -26,12 +27,14 @@
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.text.StringEscapeUtils;
import org.eclipse.mdm.api.base.model.Entity;
import org.eclipse.mdm.api.base.model.Measurement;
import org.eclipse.mdm.api.base.model.Test;
import org.eclipse.mdm.api.base.model.TestStep;
import org.eclipse.mdm.api.base.query.DataAccessException;
+import org.eclipse.mdm.api.dflt.model.Pool;
+import org.eclipse.mdm.api.dflt.model.Project;
import org.eclipse.mdm.api.odsadapter.lookup.EntityLoader;
import org.eclipse.mdm.api.odsadapter.lookup.config.EntityConfig.Key;
import org.slf4j.Logger;
@@ -53,7 +56,7 @@
/**
* This is the payload which needs to be added to the post to add a
*/
- private static final String ES_POSTDATA = "{\"query\":{\"simple_query_string\":{\"query\":\"%s\",\"default_operator\":\"or\",\"lenient\":\"true\",\"fields\":[\"name^2\",\"_all\"]}}}";
+ private static final String ES_POSTDATA = "{\"_source\": [\"source\", \"type\", \"id\"], \"query\":{\"simple_query_string\":{\"query\":\"%s\",\"default_operator\":\"or\",\"lenient\":\"true\"}}}";
/**
* mainly logs requests on INFO
@@ -68,7 +71,7 @@
/**
* The URL is hard coded
*/
- private String url;
+ private URI url;
/**
* The client is created once and reused
@@ -86,7 +89,7 @@
public ODSFreeTextSearch(EntityLoader entityLoader, String sourceName, String host) throws DataAccessException {
this.loader = entityLoader;
- url = host + "/" + sourceName.toLowerCase() + "/_search?fields=_type,_id,_index&size=50";
+ url = URI.create(host).resolve(sourceName.toLowerCase()).resolve("/_search");
client = new HttpClient();
}
@@ -151,9 +154,9 @@
* @param map the map of all ids for the class of the entity
*/
private void put(JsonElement hit, Map<Class<? extends Entity>, List<String>> map) {
- JsonObject object = hit.getAsJsonObject();
+ JsonObject object = hit.getAsJsonObject().get("_source").getAsJsonObject();
- String type = object.get("_type").getAsString();
+ String type = object.get("type").getAsString();
Class<? extends Entity> clazz = getClass4Type(type);
if (clazz != null) {
@@ -163,7 +166,7 @@
}
List<String> list = map.get(clazz);
- list.add((String) object.get("_id").getAsString());
+ list.add((String) object.get("id").getAsString());
}
}
@@ -175,15 +178,22 @@
private Class<? extends Entity> getClass4Type(String type) {
Class<? extends Entity> clazz;
switch (type) {
+ case "Project":
+ clazz = Project.class;
+ break;
+ case "Pool":
+ clazz = Pool.class;
+ break;
+ case "Test":
+ clazz = Test.class;
+ break;
case "TestStep":
clazz = TestStep.class;
break;
case "Measurement":
clazz = Measurement.class;
break;
- case "Test":
- clazz = Test.class;
- break;
+
default:
clazz = null;
}
@@ -197,7 +207,7 @@
* @return
*/
private JsonElement queryElasticSearch(String inputQuery) {
- PostMethod post = new PostMethod(url);
+ PostMethod post = new PostMethod(url.toString());
String requestJson = buildRequestJson(inputQuery);
diff --git a/src/test/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearchTest.java b/src/test/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearchTest.java
index 25b2654..a0180be 100644
--- a/src/test/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearchTest.java
+++ b/src/test/java/org/eclipse/mdm/api/odsadapter/search/ODSFreeTextSearchTest.java
@@ -14,71 +14,59 @@
package org.eclipse.mdm.api.odsadapter.search;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyCollection;
-import static org.mockito.ArgumentMatchers.anyCollectionOf;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.text.StringEscapeUtils;
+import org.apache.http.HttpHost;
import org.eclipse.mdm.api.base.model.Entity;
import org.eclipse.mdm.api.base.model.Measurement;
import org.eclipse.mdm.api.base.model.TestStep;
import org.eclipse.mdm.api.base.query.DataAccessException;
import org.eclipse.mdm.api.odsadapter.lookup.EntityLoader;
import org.eclipse.mdm.api.odsadapter.lookup.config.EntityConfig.Key;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
import org.junit.Before;
-import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+@Ignore
+//FIXME 05.02.2020: this test needs a docker to run elasticsearch and is not suitable for continous build in Jenkins.
+//Comment this in for local tests only, if docker is available
public class ODSFreeTextSearchTest {
- private static final String HOST = "http://localhost:9301";
+
private EntityLoader loader;
private ODSFreeTextSearch fts;
- private Entity ts;
+ private TestStep ts;
- private static Node elasticSearchNode;
+ private RestClient client;
- private static Client client;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- File tempDir = File.createTempFile("elasticsearch-temp", Long.toString(System.nanoTime()));
- tempDir.delete();
- tempDir.mkdir();
-
- String clusterName = UUID.randomUUID().toString();
- elasticSearchNode = NodeBuilder.nodeBuilder().local(true).clusterName(clusterName)
- .settings(Settings.settingsBuilder()
- .put("path.home", File.createTempFile("elasticsearch", "").getParent())
- .put("index.number_of_shards", 1).put("index.number_of_replicas", 0).put("http.port", 9301))
- .node();
- elasticSearchNode.start();
- client = elasticSearchNode.client();
- }
+ @ClassRule
+ public static ElasticsearchContainer elasticSearch = new ElasticsearchContainer(
+ "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0");
@SuppressWarnings("unchecked")
@Before
public void init() throws DataAccessException {
ts = mock(TestStep.class);
loader = mock(EntityLoader.class);
- List<Entity> tsList = new ArrayList<>();
- tsList.add(ts);
+
when(loader.loadAll(any(Key.class), anyCollection())).thenAnswer(new Answer<List<Entity>>() {
@Override
@@ -94,70 +82,111 @@
}
});
- fts = new ODSFreeTextSearch(loader, "mdm", HOST);
+ client = RestClient.builder(HttpHost.create(elasticSearch.getHttpHostAddress())).build();
+ fts = new ODSFreeTextSearch(loader, "mdm", "http://" + elasticSearch.getHttpHostAddress());
}
@Test
public void noIndex_emptyResult() throws DataAccessException {
- ODSFreeTextSearch ftsOtherIndex = new ODSFreeTextSearch(loader, "UNKNOWN_INDEX", HOST);
+ ODSFreeTextSearch ftsOtherIndex = new ODSFreeTextSearch(loader, "UNKNOWN_INDEX",
+ "http://" + elasticSearch.getHttpHostAddress());
Map<Class<? extends Entity>, List<Entity>> map = ftsOtherIndex.search("VAG_002");
- assertTrue(map.isEmpty());
+ assertThat(map).isEmpty();
}
@Test
public void exampleIndex_querySuccessfull() throws Exception {
- createExampleIndex("TestStep", "mdm", "asdf");
+ try {
+ createExampleIndex("mdm", "TestStep", "asdf");
- Map<Class<? extends Entity>, List<Entity>> search = fts.search("asdf");
- assertEquals(ts, search.get(TestStep.class).get(0));
+ assertThat(fts.search("asdf").get(TestStep.class).get(0)).isEqualTo(ts);
+ } finally {
+ deleteIndex("mdm");
+ }
}
@Test
public void specialCharacters_correctlyEscaped() throws InterruptedException {
- createExampleIndex("Measurement", "mdm", "hallo\"!§");
+ try {
+ createExampleIndex("mdm", "Measurement", "hallo\"!§");
- Map<Class<? extends Entity>, List<Entity>> search = fts.search("hallo\"!§");
- assertEquals(ts, search.get(Measurement.class).get(0));
+ Map<Class<? extends Entity>, List<Entity>> search = fts.search("hallo\"!§");
+ assertThat(search.get(Measurement.class).get(0)).isEqualTo(ts);
+ } finally {
+ deleteIndex("mdm");
+ }
}
@Test
public void nonMdmResults_ignored() throws InterruptedException {
- createExampleIndex("NONMDMStuff", "mdm", "test");
+ try {
+ createExampleIndex("mdm", "NONMDMStuff", "test");
- Map<Class<? extends Entity>, List<Entity>> search = fts.search("test");
- assertTrue(search.isEmpty());
+ Map<Class<? extends Entity>, List<Entity>> search = fts.search("test");
+ assertThat(search).isEmpty();
+ } finally {
+ deleteIndex("mdm");
+ }
}
@Test
public void twoResults_twoResultsRetuned() throws InterruptedException {
- createExampleIndex("Test", "mdm", "unicorn ASDF", "0");
- createExampleIndex("Test", "mdm", "unicorn XYZ", "1");
+ try {
+ createExampleIndex("mdm", "Test", "0", "unicorn ASDF");
+ createExampleIndex("mdm", "Test", "1", "unicorn XYZ");
- Map<Class<? extends Entity>, List<Entity>> search = fts.search("unicorn");
- assertEquals(2, search.get(org.eclipse.mdm.api.base.model.Test.class).size());
+ Map<Class<? extends Entity>, List<Entity>> search = fts.search("unicorn");
+ assertThat(search.get(org.eclipse.mdm.api.base.model.Test.class)).hasSize(2);
+ } finally {
+ deleteIndex("mdm");
+ }
}
@Test(expected = IllegalStateException.class)
public void illegalLoadRequest_niceExceptionIsThrown() throws DataAccessException, InterruptedException {
- loader = mock(EntityLoader.class);
- when(loader.loadAll(any(), anyCollectionOf(String.class))).thenThrow(new DataAccessException(""));
- createExampleIndex("TestStep", "mdm2", "asdf");
- ODSFreeTextSearch fts2 = new ODSFreeTextSearch(loader, "mdm2", HOST);
+ try {
+ loader = mock(EntityLoader.class);
+ when(loader.loadAll(any(), anyCollection())).thenThrow(new DataAccessException(""));
+ createExampleIndex("mdm2", "TestStep", "asdf");
+ ODSFreeTextSearch fts2 = new ODSFreeTextSearch(loader, "mdm2",
+ "http://" + elasticSearch.getHttpHostAddress());
- fts2.search("asdf");
+ fts2.search("asdf");
+ } finally {
+ deleteIndex("mdm2");
+ }
}
- private void createExampleIndex(String type, String name, String value) throws InterruptedException {
- createExampleIndex(type, name, value, "0");
+ private void createExampleIndex(String indexName, String type, String value) throws InterruptedException {
+ createExampleIndex(indexName, type, "0", value);
}
- private void createExampleIndex(String type, String name, String value, String id) throws InterruptedException {
- String json = "{\"attr\":\"" + StringEscapeUtils.escapeJson(value) + "\"}";
+ private void createExampleIndex(String indexName, String type, String id, String value)
+ throws InterruptedException {
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String, String> json = new HashMap<>();
+ json.put("id", StringEscapeUtils.escapeJson(id));
+ json.put("type", StringEscapeUtils.escapeJson(type));
+ json.put("attr", StringEscapeUtils.escapeJson(value));
- client.prepareIndex(name, type, id).setSource(json).get();
+ try {
+ Request a = new Request("PUT", indexName + "/_doc/" + type + "-" + id);
+ a.setJsonEntity(mapper.writeValueAsString(json));
+ client.performRequest(a);
+ } catch (IOException e) {
+ throw new RuntimeException("Could not index!");
+ }
Thread.sleep(1000); // let the index some time to populate
}
+ private void deleteIndex(String indexName) {
+ try {
+ client.performRequest(new Request("DELETE", indexName));
+ } catch (IOException e) {
+ throw new RuntimeException("Could not delete index!");
+ }
+ }
+
}
