Added support for multiple results and service configuration file
Change-Id: I60dc72fa671f38dd22bd5b5341362af535b5cfb3
Signed-off-by: Fauth Dirk <Dirk.Fauth@de.bosch.com>
diff --git a/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowController.java b/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowController.java
index 201c8ac..21c7d6e 100644
--- a/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowController.java
+++ b/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowController.java
@@ -15,6 +15,8 @@
import java.io.File;
import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -99,6 +101,17 @@
return "workflow";
}
+ @PostMapping("/remove/{selected}")
+ public String removeService(
+ @PathVariable(name = "selected") String selected,
+ @ModelAttribute WorkflowStatus ws) {
+
+ ws.removeSelectedService(selected);
+
+ // render the form view
+ return "workflow";
+ }
+
@GetMapping("/selectedServices")
public String getSelectedServices() {
// render the servicesList fragment contained in selectedServices.html
@@ -208,7 +221,7 @@
if (StringUtils.isEmpty(statusUrl)) {
// fallback check for Link header if Location header is not set
statusUrl = getUrlFromLink(uploadResponse.getHeaders().get("Link"), "status", baseUrl);
- } else if (!statusUrl.startsWith(baseUrl)) {
+ } else if (!isValid(statusUrl, baseUrl)) {
throw new ProcessingFailedException(
"The status URL from the Location header does not match with the service base URL '" + baseUrl + "': " + statusUrl);
}
@@ -246,58 +259,78 @@
end = System.currentTimeMillis();
- // don't request more than 30 seconds
+ // don't request for more than 30 seconds
if (end - start > 30_000) {
break;
}
}
// first check if there is a result link
- String downloadUrl = getUrlFromLink(linkHeaders, "result", baseUrl);
+ List<String> resultUrls = getUrlsFromLink(linkHeaders, "result", baseUrl);
+
String deleteUrl = null;
- if (downloadUrl != null) {
+ if (!resultUrls.isEmpty()) {
workflowStatus.addMessage(serviceName + " processing finished");
- // download file
- Path migrationSubDir = storageService.load(workflowStatus.getUuid(), "_" + serviceName.toLowerCase());
- Files.createDirectories(migrationSubDir);
- HttpResponse<File> downloadResponse = Unirest.get(downloadUrl)
- .asFile(migrationSubDir.resolve("result").toString());
+ boolean error = false;
+ for (String resultUrl : resultUrls) {
+
+ // TODO check if link or download
+
+ // download file
+ Path migrationSubDir = storageService.load(workflowStatus.getUuid(), "_" + serviceName.toLowerCase());
+ Files.createDirectories(migrationSubDir);
-
- Path migrationResult = downloadResponse.getBody().toPath();
- String filename = getFilenameFromHeader(downloadResponse.getHeaders().getFirst("Content-Disposition"));
- if (filename != null) {
- migrationResult = Files.move(migrationResult, migrationSubDir.resolve(filename));
+ HttpResponse<File> resultResponse = Unirest.get(resultUrl)
+ .asFile(migrationSubDir.resolve("result").toString());
+
+ // TODO check http status 302
+ // check the response result code
+ if (resultResponse.getStatus() == HttpStatus.SC_OK) {
+
+ Path migrationResult = resultResponse.getBody().toPath();
+ String filename = getFilenameFromHeader(resultResponse.getHeaders().getFirst("Content-Disposition"));
+ if (filename != null) {
+ migrationResult = Files.move(migrationResult, migrationSubDir.resolve(filename));
+ }
+
+ String resultName = resultResponse.getHeaders().getFirst("x-app4mc-result-name");
+ if (StringUtils.isEmpty(resultName)) {
+ resultName = serviceName;
+ }
+
+ workflowStatus.addMessage(resultName + " result downloaded");
+
+ workflowStatus.addResult(
+ resultName + " Result",
+ MvcUriComponentsBuilder.fromMethodName(
+ WorkflowController.class,
+ "serveFile",
+ workflowStatus.getUuid(),
+ "_" + serviceName.toLowerCase(),
+ migrationResult.getFileName().toString()).build().toUri().toString());
+
+ // extract header information if result should be used in workflow
+ String useResult = resultResponse.getHeaders().getFirst("x-app4mc-use-result");
+ if (useResult == null || !useResult.toLowerCase().equals("false")) {
+ // the result should be used in the workflow
+ result = migrationResult;
+ }
+ } else {
+ error = true;
+ }
+
+ if (deleteUrl == null) {
+ // extract delete
+ deleteUrl = getUrlFromLink(resultResponse.getHeaders().get("Link"), "delete", baseUrl);
+ }
}
- workflowStatus.addMessage(serviceName + " result downloaded");
-
- workflowStatus.addResult(
- serviceName + " Result",
- MvcUriComponentsBuilder.fromMethodName(
- WorkflowController.class,
- "serveFile",
- workflowStatus.getUuid(),
- "_" + serviceName.toLowerCase(),
- migrationResult.getFileName().toString()).build().toUri().toString());
-
- // extract header information if result should be used in workflow
- String useResult = downloadResponse.getHeaders().getFirst("x-app4mc-use-result");
- if (useResult == null || !useResult.toLowerCase().equals("false")) {
- // the result should not be used in the workflow
- result = migrationResult;
- }
-
- // check the response result code
- if (downloadResponse.getStatus() == HttpStatus.SC_OK) {
- workflowStatus.addMessage(serviceName + " successfull");
+ if (error) {
+ workflowStatus.addError(serviceName + " failed with errors");
} else {
- workflowStatus.addError(serviceName + " failed with errors");
+ workflowStatus.addMessage(serviceName + " successfull");
}
-
- // extract delete
- deleteUrl = getUrlFromLink(downloadResponse.getHeaders().get("Link"), "delete", baseUrl);
} else {
String errorUrl = getUrlFromLink(linkHeaders, "error", baseUrl);
if (errorUrl != null) {
@@ -329,6 +362,7 @@
}
}
+ // TODO add check for delete configuration parameter
if (deleteUrl != null) {
// delete upload again
Unirest.delete(deleteUrl).asEmpty();
@@ -344,7 +378,7 @@
return result;
} catch (StorageException | IOException e) {
- throw new ProcessingFailedException("Error on file operations in converter workflow", e);
+ throw new ProcessingFailedException("Error on file operations in " + serviceName + " workflow", e);
}
}
@@ -409,7 +443,7 @@
if (rel.equals(relValue)) {
// SECURITY: ensure that host in link matches host in configured service
- if (!url.startsWith(baseUrl)) {
+ if (!isValid(url, baseUrl)) {
throw new ProcessingFailedException(
"The link for rel '" + rel + "' does not match with the service base URL '" + baseUrl + "': " + url);
}
@@ -424,6 +458,69 @@
}
/**
+ * Extracts the url from the link header and returns the url that has the
+ * specified rel attribute set.
+ *
+ * @param linkHeaders The link headers to parse.
+ * @param rel The value for the rel param for which the url is
+ * requested.
+ * @param baseUrl The base URL of the service.
+ * @return The url for the specified rel param or <code>null</code> if there is
+ * no link for the given rel.
+ */
+ public static List<String> getUrlsFromLink(List<String> linkHeaders, String rel, String baseUrl) {
+
+ ArrayList<String> results = new ArrayList<>();
+ ArrayList<String> errors = new ArrayList<>();
+
+ for (String linkHeader : linkHeaders) {
+ String[] links = linkHeader.split(LINK_DELIMITER);
+
+ for (String link : links) {
+ String[] segments = link.split(LINK_PARAM_DELIMITER);
+
+ if (segments.length < 2) {
+ continue;
+ }
+
+ String url = segments[0].trim();
+ if (!url.startsWith("<") || !url.endsWith(">")) {
+ continue;
+ }
+ url = url.substring(1, url.length() - 1);
+
+ for (int i = 1; i < segments.length; i++) {
+ String[] param = segments[i].trim().split("=");
+ if (param.length < 2 || !"rel".equals(param[0])) {
+ continue;
+ }
+
+ String relValue = param[1];
+ if (relValue.startsWith("\"") && relValue.endsWith("\"")) {
+ relValue = relValue.substring(1, relValue.length() - 1);
+ }
+
+ if (rel.equals(relValue)) {
+ // SECURITY: ensure that host in link matches host in configured service
+ if (isValid(url, baseUrl)) {
+ results.add(url);
+ } else {
+ errors.add("The link for rel '" + rel + "' does not match with the service base URL '" + baseUrl + "': " + url);
+ }
+ }
+ }
+ }
+ }
+
+ if (results.isEmpty() && !errors.isEmpty()) {
+ String error = String.join("\n", errors);
+ throw new ProcessingFailedException(error);
+ }
+
+ return results;
+ }
+
+ /**
* Extracts the filename from the Content-Disposition header.
*
* @param header The Content-Disposition header to parse.
@@ -444,4 +541,41 @@
return null;
}
+ /**
+ * Verify if the URL to check has the same host as the given base URL. This
+ * method is used to avoid that a service result redirects to another server.
+ *
+ * @param toCheck The URL String that should be checked.
+ * @param base The base URL String to check against.
+ * @return <code>true</code> if the host parts of the URLs are equal,
+ * <code>false</code> in any other case.
+ */
+ protected static boolean isValid(String toCheck, String base) {
+ if (StringUtils.isEmpty(toCheck) || StringUtils.isEmpty(base)) {
+ throw new ProcessingFailedException("URLs to compare can not be empty");
+ }
+
+ URL toCheckUrl = null;
+ try {
+ toCheckUrl = new URL(toCheck);
+ } catch (MalformedURLException e) {
+ throw new ProcessingFailedException("URL to check is not in a valid format", e);
+ }
+
+ URL baseUrl = null;
+ try {
+ baseUrl = new URL(base);
+ } catch (MalformedURLException e) {
+ throw new ProcessingFailedException("URL to check against is not in a valid format", e);
+ }
+
+ if (baseUrl != null && toCheckUrl != null) {
+ // TODO clarify: check only host, or host:port, or host:port/path
+// return baseUrl.getAuthority().equals(toCheckUrl.getAuthority());
+// return baseUrl.getAuthority().equals(toCheckUrl.getAuthority());
+ return toCheckUrl.getAuthority().equals(baseUrl.getAuthority()) && toCheckUrl.getPath().startsWith(baseUrl.getPath());
+ }
+
+ return false;
+ }
}
\ No newline at end of file
diff --git a/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowStatus.java b/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowStatus.java
index 0fb9b85..1e7eae3 100644
--- a/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowStatus.java
+++ b/manager/src/main/java/org/eclipse/app4mc/cloud/manager/WorkflowStatus.java
@@ -41,6 +41,10 @@
this.selectedServices.add(service);
}
+ public void removeSelectedService(String service) {
+ this.selectedServices.remove(service);
+ }
+
public ArrayList<String> getMessages() {
return messages;
}
diff --git a/manager/src/main/java/org/eclipse/app4mc/cloud/manager/administration/ApplicationConfig.java b/manager/src/main/java/org/eclipse/app4mc/cloud/manager/administration/ApplicationConfig.java
index a6ae0b8..ef8f86b 100644
--- a/manager/src/main/java/org/eclipse/app4mc/cloud/manager/administration/ApplicationConfig.java
+++ b/manager/src/main/java/org/eclipse/app4mc/cloud/manager/administration/ApplicationConfig.java
@@ -13,11 +13,18 @@
*/
package org.eclipse.app4mc.cloud.manager.administration;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import org.springframework.util.ResourceUtils;
import org.springframework.web.context.annotation.ApplicationScope;
@Configuration
@@ -26,9 +33,30 @@
@Bean
@ApplicationScope
public List<CloudServiceDefinition> cloudServiceDefinitions() {
- ArrayList<CloudServiceDefinition> definitions = new ArrayList<>();
- definitions.add(new CloudServiceDefinition("Migration", "http://localhost:8080/app4mc/converter/", "Model Migration Service"));
- definitions.add(new CloudServiceDefinition("Validation", "http://localhost:8181/app4mc/validation/", "Model Validation Service"));
+ ArrayList<CloudServiceDefinition> definitions = new ArrayList<>();
+
+ String config = System.getProperty("service.configuration");
+ if (config != null) {
+ try {
+ File file = ResourceUtils.getFile(config);
+ Path path = Paths.get(file.toURI());
+ List<CloudServiceDefinition> services = Files.lines(path).map(line -> {
+ String[] split = line.split(";");
+ String name = split[0];
+ String url = split[1];
+ String desc = split.length >= 3 ? split[2] : "";
+ return new CloudServiceDefinition(name, url, desc);
+ }).collect(Collectors.toList());
+ definitions.addAll(services);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ if (definitions.isEmpty()) {
+ definitions.add(new CloudServiceDefinition("Migration", "http://localhost:8080/app4mc/converter/", "Model Migration Service"));
+ definitions.add(new CloudServiceDefinition("Validation", "http://localhost:8181/app4mc/validation/", "Model Validation Service"));
+ }
return definitions;
}
diff --git a/manager/src/main/resources/application.properties b/manager/src/main/resources/application.properties
index 51ad5eb..52ed629 100644
--- a/manager/src/main/resources/application.properties
+++ b/manager/src/main/resources/application.properties
@@ -1 +1,2 @@
-server.port=9090
+spring.servlet.multipart.max-file-size=20MB
+spring.servlet.multipart.max-request-size=20MB
diff --git a/manager/src/main/resources/services.txt b/manager/src/main/resources/services.txt
new file mode 100644
index 0000000..9789c18
--- /dev/null
+++ b/manager/src/main/resources/services.txt
@@ -0,0 +1,6 @@
+Migration;http://localhost:8080/app4mc/converter/;Model Migration Service
+Validation;http://localhost:8181/app4mc/validation/;Model Validation Service
+INCHRON;https://am2inc.dev1.inchron.de/projects;INCHRON Transformation Service
+RTC Analysis;http://localhost:8081/app4mc/analysis/;RTC Analysis Service
+Chart Interpreter;http://localhost:8082/rtc/nodes/;Chart Interpreter Service
+Chart Visualizer;http://localhost:8083/charts/barchart/;Chart Visualizer Service
\ No newline at end of file
diff --git a/manager/src/main/resources/templates/selectedServices.html b/manager/src/main/resources/templates/selectedServices.html
index 904f5f0..3709203 100644
--- a/manager/src/main/resources/templates/selectedServices.html
+++ b/manager/src/main/resources/templates/selectedServices.html
@@ -7,15 +7,9 @@
<body>
<div th:fragment="servicesList" id="selectedServices">
<span th:each="selected : ${workflowStatus.selectedServices}">
- <select name="services" id="services" onchange="updateSelectedServices(this.value)">
- <option></option>
- <option
- th:each="service : ${cloudServiceDefinitions}"
- th:text="${service.name}"
- th:value="${service.name}"
- th:selected="${service.name == selected}">
- </option>
- </select><br>
+ <span th:text="${selected}">Service</span>
+ <button type="button" class="btn btn-default" th:attr="onclick=|removeSelectedServices('${selected}')|" >remove</button>
+ <br>
</span>
<select name="services" id="services" onchange="updateSelectedServices(this.value)">
<option></option>
diff --git a/manager/src/main/resources/templates/workflow.html b/manager/src/main/resources/templates/workflow.html
index 48d1c34..8268cde 100644
--- a/manager/src/main/resources/templates/workflow.html
+++ b/manager/src/main/resources/templates/workflow.html
@@ -14,6 +14,16 @@
});
}
+function removeSelectedServices(service) {
+ $.ajax({
+ type: 'POST',
+ url: '/remove/' + service,
+ success: function(result) {
+ $('#selectedServicesBlock').load('/selectedServices');
+ }
+ });
+}
+
$(document).ready(function(){
$('#selectedServicesBlock').load('/selectedServices');
});
@@ -33,8 +43,8 @@
</div>
</td>
</tr>
- <tr>
- <td>Select validations to perform:</td>
+ <tr th:if="${allProfiles != null and not allProfiles.isEmpty()}">
+ <td valign="top">Select validations to perform:</td>
<td>
<ul>
<li th:each="profile : ${allProfiles}">
diff --git a/manager/src/test/java/org/eclipse/app4mc/cloud/manager/WorkflowControllerTest.java b/manager/src/test/java/org/eclipse/app4mc/cloud/manager/WorkflowControllerTest.java
new file mode 100644
index 0000000..7938d9b
--- /dev/null
+++ b/manager/src/test/java/org/eclipse/app4mc/cloud/manager/WorkflowControllerTest.java
@@ -0,0 +1,143 @@
+/*********************************************************************************
+ * Copyright (c) 2020 Robert Bosch GmbH and others.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Robert Bosch GmbH - initial API and implementation
+ ********************************************************************************
+ */
+package org.eclipse.app4mc.cloud.manager;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+class WorkflowControllerTest {
+
+ @Test
+ void shouldCheckValidUrls() {
+ String baseUrl = "http://localhost:8080/app4mc/converter";
+ String toCheckUrl = "http://localhost:8080/app4mc/converter/5240471251956758767";
+
+ assertTrue(WorkflowController.isValid(toCheckUrl, baseUrl));
+ }
+
+ @Test
+ void shouldCheckValidUrlsDifferentProtocols() {
+ String baseUrl = "http://localhost:8080/app4mc/converter";
+ String toCheckUrl = "https://localhost:8080/app4mc/converter/5240471251956758767";
+
+ assertTrue(WorkflowController.isValid(toCheckUrl, baseUrl));
+ }
+
+ @Test
+ void shouldFireExceptionOnEmptyParameter() {
+ String baseUrl = "http://localhost:8080/app4mc/converter";
+ String toCheckUrl = "https://localhost:8080/app4mc/converter/5240471251956758767";
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid(null, baseUrl);
+ });
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid(toCheckUrl, null);
+ });
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid(null, null);
+ });
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid("", baseUrl);
+ });
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid(toCheckUrl, "");
+ });
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid("", "");
+ });
+ }
+
+ @Test
+ void shouldFireExceptionOnInvalidUrl() {
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid("https://localhost:8080/app4mc/converter/5240471251956758767", "localhost:8080/app4mc/converter");
+ });
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.isValid("localhost:8080/app4mc/converter/5240471251956758767", "https://localhost:8080/app4mc/converter");
+ });
+ }
+
+ @Test
+ void shouldGetResultUrlFromLinkHeader() {
+ List<String> linkHeaders = Arrays.asList(
+ "<http://localhost:8080/app4mc/converter/5240471251956758767>;rel=\"self\"",
+ "<http://localhost:8080/app4mc/converter/5240471251956758767/download>;rel=\"result\"");
+
+ String result = WorkflowController.getUrlFromLink(linkHeaders, "result", "http://localhost:8080/app4mc/converter");
+
+ assertEquals("http://localhost:8080/app4mc/converter/5240471251956758767/download", result);
+ }
+
+ @Test
+ void shouldReturnNullForNonExistingRel() {
+ List<String> linkHeaders = Arrays.asList(
+ "<http://localhost:8080/app4mc/converter/5240471251956758767>;rel=\"self\"",
+ "<http://localhost:8080/app4mc/converter/5240471251956758767/download>;rel=\"result\"");
+
+ String result = WorkflowController.getUrlFromLink(linkHeaders, "delete", "http://localhost:8080/app4mc/converter");
+
+ assertNull(result);
+ }
+
+ @Test
+ void shouldReturnMultipleResults() {
+ List<String> linkHeaders = Arrays.asList(
+ "<http://localhost:8080/app4mc/converter/5240471251956758767/simple>;rel=\"result\"",
+ "<http://localhost:8080/app4mc/converter/5240471251956758767/download>;rel=\"result\"");
+
+ List<String> urls = WorkflowController.getUrlsFromLink(linkHeaders, "result", "http://localhost:8080/app4mc/converter");
+
+ assertEquals(2, urls.size());
+ assertTrue(urls.contains("http://localhost:8080/app4mc/converter/5240471251956758767/simple"));
+ assertTrue(urls.contains("http://localhost:8080/app4mc/converter/5240471251956758767/download"));
+ }
+
+ @Test
+ void shouldOnlyReturnValidResult() {
+ List<String> linkHeaders = Arrays.asList(
+ "<http://localhost:8080/app4mc/converter/5240471251956758767/simple>;rel=\"result\"",
+ "<http://www.google.com:8080/app4mc/converter/5240471251956758767/download>;rel=\"result\"");
+
+ List<String> urls = WorkflowController.getUrlsFromLink(linkHeaders, "result", "http://localhost:8080/app4mc/converter");
+
+ assertEquals(1, urls.size());
+ assertTrue(urls.contains("http://localhost:8080/app4mc/converter/5240471251956758767/simple"));
+ }
+
+ @Test
+ void shouldThrowExceptionForNonEqualHosts() {
+ List<String> linkHeaders = Arrays.asList(
+ "<http://www.amazon.com/app4mc/converter/5240471251956758767>;rel=\"result\"",
+ "<http://www.google.com:8080/app4mc/converter/5240471251956758767/download>;rel=\"result\"");
+
+ assertThrows(ProcessingFailedException.class, () -> {
+ WorkflowController.getUrlsFromLink(linkHeaders, "result", "http://localhost:8080/app4mc/converter");
+ });
+
+ }
+}
