Reworks test utils to allow HTTP based smoke tests
diff --git a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/AbstractSmokeTests.java b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/AbstractSmokeTests.java
index 7dcd77d..26dc190 100644
--- a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/AbstractSmokeTests.java
+++ b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/AbstractSmokeTests.java
@@ -11,17 +11,40 @@
 
 package org.eclipse.virgo.test.tools;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.eclipse.virgo.test.tools.JmxUtils.isDefaultJmxPortAvailable;
+import static org.eclipse.virgo.test.tools.JmxUtils.waitForVirgoServerStartFully;
+import static org.eclipse.virgo.test.tools.VirgoServerShutdownThread.shutdown;
+import static org.eclipse.virgo.test.tools.VirgoServerStartupThread.startup;
+import static org.eclipse.virgo.util.io.FileCopyUtils.copy;
+import static org.junit.Assert.assertTrue;
+
 import java.io.File;
-import java.io.IOException;
 
-import org.eclipse.virgo.util.io.FileCopyUtils;
+import org.junit.After;
+import org.junit.Before;
 
-public class AbstractSmokeTests {
+public abstract class AbstractSmokeTests {
 
     private String srcDir = "src/smokeTest/resources";
 
     private File bundlesDir = null;
 
+    protected abstract String getVirgoFlavor();
+
+    @Before
+    public void startServer() throws Exception {
+        assertTrue("Default JMX port in use. Is another Virgo server still up and running?!", isDefaultJmxPortAvailable());
+        startup(ServerUtils.getBinDir(getVirgoFlavor()));
+        assertTrue("Server '" + getVirgoFlavor() + "' not started properly.", waitForVirgoServerStartFully());
+    }
+
+    @After
+    public void shutdownServer() throws Exception {
+        shutdown(ServerUtils.getBinDir(getVirgoFlavor()));
+        assertTrue("Server '" + getVirgoFlavor() + "' not shut down properly.", JmxUtils.waitForVirgoServerShutdownFully());
+    }
+
     private File setupBundleResourcesDir() {
         if (bundlesDir == null) {
             File testExpanded = new File("./" + srcDir);
@@ -30,20 +53,20 @@
         return bundlesDir;
     }
 
-    public void deployTestBundles(String flavor, String bundleName) {
+    public void deployTestBundles(String flavor, String bundleName) throws Exception {
         setupBundleResourcesDir();
-        try {
-            FileCopyUtils.copy(new File(bundlesDir, bundleName), new File(ServerUtils.getPickupDir(flavor), bundleName));
-        } catch (IOException e) {
-            throw new IllegalStateException("Failed to deploy '" + bundleName + "'.");
-        }
+        copy(new File(bundlesDir, bundleName), new File(ServerUtils.getPickupDir(flavor), bundleName));
+        // allow the Server to finish the deployment
+        SECONDS.sleep(5);
     }
 
-    public void undeployTestBundles(String flavor, String bundleName) {
+    public void undeployTestBundles(String flavor, String bundleName) throws Exception {
         setupBundleResourcesDir();
         File file = new File(ServerUtils.getPickupDir(flavor), bundleName);
         if (file.exists()) {
             file.delete();
+            // allow the Server to finish the undeployment
+            SECONDS.sleep(5);
         }
     }
 
diff --git a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/JmxUtils.java b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/JmxUtils.java
index bac5009..524bfc5 100644
--- a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/JmxUtils.java
+++ b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/JmxUtils.java
@@ -1,6 +1,8 @@
 
 package org.eclipse.virgo.test.tools;
 
+import static java.time.Instant.now;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -29,25 +31,24 @@
 
 import org.eclipse.virgo.util.io.NetUtils;
 
-// TODO - find a better name
 // TODO - rework static usage of virgoHome
 public class JmxUtils {
 
-    public static final Duration HALF_SECOND = Duration.ofMillis(500);
+    public static File virgoHome;
 
-    public static final Duration TEN_SECONDS = Duration.ofSeconds(10);
+    private static final Duration HALF_SECOND = Duration.ofMillis(500);
 
-    public static final Duration THIRTY_SECONDS = Duration.ofSeconds(30);
+    private static final Duration THIRTY_SECONDS = Duration.ofSeconds(30);
 
-    public static final Duration TWO_MINUTES = Duration.ofMinutes(2);
+    private static final Duration TWO_MINUTES = Duration.ofMinutes(2);
 
     private static final String ARTIFACT_BUNDLE = "bundle";
 
     private static MBeanServerConnection connection = null;
 
-    public static File virgoHome;
+    private final static int JMX_DEFAULT_PORT = 9875;
 
-    private static final String JMXURL = "service:jmx:rmi:///jndi/rmi://localhost:9875/jmxrmi";
+    private static final String JMXURL = "service:jmx:rmi:///jndi/rmi://localhost:" + JMX_DEFAULT_PORT + "/jmxrmi";
 
     private static final String KEYSTORE = "/configuration/keystore";
 
@@ -57,68 +58,62 @@
 
     private static final String[] DEPLOYMENT_IDENTITY_FIELDS = new String[] { "type", "symbolicName", "version" };
 
-    public final static String STATUS_STARTED = "STARTED";
+    private final static String STATUS_STARTED = "STARTED";
 
-    public final static String STATUS_STARTING = "STARTING";
+    private final static String STATUS_STARTING = "STARTING";
 
-    public static void waitForVirgoServerStartFully() {
-        waitForVirgoServerStartFully(THIRTY_SECONDS, HALF_SECOND);
+    public static boolean isDefaultJmxPortAvailable() {
+        return NetUtils.isPortAvailable(JMX_DEFAULT_PORT);
     }
 
-    private static void waitForVirgoServerStartFully(Duration duration, Duration interval) {
+    public static boolean isKernelStarted() {
+        return JmxUtils.STATUS_STARTED.equals(getKernelStatus());
+    }
+
+    public static boolean waitForVirgoServerStartFully() throws Exception {
         Instant start = Instant.now();
-        while (start.plus(duration).isAfter(Instant.now())) {
+        while (start.plus(THIRTY_SECONDS).isAfter(Instant.now())) {
             try {
-                if (getKernelStatus().equals(STATUS_STARTED)) {
-                    return;
-                } else if (getKernelStatus().equals(STATUS_STARTING)) {
-                    continue;
+                String currentKernelStatus = getKernelStatus();
+                System.out.println("Current kernel status: '" + currentKernelStatus + "'");
+                switch (currentKernelStatus) {
+                    case STATUS_STARTED:
+                        // wait for startup to complete
+                        SECONDS.sleep(15);
+                        return true;
+                    case STATUS_STARTING:
+                    default:
+                        break;
                 }
-            } catch (Exception e) {
-                // ignore and try again
+            } catch (IllegalStateException e) {
+                // ignore JMX related exception and try again?
             }
-            try {
-                Thread.sleep(interval.toMillis());
-            } catch (InterruptedException e) {
-                Thread.interrupted();
-            }
+            SECONDS.sleep(1);
         }
-        // allow some more time to finish startup
-        try {
-            Thread.sleep(TEN_SECONDS.toMillis());
-        } catch (InterruptedException e) {
-            Thread.interrupted();
-        }
+        // startup failed
+        return false;
     }
 
-    public static void waitForVirgoServerShutdownFully() {
-        waitForVirgoServerShutdownFully(THIRTY_SECONDS, HALF_SECOND);
-    }
-
-    private static void waitForVirgoServerShutdownFully(Duration duration, Duration interval) {
-        Instant start = Instant.now();
-        while (start.plus(duration).isAfter(Instant.now())) {
+    public static boolean waitForVirgoServerShutdownFully() throws Exception {
+        Instant start = now();
+        while (start.plus(THIRTY_SECONDS).isAfter(now())) {
+            if (isDefaultJmxPortAvailable()) {
+                // allow some more time to finish shutdown
+                SECONDS.sleep(5);
+                return true;
+            }
             try {
+                String currentKernelStatus = getKernelStatus();
+                System.out.println("Current kernel status: '" + currentKernelStatus + "'");
                 if (!getKernelStatus().equals(STATUS_STARTED)) {
-                    if (NetUtils.isPortAvailable(9875)) {
-                        return;
-                    }
                 }
-            } catch (Exception e) {
-                // ignore and try again
+            } catch (IllegalStateException e) {
+                // ignore JMX related exception and try again?
             }
-            try {
-                Thread.sleep(interval.toMillis());
-            } catch (InterruptedException e) {
-                Thread.interrupted();
-            }
+            SECONDS.sleep(1);
         }
-        // allow some more time to finish shutdown
-        try {
-            Thread.sleep(TEN_SECONDS.toMillis());
-        } catch (InterruptedException e) {
-            Thread.interrupted();
-        }
+        // shutdown failed
+        return false;
     }
 
     public static void waitForMBean(String mBeanName) throws Exception {
@@ -137,7 +132,7 @@
                     return;
                 }
             } catch (IOException e) {
-                // swallow and retry
+                // swallow and retry - No JMX server available (yet)
             }
             Thread.sleep(sleepDuration.toMillis());
         }
@@ -182,7 +177,7 @@
         fail(String.format("After %d ms, artifact %s mbean Status was", duration, name) + mbeanStatus);
     }
 
-    public static MBeanServerConnection getMBeanServerConnection() {
+    public static MBeanServerConnection getMBeanServerConnection() throws IOException {
         String severDir = null;
 
         String[] creds = { "admin", "admin" };
@@ -203,8 +198,6 @@
             connection = JMXConnectorFactory.connect(url, env).getMBeanServerConnection();
         } catch (MalformedURLException e) {
             throw new IllegalStateException("Failed to create JMX connection to '" + JMXURL + "'.", e);
-        } catch (IOException e) {
-            throw new IllegalStateException("Failed to create JMX connection to '" + JMXURL + "'.", e);
         }
         return connection;
     }
diff --git a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/UrlWaitLatch.java b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/UrlWaitLatch.java
index 6def8ed..98269cd 100644
--- a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/UrlWaitLatch.java
+++ b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/UrlWaitLatch.java
@@ -11,8 +11,14 @@
 
 package org.eclipse.virgo.test.tools;
 
+import static java.time.Instant.now;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.apache.http.HttpStatus.SC_UNAUTHORIZED;
 import static org.junit.Assert.fail;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Timer;
 import java.util.TimerTask;
 
@@ -20,12 +26,10 @@
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.HttpHostConnectException;
 import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.HttpClientBuilder;
 
 public class UrlWaitLatch {
 
@@ -42,56 +46,53 @@
     }
 
     public static int waitFor(String url, long interval, long duration) {
-        CloseableHttpClient client = HttpClients.createDefault();
-        return wait(url, client, null, interval, duration);
+        return waitFor(url, null, null, interval, duration);
     }
 
     public static int waitFor(String url, String username, String password, long interval, long duration) {
-        CloseableHttpClient client = HttpClients.createDefault();
-        CredentialsProvider credsProvider = new BasicCredentialsProvider();
-        credsProvider.setCredentials(
-            new AuthScope("localhost", AuthScope.ANY_PORT), 
-            new UsernamePasswordCredentials("admin", "admin"));
-        HttpClientContext context = HttpClientContext.create();
-        context.setCredentialsProvider(credsProvider);
-        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(25).build();
-		context.setRequestConfig(requestConfig);
-        return wait(url, client, context, interval, duration);
-    }
 
-    private static int wait(String url, HttpClient client, HttpClientContext context, long interval, long duration) {
+        HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
+        if (username != null) {
+            CredentialsProvider provider = new BasicCredentialsProvider();
+            UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(username, password);
+            provider.setCredentials(AuthScope.ANY, credentials);
+            httpClientBuilder.setDefaultCredentialsProvider(provider);
+        }
+        HttpClient client = httpClientBuilder.build();
+
         final HttpGet get = new HttpGet(url);
-
-        int hardTimeoutInSeconds = 75;
         TimerTask task = new TimerTask() {
+
             @Override
             public void run() {
                 if (get != null) {
-                	System.err.println("Operation timed out. Aborting request.");
+                    System.err.println("Operation timed out. Aborting request.");
                     get.abort();
                 }
             }
         };
-        new Timer(true).schedule(task, hardTimeoutInSeconds * 1000);
+        new Timer(true).schedule(task, Duration.ofSeconds(60).toMillis());
         try {
-            long startTime = System.currentTimeMillis();
-            int statusCode = 999;
-            while (System.currentTimeMillis() - startTime < duration) {
-            	if (context != null) {
-            		statusCode = client.execute(get).getStatusLine().getStatusCode();
-				} else {
-					statusCode = client.execute(get, context).getStatusLine().getStatusCode();
-				}
-                if (statusCode != 200 || statusCode != 404) {
-                	task.cancel();
-                    return statusCode;
+            int statusCode = -1;
+            Instant start = now();
+            while (start.plus(Duration.ofSeconds(30)).isAfter(now())) {
+                try {
+                    statusCode = client.execute(get).getStatusLine().getStatusCode();
+                } catch (HttpHostConnectException e) {
+                    System.out.println("Connection refused. The servlet container seems not be ready yet.");
                 }
                 System.out.println("Current status Code: " + statusCode);
-                Thread.sleep(interval);
+                if (statusCode == SC_OK || statusCode == SC_UNAUTHORIZED) {
+                    task.cancel();
+                    return statusCode;
+                }
+                System.out.println("Sleeping for " + interval + " millis.");
+                MILLISECONDS.sleep(interval);
             }
 
             fail(String.format("After %d ms, status code was %d", duration, statusCode));
         } catch (Exception e) {
+            e.printStackTrace();
             fail("Failed to connect to '" + url + "'.");
         } finally {
             get.releaseConnection();
@@ -99,21 +100,22 @@
         throw new RuntimeException("Failed to connect to '" + url + "'.");
     }
 
-    public static void main(String[] args) {
-		checkHttpPage("Checking splash screen...", "http://localhost:8080/");
-		checkHttpPage("Checking admin screen...", "http://localhost:8080/admin");
-		checkHttpPage("Checking admin login...", "http://localhost:8080/admin/content/overview", "admin", "admin");
-	}
+    public static void main(String[] args) throws Exception {
+         checkHttpPage("Checking splash screen...", "http://localhost:8080/");
+         checkHttpPage("Checking admin screen...", "http://localhost:8080/admin");
+         checkHttpPage("Checking admin login with (admin/admin)...", "http://localhost:8080/admin/content/overview", "admin", "admin");
+         checkHttpPage("Checking admin login with (foo/bar)...", "http://localhost:8080/admin/content/overview", "foo", "bar");
+    }
 
-	private static void checkHttpPage(String message, String url) {
-		System.out.print(message);
+    private static void checkHttpPage(String message, String url) {
+        System.out.print(message);
         int returnCode = UrlWaitLatch.waitFor(url);
         System.out.println(returnCode);
-	}
+    }
 
-	private static void checkHttpPage(String message, String url, String username, String password) {
-		System.out.print(message);
-		int returnCode = UrlWaitLatch.waitFor(url, username, password);
-		System.out.println(returnCode);
-	}
+    private static void checkHttpPage(String message, String url, String username, String password) {
+        System.out.print(message);
+        int returnCode = UrlWaitLatch.waitFor(url, username, password);
+        System.out.println(returnCode);
+    }
 }
diff --git a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerShutdownThread.java b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerShutdownThread.java
index e49573d..1563800 100644
--- a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerShutdownThread.java
+++ b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerShutdownThread.java
@@ -1,9 +1,9 @@
+
 package org.eclipse.virgo.test.tools;
 
 import java.io.File;
 import java.io.IOException;
 
-
 public class VirgoServerShutdownThread extends AbstractServerCommandThread {
 
     private File shutdown = null;
@@ -12,7 +12,11 @@
 
     private File shutdownURI = null;
 
-    public VirgoServerShutdownThread(String binDir) {
+    public static void shutdown(String binDir) {
+        new Thread(new VirgoServerShutdownThread(binDir)).start();
+    }
+
+    private VirgoServerShutdownThread(String binDir) {
         super(binDir);
     }
 
diff --git a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerStartupThread.java b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerStartupThread.java
index 61c6f29..7972a4e 100644
--- a/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerStartupThread.java
+++ b/org.eclipse.virgo.test.tools/src/main/java/org/eclipse/virgo/test/tools/VirgoServerStartupThread.java
@@ -12,10 +12,14 @@
 
     private File startupURI = null;
 
-    public VirgoServerStartupThread(String binDir) {
+    public static void startup(String binDir) {
+        new Thread(new VirgoServerStartupThread(binDir)).start();
+    }
+
+    private VirgoServerStartupThread(String binDir) {
         super(binDir);
     }
-    
+
     @Override
     public void run() {
         try {