Bug 545365: [RJ-Servi] Add lifetime control to pool server to make sure
that all nodes are evicted on shutdown

Change-Id: I825e6eb8f6642012da0e8c2c2b6402749259b263
diff --git a/examples/org.eclipse.statet.rj.example.demo/RJ-Servi StandalonePoolServer.prototype b/examples/org.eclipse.statet.rj.example.demo/RJ-Servi StandalonePoolServer.prototype
new file mode 100644
index 0000000..cf48f88
--- /dev/null
+++ b/examples/org.eclipse.statet.rj.example.demo/RJ-Servi StandalonePoolServer.prototype
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>

+<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication" visibleAttributes="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES, org.eclipse.jdt.launching.VM_ARGUMENTS, org.eclipse.jdt.launching.PROJECT_ATTR, org.eclipse.jdt.launching.PROGRAM_ARGUMENTS, org.eclipse.jdt.launching.MAIN_TYPE, org.eclipse.debug.core.MAPPED_RESOURCE_PATHS, org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD">

+    <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">

+        <listEntry value="/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/StandalonePoolServer.java"/>

+    </listAttribute>

+    <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">

+        <listEntry value="1"/>

+    </listAttribute>

+    <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>

+    <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.eclipse.statet.rj.servi.pool.StandalonePoolServer"/>

+    <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="rservi-pool"/>

+    <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="org.eclipse.statet.rj.example.demo"/>

+    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dorg.eclipse.statet.rj.Path=${resource_loc:/org.eclipse.statet.jcommons.util}/..:,:${resource_loc:/org.eclipse.statet.rj.servi}/.."/>

+</launchConfiguration>

diff --git a/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/internal/rj/servi/PoolManager.java b/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/internal/rj/servi/PoolManager.java
index 11ed62b..6c7273d 100644
--- a/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/internal/rj/servi/PoolManager.java
+++ b/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/internal/rj/servi/PoolManager.java
@@ -41,6 +41,9 @@
 public class PoolManager implements RServiPool, RServiPoolManager {
 	
 	
+	private static final int STOP_ON_ERROR= 1 << 24;
+	
+	
 	private final String id;
 	
 	private final RMIRegistry registry;
@@ -129,7 +132,7 @@
 			}
 			catch (final Exception e) {
 				try {
-					stop(8);
+					stop(STOP_ON_ERROR);
 				}
 				catch (final Exception ignore) {}
 				Utils.logError("An error occurred when binding the pool in the registry.", e);
@@ -155,7 +158,7 @@
 				this.registry.getRegistry().unbind(PoolConfig.getPoolName(this.id));
 			}
 			catch (final Exception e) {
-				if (mode != 8) {
+				if ((mode & STOP_ON_ERROR) != 0) {
 					Utils.logError("An error occurred when unbinding the pool from the registry.", e);
 				}
 			}
@@ -166,14 +169,14 @@
 				UnicastRemoteObject.unexportObject(this, true);
 			}
 			catch (final Exception e) {
-				if (mode != 8) {
+				if ((mode & STOP_ON_ERROR) != 0) {
 					Utils.logError("An error occurred when unexport the pool.", e);
 				}
 			}
 		}
 		
 		try {
-			Thread.sleep(1000);
+			Thread.sleep(500);
 		}
 		catch (final InterruptedException e) {
 		}
diff --git a/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/JMPoolServer.java b/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/JMPoolServer.java
index 782d1c1..bc59ba5 100644
--- a/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/JMPoolServer.java
+++ b/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/JMPoolServer.java
@@ -29,10 +29,14 @@
 import javax.management.OperationsException;
 import javax.rmi.ssl.SslRMIClientSocketFactory;
 
+import org.eclipse.statet.jcommons.collections.CopyOnWriteIdentityListSet;
 import org.eclipse.statet.jcommons.collections.ImCollections;
+import org.eclipse.statet.jcommons.lang.Disposable;
+import org.eclipse.statet.jcommons.lang.Nullable;
 import org.eclipse.statet.jcommons.rmi.RMIAddress;
 import org.eclipse.statet.jcommons.rmi.RMIRegistry;
 import org.eclipse.statet.jcommons.rmi.RMIRegistryManager;
+import org.eclipse.statet.jcommons.runtime.CommonsRuntime;
 import org.eclipse.statet.jcommons.status.NullProgressMonitor;
 import org.eclipse.statet.jcommons.status.StatusException;
 
@@ -61,9 +65,70 @@
 public class JMPoolServer implements PoolServer, PoolServerMXBean {
 	
 	
+	private static final byte DOWN= 0;
+	private static final byte READY= 1;
+	
+	
+	private class LifetimeController extends Thread implements Disposable {
+		
+		private final CopyOnWriteIdentityListSet<PoolManager> stoppedPoolManagers= new CopyOnWriteIdentityListSet<>();
+		
+		public LifetimeController() {
+			super(String.format("RServiPool(%1$s)-KeepAlive", JMPoolServer.this.id));
+			setDaemon(false);
+			setPriority(NORM_PRIORITY - 1);
+			CommonsRuntime.getEnvironment().addStoppingListener(this);
+		}
+		
+		public void add(final PoolManager poolManager) {
+			this.stoppedPoolManagers.add(poolManager);
+		}
+		
+		private int getNodeCount() {
+			int sum= 0;
+			for (final PoolManager manager : this.stoppedPoolManagers) {
+				final int count= manager.getPoolNodeObjects().size();
+				if (count == 0) {
+					this.stoppedPoolManagers.remove(manager);
+				}
+				sum += count;
+			}
+			return sum;
+		}
+		
+		@Override
+		public void run() {
+			while (true) {
+				final int nodeCount= getNodeCount();
+				if (JMPoolServer.this.state == DOWN) {
+					if (nodeCount == 0) {
+						return;
+					}
+				}
+				try {
+					sleep(200);
+				}
+				catch (final InterruptedException e) {
+				}
+			}
+		}
+		
+		@Override
+		public void dispose() {
+			if (JMPoolServer.this.state != DOWN) {
+				shutdown();
+			}
+		}
+		
+	}
+	
+	
 	private final String id;
 	private final RJContext context;
 	
+	private volatile byte state;
+	private final LifetimeController lifetimeController;
+	
 	private final String jmBaseName;
 	private ObjectName jmxName;
 	
@@ -130,10 +195,14 @@
 			if (enableJM) {
 				this.jmNodeConfig.initJM();
 			}
+			
+			this.state= READY;
+			this.lifetimeController= new LifetimeController();
+			this.lifetimeController.start();
 		}
 		catch (final Exception e) {
 			try {
-				shutdown();
+				disposeServer();
 			}
 			catch (final Exception e2) {}
 			throw new RjInitFailedException("Initializing JMX for pool server failed.", e);
@@ -146,6 +215,31 @@
 		}
 	}
 	
+	private void disposeServer() {
+		try {
+			if (this.jmPoolConfig != null) {
+				this.jmPoolConfig.disposeJM();
+			}
+			if (this.jmNetConfig != null) {
+				this.jmNetConfig.disposeJM();
+			}
+			if (this.jmNodeConfig != null) {
+				this.jmNodeConfig.disposeJM();
+			}
+			
+			if (this.jmxName != null) {
+				ManagementFactory.getPlatformMBeanServer().unregisterMBean(this.jmxName);
+				this.jmxName= null;
+			}
+		}
+		catch (final Exception e) {
+			Utils.logError("An error occured when disposing JMX for pool server.", e);
+		}
+		finally {
+			this.state= DOWN;
+		}
+	}
+	
 	
 	@Override
 	public String getId() {
@@ -343,14 +437,19 @@
 				manager.stop(0);
 			}
 			catch (final RjException e) {
-				e.printStackTrace();
+				Utils.logError("An error occured when stopping the pool manager.", e);
+			}
+			finally {
+				if (this.lifetimeController != null) {
+					this.lifetimeController.add(manager);
+				}
 			}
 		}
 	}
 	
 	
 	@Override
-	public PoolManager getManager() {
+	public @Nullable PoolManager getManager() {
 		return this.poolManager;
 	}
 	
@@ -391,6 +490,9 @@
 	
 	@Override
 	public synchronized void start() throws OperationsException {
+		if (this.state == DOWN) {
+			throw new OperationsException("RServi pool server is shut down.");
+		}
 		try {
 			final PoolManager manager= this.poolManager;
 			if (manager != null) {
@@ -419,25 +521,17 @@
 	}
 	
 	public synchronized void shutdown() {
-		stopManager();
-		try {
-			if (this.jmPoolConfig != null) {
-				this.jmPoolConfig.disposeJM();
-			}
-			if (this.jmNetConfig != null) {
-				this.jmNetConfig.disposeJM();
-			}
-			if (this.jmNodeConfig != null) {
-				this.jmNodeConfig.disposeJM();
-			}
-			
-			if (this.jmxName != null) {
-				ManagementFactory.getPlatformMBeanServer().unregisterMBean(this.jmxName);
-				this.jmxName= null;
-			}
+		if (this.state == DOWN) {
+			return;
 		}
-		catch (final Exception e) {
-			Utils.logError("An error occured when disposing JMX for pool server.", e);
+		
+		stopManager();
+		disposeServer();
+	}
+	
+	public void waitForDisposal(final long timeoutMillis) throws InterruptedException {
+		if (this.lifetimeController != null) {
+			this.lifetimeController.join(timeoutMillis);
 		}
 	}
 	
diff --git a/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/StandalonePoolServer.java b/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/StandalonePoolServer.java
index b600e34..d66b783 100644
--- a/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/StandalonePoolServer.java
+++ b/servi/org.eclipse.statet.rj.servi/srcServiPool/org/eclipse/statet/rj/servi/pool/StandalonePoolServer.java
@@ -15,7 +15,6 @@
 package org.eclipse.statet.rj.servi.pool;
 
 import java.io.File;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import javax.management.OperationsException;
 
@@ -71,44 +70,8 @@
 	}
 	
 	
-	private final AtomicBoolean running= new AtomicBoolean(true);
-	
-	
 	protected StandalonePoolServer(final String id, final RJContext context) throws RjInitFailedException {
 		super(id, context);
-		
-		new Thread("KeepAlive") {
-			{
-				setDaemon(false);
-			}
-			@Override
-			public void run() {
-				while (StandalonePoolServer.this.running.get()) {
-					synchronized (StandalonePoolServer.this.running) {
-						try {
-							StandalonePoolServer.this.running.wait();
-						}
-						catch (final InterruptedException e) {
-							Thread.interrupted();
-						}
-					}
-				}
-			}
-		}.start();
-	}
-	
-	
-	@Override
-	public synchronized void shutdown() {
-		try {
-			super.shutdown();
-		}
-		finally {
-			this.running.set(false);
-			synchronized (this.running) {
-				this.running.notifyAll();
-			}
-		}
 	}