Merged branch 'jetty-9.3.x' into 'master'.
diff --git a/VERSION.txt b/VERSION.txt
index 252d0a8..cad370f 100644
--- a/VERSION.txt
+++ b/VERSION.txt
@@ -1,4 +1,4 @@
-jetty-9.3.8-SNAPSHOT
+jetty-9.4.0-SNAPSHOT
 
 jetty-9.3.7.v20160115 - 15 January 2016
  + 471171 Support SYNC_FLUSH in GzipHandler
diff --git a/aggregates/jetty-all-compact3/pom.xml b/aggregates/jetty-all-compact3/pom.xml
index 1af2650..c323c8b 100644
--- a/aggregates/jetty-all-compact3/pom.xml
+++ b/aggregates/jetty-all-compact3/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
diff --git a/aggregates/jetty-all/pom.xml b/aggregates/jetty-all/pom.xml
index 1c938a1..3e0e91e 100644
--- a/aggregates/jetty-all/pom.xml
+++ b/aggregates/jetty-all/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
diff --git a/aggregates/jetty-websocket-all/pom.xml b/aggregates/jetty-websocket-all/pom.xml
index 3c9c268..ee65e23 100644
--- a/aggregates/jetty-websocket-all/pom.xml
+++ b/aggregates/jetty-websocket-all/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.1.0-SNAPSHOT</version>
+    <version>9.1.3-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -24,7 +24,7 @@
             </goals>
             <configuration>
               <excludes>**/MANIFEST.MF</excludes>
-              <excludeGroupIds>org.slf4j,org.eclipse.jetty.orbit,org.mortbay.jetty.alpn</excludeGroupIds>
+              <excludeGroupIds>javax.annotations,org.objectweb.asm,javax.servlet,org.slf4j,org.eclipse.jetty.orbit,org.mortbay.jetty.npn</excludeGroupIds>
               <outputDirectory>${project.build.directory}/classes</outputDirectory>
               <overWriteReleases>false</overWriteReleases>
               <overWriteSnapshots>true</overWriteSnapshots>
diff --git a/apache-jsp/pom.xml b/apache-jsp/pom.xml
index bd7e796..66ecfd4 100644
--- a/apache-jsp/pom.xml
+++ b/apache-jsp/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>apache-jsp</artifactId>
diff --git a/apache-jsp/src/main/config/modules/apache-jsp.mod b/apache-jsp/src/main/config/modules/apache-jsp.mod
index 5123670..c816f61 100644
--- a/apache-jsp/src/main/config/modules/apache-jsp.mod
+++ b/apache-jsp/src/main/config/modules/apache-jsp.mod
@@ -1,6 +1,5 @@
-#
-# Apache JSP Module
-#
+[description]
+Enables use of the apache implementation of JSP
 
 [name]
 apache-jsp
diff --git a/apache-jstl/pom.xml b/apache-jstl/pom.xml
index 8813af1..3396c84 100644
--- a/apache-jstl/pom.xml
+++ b/apache-jstl/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>apache-jstl</artifactId>
diff --git a/apache-jstl/src/main/config/modules/apache-jstl.mod b/apache-jstl/src/main/config/modules/apache-jstl.mod
index e4a9001..d7c703e 100644
--- a/apache-jstl/src/main/config/modules/apache-jstl.mod
+++ b/apache-jstl/src/main/config/modules/apache-jstl.mod
@@ -1,6 +1,5 @@
-#
-# Apache JSTL 
-#
+[description]
+Enables the apache version of JSTL
 
 [name]
 apache-jstl
diff --git a/examples/async-rest/async-rest-jar/pom.xml b/examples/async-rest/async-rest-jar/pom.xml
index 72bf17a..1d3dc19 100644
--- a/examples/async-rest/async-rest-jar/pom.xml
+++ b/examples/async-rest/async-rest-jar/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>example-async-rest</artifactId>
-       <version>9.3.8-SNAPSHOT</version>
+       <version>9.4.0-SNAPSHOT</version>
   </parent>  
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.eclipse.jetty.example-async-rest</groupId>
diff --git a/examples/async-rest/async-rest-webapp/pom.xml b/examples/async-rest/async-rest-webapp/pom.xml
index 863dc9a..52b08a7 100644
--- a/examples/async-rest/async-rest-webapp/pom.xml
+++ b/examples/async-rest/async-rest-webapp/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>example-async-rest</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>  
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.eclipse.jetty.example-async-rest</groupId>
diff --git a/examples/async-rest/pom.xml b/examples/async-rest/pom.xml
index 1daa80a..3a3f023 100644
--- a/examples/async-rest/pom.xml
+++ b/examples/async-rest/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty.examples</groupId>
     <artifactId>examples-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>  
   <modelVersion>4.0.0</modelVersion>
diff --git a/examples/embedded/pom.xml b/examples/embedded/pom.xml
index cc74e19..93806b9 100644
--- a/examples/embedded/pom.xml
+++ b/examples/embedded/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty.examples</groupId>
     <artifactId>examples-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
diff --git a/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebApp.java b/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebApp.java
index f20c621..59f29d5 100644
--- a/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebApp.java
+++ b/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebApp.java
@@ -52,9 +52,8 @@
         WebAppContext webapp = new WebAppContext();
         webapp.setContextPath("/");
         File warFile = new File(
-                "../../jetty-distribution/target/distribution/test/webapps/test/");
+                "../../tests/test-jmx/jmx-webapp/target/jmx-webapp");
         webapp.setWar(warFile.getAbsolutePath());
-        webapp.addAliasCheck(new AllowSymLinkAliasChecker());
 
         // A WebAppContext is a ContextHandler as well so it needs to be set to
         // the server so it is aware of where to send the appropriate requests.
diff --git a/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebAppWithJsp.java b/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebAppWithJsp.java
index b296428..4a86662 100644
--- a/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebAppWithJsp.java
+++ b/examples/embedded/src/main/java/org/eclipse/jetty/embedded/OneWebAppWithJsp.java
@@ -62,6 +62,7 @@
                     + warFile.getAbsolutePath() );
         }
         webapp.setWar( warFile.getAbsolutePath() );
+        webapp.setExtractWAR(true);
 
         // This webapp will use jsps and jstl. We need to enable the
         // AnnotationConfiguration in order to correctly
@@ -100,6 +101,8 @@
 
         // Start things up! 
         server.start();
+        
+        server.dumpStdErr();
 
         // The use of server.join() the will make the current thread join and
         // wait until the server is done executing.
diff --git a/examples/pom.xml b/examples/pom.xml
index f7d5888..230e23d 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -1,27 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!-- 
-// ========================================================================
-// Copyright (c) Webtide LLC
-// 
-// All rights reserved. This program and the accompanying materials
-// are made available under the terms of the Eclipse Public License v1.0
-// and Apache License v2.0 which accompanies this distribution.
-//
-// The Eclipse Public License is available at 
-// http://www.eclipse.org/legal/epl-v10.html
-//
-// The Apache License v2.0 is available at
-// http://www.apache.org/licenses/LICENSE-2.0.txt
-//
-// You may elect to redistribute this code under either of these licenses. 
-// ========================================================================
- -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <groupId>org.eclipse.jetty.examples</groupId>
diff --git a/jetty-alpn/jetty-alpn-client/pom.xml b/jetty-alpn/jetty-alpn-client/pom.xml
index 79193a0..5128921 100644
--- a/jetty-alpn/jetty-alpn-client/pom.xml
+++ b/jetty-alpn/jetty-alpn-client/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-alpn-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-alpn-client</artifactId>
diff --git a/jetty-alpn/jetty-alpn-server/pom.xml b/jetty-alpn/jetty-alpn-server/pom.xml
index 250efcb..98fc525 100644
--- a/jetty-alpn/jetty-alpn-server/pom.xml
+++ b/jetty-alpn/jetty-alpn-server/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-alpn-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-alpn-server</artifactId>
diff --git a/jetty-alpn/jetty-alpn-server/src/main/config/modules/alpn.mod b/jetty-alpn/jetty-alpn-server/src/main/config/modules/alpn.mod
index 7928e64..1099750 100644
--- a/jetty-alpn/jetty-alpn-server/src/main/config/modules/alpn.mod
+++ b/jetty-alpn/jetty-alpn-server/src/main/config/modules/alpn.mod
@@ -1,11 +1,10 @@
-# ALPN is provided via a -Xbootclasspath that modifies the secure connections
-# in java to support the ALPN layer needed for HTTP/2.
-#
-# This modification has a tight dependency on specific recent updates of
-# Java 1.7 and Java 1.8 (Java versions prior to 1.7u40 are not supported).
-#
-# The alpn module will use an appropriate alpn-boot jar for your
-# specific version of Java.
+[description]
+Enables the ALPN extension to TLS(SSL) by adding modified classes to
+the JVM bootpath. 
+This modification has a tight dependency on specific recent updates of
+Java 1.7 and Java 1.8 (Java versions prior to 1.7u40 are not supported).
+The alpn module will use an appropriate alpn-boot jar for your
+specific version of Java.
 #
 # IMPORTANT: Versions of Java that exist after this module was created are
 #            not guaranteed to work with existing alpn-boot jars, and might
diff --git a/jetty-alpn/pom.xml b/jetty-alpn/pom.xml
index d54ab79..4157d6c 100644
--- a/jetty-alpn/pom.xml
+++ b/jetty-alpn/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-alpn-parent</artifactId>
diff --git a/jetty-annotations/pom.xml b/jetty-annotations/pom.xml
index c57d6bc..8f43209 100644
--- a/jetty-annotations/pom.xml
+++ b/jetty-annotations/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-annotations</artifactId>
diff --git a/jetty-annotations/src/main/config/modules/annotations.mod b/jetty-annotations/src/main/config/modules/annotations.mod
index 65e4654..4217be5 100644
--- a/jetty-annotations/src/main/config/modules/annotations.mod
+++ b/jetty-annotations/src/main/config/modules/annotations.mod
@@ -1,15 +1,11 @@
-#
-# Jetty Annotation Scanning Module
-#
+[description]
+Enables Annotation scanning for deployed webapplications.
 
 [depend]
-# Annotations needs plus, and jndi features
 plus
 
 [lib]
-# Annotations needs jetty annotation jars
 lib/jetty-annotations-${jetty.version}.jar
-# Need annotation processing jars too
 lib/annotations/*.jar
 
 [xml]
diff --git a/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/AnnotationParser.java b/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/AnnotationParser.java
index 14bec7e..8724882 100644
--- a/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/AnnotationParser.java
+++ b/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/AnnotationParser.java
@@ -558,7 +558,7 @@
             if (!isParsed(className) || resolver.shouldOverride(className))
             {
                 className = className.replace('.', '/')+".class";
-                URL resource = Loader.getResource(this.getClass(), className);
+                URL resource = Loader.getResource(className);
                 if (resource!= null)
                 {
                     Resource r = Resource.newResource(resource);
@@ -593,7 +593,7 @@
                 if (!isParsed(cz.getName()) || resolver.shouldOverride(cz.getName()))
                 {
                     String nameAsResource = cz.getName().replace('.', '/')+".class";
-                    URL resource = Loader.getResource(this.getClass(), nameAsResource);
+                    URL resource = Loader.getResource(nameAsResource);
                     if (resource!= null)
                     {
                         Resource r = Resource.newResource(resource);
@@ -652,7 +652,7 @@
                 if ((resolver == null) || (!resolver.isExcluded(s) &&  (!isParsed(s) || resolver.shouldOverride(s))))
                 {
                     s = s.replace('.', '/')+".class";
-                    URL resource = Loader.getResource(this.getClass(), s);
+                    URL resource = Loader.getResource(s);
                     if (resource!= null)
                     {
                         Resource r = Resource.newResource(resource);
diff --git a/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/Util.java b/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/Util.java
index a277316..9aa259c 100644
--- a/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/Util.java
+++ b/jetty-annotations/src/main/java/org/eclipse/jetty/annotations/Util.java
@@ -201,7 +201,7 @@
             }
             case Type.OBJECT:
             {
-                return (Loader.loadClass(null, t.getClassName()));
+                return (Loader.loadClass(t.getClassName()));
             }
             case Type.SHORT:
             {
diff --git a/jetty-ant/pom.xml b/jetty-ant/pom.xml
index 5576f54..3d97aac 100644
--- a/jetty-ant/pom.xml
+++ b/jetty-ant/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-ant</artifactId>
diff --git a/jetty-cdi/cdi-core/pom.xml b/jetty-cdi/cdi-core/pom.xml
index cd16e02..ea31517 100644
--- a/jetty-cdi/cdi-core/pom.xml
+++ b/jetty-cdi/cdi-core/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty.cdi</groupId>
     <artifactId>jetty-cdi-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>cdi-core</artifactId>
diff --git a/jetty-cdi/cdi-full-servlet/pom.xml b/jetty-cdi/cdi-full-servlet/pom.xml
index fead448..f850914 100644
--- a/jetty-cdi/cdi-full-servlet/pom.xml
+++ b/jetty-cdi/cdi-full-servlet/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty.cdi</groupId>
     <artifactId>jetty-cdi-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>cdi-full-servlet</artifactId>
diff --git a/jetty-cdi/cdi-servlet/pom.xml b/jetty-cdi/cdi-servlet/pom.xml
index 1b05b0d..e3b0593 100644
--- a/jetty-cdi/cdi-servlet/pom.xml
+++ b/jetty-cdi/cdi-servlet/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty.cdi</groupId>
     <artifactId>jetty-cdi-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>cdi-servlet</artifactId>
diff --git a/jetty-cdi/cdi-servlet/src/main/config/modules/cdi.mod b/jetty-cdi/cdi-servlet/src/main/config/modules/cdi.mod
index ebffb55..68a926d 100644
--- a/jetty-cdi/cdi-servlet/src/main/config/modules/cdi.mod
+++ b/jetty-cdi/cdi-servlet/src/main/config/modules/cdi.mod
@@ -1,6 +1,5 @@
-#
-# [EXPERIMENTAL] CDI / Weld Jetty module
-#
+[description]
+Experimental CDI/Weld integration
 
 [depend]
 deploy
diff --git a/jetty-cdi/cdi-websocket/pom.xml b/jetty-cdi/cdi-websocket/pom.xml
index fd8bfc6..07c6a5a 100644
--- a/jetty-cdi/cdi-websocket/pom.xml
+++ b/jetty-cdi/cdi-websocket/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty.cdi</groupId>
     <artifactId>jetty-cdi-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>cdi-websocket</artifactId>
diff --git a/jetty-cdi/pom.xml b/jetty-cdi/pom.xml
index eab873b..b1cd311 100644
--- a/jetty-cdi/pom.xml
+++ b/jetty-cdi/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.eclipse.jetty.cdi</groupId>
diff --git a/jetty-cdi/test-cdi-it/pom.xml b/jetty-cdi/test-cdi-it/pom.xml
index 160bdb0..5329541 100644
--- a/jetty-cdi/test-cdi-it/pom.xml
+++ b/jetty-cdi/test-cdi-it/pom.xml
@@ -1,21 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!-- 
-// ========================================================================
-// Copyright (c) Webtide LLC
-// 
-// All rights reserved. This program and the accompanying materials
-// are made available under the terms of the Eclipse Public License v1.0
-// and Apache License v2.0 which accompanies this distribution.
-//
-// The Eclipse Public License is available at 
-// http://www.eclipse.org/legal/epl-v10.html
-//
-// The Apache License v2.0 is available at
-// http://www.apache.org/licenses/LICENSE-2.0.txt
-//
-// You may elect to redistribute this code under either of these licenses. 
-// ========================================================================
- -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <parent>
diff --git a/jetty-cdi/test-cdi-webapp/pom.xml b/jetty-cdi/test-cdi-webapp/pom.xml
index 51cd8f1..8044dc3 100644
--- a/jetty-cdi/test-cdi-webapp/pom.xml
+++ b/jetty-cdi/test-cdi-webapp/pom.xml
@@ -1,26 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  // ========================================================================
-  // Copyright (c) Webtide LLC
-  //
-  // All rights reserved. This program and the accompanying materials
-  // are made available under the terms of the Eclipse Public License v1.0
-  // and Apache License v2.0 which accompanies this distribution.
-  //
-  // The Eclipse Public License is available at
-  // http://www.eclipse.org/legal/epl-v10.html
-  //
-  // The Apache License v2.0 is available at
-  // http://www.apache.org/licenses/LICENSE-2.0.txt
-  //
-  // You may elect to redistribute this code under either of these licenses.
-  // ========================================================================
--->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <parent>
     <groupId>org.eclipse.jetty.cdi</groupId>
     <artifactId>jetty-cdi-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>test-cdi-webapp</artifactId>
diff --git a/jetty-client/pom.xml b/jetty-client/pom.xml
index f2d2613..9450cdc 100644
--- a/jetty-client/pom.xml
+++ b/jetty-client/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
@@ -48,6 +48,44 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.4.2</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <shadedArtifactAttached>true</shadedArtifactAttached>
+              <shadedClassifierName>hybrid</shadedClassifierName>
+              <artifactSet>
+                <includes>
+                  <include>org.eclipse.jetty:jetty-http</include>
+                  <include>org.eclipse.jetty:jetty-io</include>
+                  <include>org.eclipse.jetty:jetty-util</include>
+                </includes>
+              </artifactSet>
+              <relocations>
+                <relocation>
+                  <pattern>org.eclipse.jetty.http</pattern>
+                  <shadedPattern>org.eclipse.jetty.client.shaded.http</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.eclipse.jetty.io</pattern>
+                  <shadedPattern>org.eclipse.jetty.client.shaded.io</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.eclipse.jetty.util</pattern>
+                  <shadedPattern>org.eclipse.jetty.client.shaded.util</shadedPattern>
+                </relocation>
+              </relocations>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 
diff --git a/jetty-client/src/main/config/modules/client.mod b/jetty-client/src/main/config/modules/client.mod
index 39b58d4..e9d13c8 100644
--- a/jetty-client/src/main/config/modules/client.mod
+++ b/jetty-client/src/main/config/modules/client.mod
@@ -1,6 +1,5 @@
-#
-# Client Feature
-#
+[description]
+Adds the Jetty HTTP client to the server classpath.
 
 [lib]
 lib/jetty-client-${jetty.version}.jar
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractConnectionPool.java b/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractConnectionPool.java
new file mode 100644
index 0000000..bcac677
--- /dev/null
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractConnectionPool.java
@@ -0,0 +1,199 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.client;
+
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.eclipse.jetty.client.api.Connection;
+import org.eclipse.jetty.client.api.Destination;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.Promise;
+import org.eclipse.jetty.util.annotation.ManagedAttribute;
+import org.eclipse.jetty.util.component.ContainerLifeCycle;
+import org.eclipse.jetty.util.component.Dumpable;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+
+public abstract class AbstractConnectionPool implements ConnectionPool, Dumpable
+{
+    private static final Logger LOG = Log.getLogger(AbstractConnectionPool.class);
+
+    private final AtomicBoolean closed = new AtomicBoolean();
+    private final AtomicInteger connectionCount = new AtomicInteger();
+    private final Destination destination;
+    private final int maxConnections;
+    private final Callback requester;
+
+    protected AbstractConnectionPool(Destination destination, int maxConnections, Callback requester)
+    {
+        this.destination = destination;
+        this.maxConnections = maxConnections;
+        this.requester = requester;
+    }
+
+    @ManagedAttribute(value = "The max number of connections", readonly = true)
+    public int getMaxConnectionCount()
+    {
+        return maxConnections;
+    }
+
+    @ManagedAttribute(value = "The number of connections", readonly = true)
+    public int getConnectionCount()
+    {
+        return connectionCount.get();
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return connectionCount.get() == 0;
+    }
+
+    @Override
+    public boolean isClosed()
+    {
+        return closed.get();
+    }
+
+    @Override
+    public Connection acquire()
+    {
+        Connection connection = activate();
+        if (connection == null)
+            connection = tryCreate();
+        return connection;
+    }
+
+    private Connection tryCreate()
+    {
+        while (true)
+        {
+            int current = getConnectionCount();
+            final int next = current + 1;
+
+            if (next > maxConnections)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Max connections {}/{} reached", current, maxConnections);
+                // Try again the idle connections
+                return activate();
+            }
+
+            if (connectionCount.compareAndSet(current, next))
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Connection {}/{} creation", next, maxConnections);
+
+                destination.newConnection(new Promise<Connection>()
+                {
+                    @Override
+                    public void succeeded(Connection connection)
+                    {
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("Connection {}/{} creation succeeded {}", next, maxConnections, connection);
+                        onCreated(connection);
+                        proceed();
+                    }
+
+                    @Override
+                    public void failed(Throwable x)
+                    {
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("Connection " + next + "/" + maxConnections + " creation failed", x);
+                        connectionCount.decrementAndGet();
+                        requester.failed(x);
+                    }
+                });
+
+                // Try again the idle connections
+                return activate();
+            }
+        }
+    }
+
+    protected abstract void onCreated(Connection connection);
+
+    protected void proceed()
+    {
+        requester.succeeded();
+    }
+
+    protected abstract Connection activate();
+
+    protected Connection active(Connection connection)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("Connection active {}", connection);
+        acquired(connection);
+        return connection;
+    }
+
+    protected void acquired(Connection connection)
+    {
+    }
+
+    protected boolean idle(Connection connection, boolean close)
+    {
+        if (close)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Connection idle close {}", connection);
+            return false;
+        }
+        else
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Connection idle {}", connection);
+            return true;
+        }
+    }
+
+    protected void released(Connection connection)
+    {
+    }
+
+    protected void removed(Connection connection)
+    {
+        int pooled = connectionCount.decrementAndGet();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Connection removed {} - pooled: {}", connection, pooled);
+    }
+
+    @Override
+    public void close()
+    {
+        if (closed.compareAndSet(false, true))
+        {
+            connectionCount.set(0);
+        }
+    }
+
+    protected void close(Collection<Connection> connections)
+    {
+        connections.forEach(Connection::close);
+    }
+
+    @Override
+    public String dump()
+    {
+        return ContainerLifeCycle.dump(this);
+    }
+}
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java b/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java
index 544ab0a..441224c 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java
@@ -22,6 +22,7 @@
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.net.SocketException;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.SocketChannel;
 import java.util.Map;
@@ -31,6 +32,7 @@
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.SelectChannelEndPoint;
 import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.io.SocketChannelEndPoint;
 import org.eclipse.jetty.io.ssl.SslClientConnectionFactory;
 import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
@@ -173,13 +175,15 @@
         }
 
         @Override
-        protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key)
+        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key)
         {
-            return new SelectChannelEndPoint(channel, selector, key, getScheduler(), client.getIdleTimeout());
+            SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler());
+            endp.setIdleTimeout(client.getIdleTimeout());
+            return endp;
         }
 
         @Override
-        public org.eclipse.jetty.io.Connection newConnection(SocketChannel channel, EndPoint endPoint, Object attachment) throws IOException
+        public org.eclipse.jetty.io.Connection newConnection(SelectableChannel channel, EndPoint endPoint, Object attachment) throws IOException
         {
             @SuppressWarnings("unchecked")
             Map<String, Object> context = (Map<String, Object>)attachment;
@@ -188,7 +192,7 @@
         }
 
         @Override
-        protected void connectionFailed(SocketChannel channel, Throwable x, Object attachment)
+        protected void connectionFailed(SelectableChannel channel, Throwable x, Object attachment)
         {
             @SuppressWarnings("unchecked")
             Map<String, Object> context = (Map<String, Object>)attachment;
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/ConnectionPool.java b/jetty-client/src/main/java/org/eclipse/jetty/client/ConnectionPool.java
index f85c32f..9642fc7 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/ConnectionPool.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/ConnectionPool.java
@@ -18,17 +18,24 @@
 
 package org.eclipse.jetty.client;
 
-import org.eclipse.jetty.client.api.Destination;
-import org.eclipse.jetty.util.Callback;
+import java.io.Closeable;
 
-/**
- * @deprecated use {@link DuplexConnectionPool} instead
- */
-@Deprecated
-public class ConnectionPool extends DuplexConnectionPool
+import org.eclipse.jetty.client.api.Connection;
+
+public interface ConnectionPool extends Closeable
 {
-    public ConnectionPool(Destination destination, int maxConnections, Callback requester)
-    {
-        super(destination, maxConnections, requester);
-    }
+    boolean isActive(Connection connection);
+
+    boolean isEmpty();
+
+    boolean isClosed();
+
+    Connection acquire();
+
+    boolean release(Connection connection);
+
+    boolean remove(Connection connection);
+
+    @Override
+    void close();
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/DuplexConnectionPool.java b/jetty-client/src/main/java/org/eclipse/jetty/client/DuplexConnectionPool.java
index efe7cf6..18f0ff4 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/DuplexConnectionPool.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/DuplexConnectionPool.java
@@ -18,21 +18,20 @@
 
 package org.eclipse.jetty.client;
 
-import java.io.Closeable;
 import java.io.IOException;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Deque;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Queue;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Set;
 import java.util.concurrent.locks.ReentrantLock;
 
 import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.Destination;
-import org.eclipse.jetty.util.BlockingArrayQueue;
 import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
 import org.eclipse.jetty.util.annotation.ManagedObject;
 import org.eclipse.jetty.util.component.ContainerLifeCycle;
@@ -42,31 +41,29 @@
 import org.eclipse.jetty.util.thread.Sweeper;
 
 @ManagedObject("The connection pool")
-public class DuplexConnectionPool implements Closeable, Dumpable, Sweeper.Sweepable
+public class DuplexConnectionPool extends AbstractConnectionPool implements Dumpable, Sweeper.Sweepable
 {
     private static final Logger LOG = Log.getLogger(DuplexConnectionPool.class);
 
-    private final AtomicInteger connectionCount = new AtomicInteger();
     private final ReentrantLock lock = new ReentrantLock();
-    private final Destination destination;
-    private final int maxConnections;
-    private final Callback requester;
     private final Deque<Connection> idleConnections;
-    private final Queue<Connection> activeConnections;
+    private final Set<Connection> activeConnections;
 
     public DuplexConnectionPool(Destination destination, int maxConnections, Callback requester)
     {
-        this.destination = destination;
-        this.maxConnections = maxConnections;
-        this.requester = requester;
-        this.idleConnections = new LinkedBlockingDeque<>(maxConnections);
-        this.activeConnections = new BlockingArrayQueue<>(maxConnections);
+        super(destination, maxConnections, requester);
+        this.idleConnections = new ArrayDeque<>(maxConnections);
+        this.activeConnections = new HashSet<>(maxConnections);
     }
 
-    @ManagedAttribute(value = "The number of connections", readonly = true)
-    public int getConnectionCount()
+    protected void lock()
     {
-        return connectionCount.get();
+        lock.lock();
+    }
+
+    protected void unlock()
+    {
+        lock.unlock();
     }
 
     @ManagedAttribute(value = "The number of idle connections", readonly = true)
@@ -102,139 +99,76 @@
         return idleConnections;
     }
 
-    public Queue<Connection> getActiveConnections()
+    public Collection<Connection> getActiveConnections()
     {
         return activeConnections;
     }
 
-    public Connection acquire()
+    @Override
+    public boolean isActive(Connection connection)
     {
-        Connection connection = activateIdle();
-        if (connection == null)
-            connection = tryCreate();
-        return connection;
-    }
-
-    private Connection tryCreate()
-    {
-        while (true)
+        lock();
+        try
         {
-            int current = getConnectionCount();
-            final int next = current + 1;
-
-            if (next > maxConnections)
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Max connections {}/{} reached", current, maxConnections);
-                // Try again the idle connections
-                return activateIdle();
-            }
-
-            if (connectionCount.compareAndSet(current, next))
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Connection {}/{} creation", next, maxConnections);
-
-                destination.newConnection(new Promise<Connection>()
-                {
-                    @Override
-                    public void succeeded(Connection connection)
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Connection {}/{} creation succeeded {}", next, maxConnections, connection);
-
-                        idleCreated(connection);
-
-                        proceed();
-                    }
-
-                    @Override
-                    public void failed(Throwable x)
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Connection " + next + "/" + maxConnections + " creation failed", x);
-
-                        connectionCount.decrementAndGet();
-
-                        requester.failed(x);
-                    }
-                });
-
-                // Try again the idle connections
-                return activateIdle();
-            }
+            return activeConnections.contains(connection);
+        }
+        finally
+        {
+            unlock();
         }
     }
 
-    protected void proceed()
+    @Override
+    protected void onCreated(Connection connection)
     {
-        requester.succeeded();
-    }
-
-    protected void idleCreated(Connection connection)
-    {
-        boolean idle;
         lock();
         try
         {
             // Use "cold" new connections as last.
-            idle = idleConnections.offerLast(connection);
+            idleConnections.offer(connection);
         }
         finally
         {
             unlock();
         }
 
-        idle(connection, idle);
+        idle(connection, false);
     }
 
-    private Connection activateIdle()
+    @Override
+    protected Connection activate()
     {
-        boolean acquired;
         Connection connection;
         lock();
         try
         {
-            connection = idleConnections.pollFirst();
+            connection = idleConnections.poll();
             if (connection == null)
                 return null;
-            acquired = activeConnections.offer(connection);
+            activeConnections.add(connection);
         }
         finally
         {
             unlock();
         }
 
-        if (acquired)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Connection active {}", connection);
-            acquired(connection);
-            return connection;
-        }
-        else
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Connection active overflow {}", connection);
-            connection.close();
-            return null;
-        }
-    }
-
-    protected void acquired(Connection connection)
-    {
+        return active(connection);
     }
 
     public boolean release(Connection connection)
     {
-        boolean idle;
+        boolean closed = isClosed();
         lock();
         try
         {
             if (!activeConnections.remove(connection))
                 return false;
-            // Make sure we use "hot" connections first.
-            idle = offerIdle(connection);
+
+            if (!closed)
+            {
+                // Make sure we use "hot" connections first.
+                deactivate(connection);
+            }
         }
         finally
         {
@@ -242,35 +176,14 @@
         }
 
         released(connection);
-        return idle(connection, idle);
+        return idle(connection, closed);
     }
 
-    protected boolean offerIdle(Connection connection)
+    protected boolean deactivate(Connection connection)
     {
         return idleConnections.offerFirst(connection);
     }
 
-    protected boolean idle(Connection connection, boolean idle)
-    {
-        if (idle)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Connection idle {}", connection);
-            return true;
-        }
-        else
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Connection idle overflow {}", connection);
-            connection.close();
-            return false;
-        }
-    }
-
-    protected void released(Connection connection)
-    {
-    }
-
     public boolean remove(Connection connection)
     {
         return remove(connection, false);
@@ -295,55 +208,21 @@
             released(connection);
         boolean removed = activeRemoved || idleRemoved || force;
         if (removed)
-        {
-            int pooled = connectionCount.decrementAndGet();
-            if (LOG.isDebugEnabled())
-                LOG.debug("Connection removed {} - pooled: {}", connection, pooled);
-        }
+            removed(connection);
         return removed;
     }
 
-    public boolean isActive(Connection connection)
-    {
-        lock();
-        try
-        {
-            return activeConnections.contains(connection);
-        }
-        finally
-        {
-            unlock();
-        }
-    }
-
-    public boolean isIdle(Connection connection)
-    {
-        lock();
-        try
-        {
-            return idleConnections.contains(connection);
-        }
-        finally
-        {
-            unlock();
-        }
-    }
-
-    public boolean isEmpty()
-    {
-        return connectionCount.get() == 0;
-    }
-
     public void close()
     {
-        List<Connection> idles = new ArrayList<>();
-        List<Connection> actives = new ArrayList<>();
+        super.close();
+
+        List<Connection> connections = new ArrayList<>();
         lock();
         try
         {
-            idles.addAll(idleConnections);
+            connections.addAll(idleConnections);
             idleConnections.clear();
-            actives.addAll(activeConnections);
+            connections.addAll(activeConnections);
             activeConnections.clear();
         }
         finally
@@ -351,32 +230,18 @@
             unlock();
         }
 
-        connectionCount.set(0);
-
-        for (Connection connection : idles)
-            connection.close();
-
-        // A bit drastic, but we cannot wait for all requests to complete
-        for (Connection connection : actives)
-            connection.close();
-    }
-
-    @Override
-    public String dump()
-    {
-        return ContainerLifeCycle.dump(this);
+        close(connections);
     }
 
     @Override
     public void dump(Appendable out, String indent) throws IOException
     {
-        List<Connection> actives = new ArrayList<>();
-        List<Connection> idles = new ArrayList<>();
+        List<Connection> connections = new ArrayList<>();
         lock();
         try
         {
-            actives.addAll(activeConnections);
-            idles.addAll(idleConnections);
+            connections.addAll(activeConnections);
+            connections.addAll(idleConnections);
         }
         finally
         {
@@ -384,7 +249,7 @@
         }
 
         ContainerLifeCycle.dumpObject(out, this);
-        ContainerLifeCycle.dump(out, indent, actives, idles);
+        ContainerLifeCycle.dump(out, indent, connections);
     }
 
     @Override
@@ -422,16 +287,6 @@
         return false;
     }
 
-    protected void lock()
-    {
-        lock.lock();
-    }
-
-    protected void unlock()
-    {
-        lock.unlock();
-    }
-
     @Override
     public String toString()
     {
@@ -450,8 +305,8 @@
 
         return String.format("%s[c=%d/%d,a=%d,i=%d]",
                 getClass().getSimpleName(),
-                connectionCount.get(),
-                maxConnections,
+                getConnectionCount(),
+                getMaxConnectionCount(),
                 activeSize,
                 idleSize);
     }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpChannel.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpChannel.java
index 24058f0..342bae3 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpChannel.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpChannel.java
@@ -129,6 +129,11 @@
         return getHttpReceiver().abort(exchange, failure);
     }
 
+    public Result exchangeTerminating(HttpExchange exchange, Result result)
+    {
+        return result;
+    }
+
     public void exchangeTerminated(HttpExchange exchange, Result result)
     {
         disassociate(exchange);
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java
index 3852ecd..ba4e658 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java
@@ -524,15 +524,12 @@
      */
     public List<Destination> getDestinations()
     {
-        return new ArrayList<Destination>(destinations.values());
+        return new ArrayList<>(destinations.values());
     }
 
     protected void send(final HttpRequest request, List<Response.ResponseListener> listeners)
     {
         String scheme = request.getScheme().toLowerCase(Locale.ENGLISH);
-        if (!HttpScheme.HTTP.is(scheme) && !HttpScheme.HTTPS.is(scheme))
-            throw new IllegalArgumentException("Invalid protocol " + scheme);
-
         String host = request.getHost().toLowerCase(Locale.ENGLISH);
         HttpDestination destination = destinationFor(scheme, host, request.getPort());
         destination.send(request, listeners);
@@ -1040,12 +1037,25 @@
 
     protected int normalizePort(String scheme, int port)
     {
-        return port > 0 ? port : HttpScheme.HTTPS.is(scheme) ? 443 : 80;
+        if (port > 0)
+            return port;
+        else if (isSchemeSecure(scheme))
+            return 443;
+        else
+            return 80;
     }
 
     public boolean isDefaultPort(String scheme, int port)
     {
-        return HttpScheme.HTTPS.is(scheme) ? port == 443 : port == 80;
+        if (isSchemeSecure(scheme))
+            return port == 443;
+        else
+            return port == 80;
+    }
+
+    public boolean isSchemeSecure(String scheme)
+    {
+        return HttpScheme.HTTPS.is(scheme) || HttpScheme.WSS.is(scheme);
     }
 
     private class ContentDecoderFactorySet implements Set<ContentDecoder.Factory>
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpDestination.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpDestination.java
index 169eb9a..9e16e10 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpDestination.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpDestination.java
@@ -22,19 +22,21 @@
 import java.io.IOException;
 import java.nio.channels.AsynchronousCloseException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.RejectedExecutionException;
 
 import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.Destination;
+import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
-import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.ssl.SslClientConnectionFactory;
 import org.eclipse.jetty.util.BlockingArrayQueue;
+import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
 import org.eclipse.jetty.util.annotation.ManagedObject;
@@ -42,9 +44,10 @@
 import org.eclipse.jetty.util.component.Dumpable;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.Sweeper;
 
 @ManagedObject
-public abstract class HttpDestination extends ContainerLifeCycle implements Destination, Closeable, Dumpable
+public abstract class HttpDestination extends ContainerLifeCycle implements Destination, Closeable, Callback, Dumpable
 {
     protected static final Logger LOG = Log.getLogger(HttpDestination.class);
 
@@ -56,6 +59,7 @@
     private final ProxyConfiguration.Proxy proxy;
     private final ClientConnectionFactory connectionFactory;
     private final HttpField hostField;
+    private ConnectionPool connectionPool;
 
     public HttpDestination(HttpClient client, Origin origin)
     {
@@ -76,7 +80,7 @@
         }
         else
         {
-            if (HttpScheme.HTTPS.is(getScheme()))
+            if (isSecure())
                 connectionFactory = newSslClientConnectionFactory(connectionFactory);
         }
         this.connectionFactory = connectionFactory;
@@ -87,6 +91,29 @@
         hostField = new HttpField(HttpHeader.HOST, host);
     }
 
+    @Override
+    protected void doStart() throws Exception
+    {
+        this.connectionPool = newConnectionPool(client);
+        addBean(connectionPool);
+        super.doStart();
+        Sweeper sweeper = client.getBean(Sweeper.class);
+        if (sweeper != null && connectionPool instanceof Sweeper.Sweepable)
+            sweeper.offer((Sweeper.Sweepable)connectionPool);
+    }
+
+    @Override
+    protected void doStop() throws Exception
+    {
+        Sweeper sweeper = client.getBean(Sweeper.class);
+        if (sweeper != null && connectionPool instanceof Sweeper.Sweepable)
+            sweeper.remove((Sweeper.Sweepable)connectionPool);
+        super.doStop();
+        removeBean(connectionPool);
+    }
+
+    protected abstract ConnectionPool newConnectionPool(HttpClient client);
+
     protected Queue<HttpExchange> newExchangeQueue(HttpClient client)
     {
         return new BlockingArrayQueue<>(client.getMaxRequestsQueuedPerDestination());
@@ -97,6 +124,11 @@
         return new SslClientConnectionFactory(client.getSslContextFactory(), client.getByteBufferPool(), client.getExecutor(), connectionFactory);
     }
 
+    public boolean isSecure()
+    {
+        return client.isSchemeSecure(getScheme());
+    }
+
     public HttpClient getHttpClient()
     {
         return client;
@@ -171,6 +203,24 @@
         return hostField;
     }
 
+    @ManagedAttribute(value = "The connection pool", readonly = true)
+    public ConnectionPool getConnectionPool()
+    {
+        return connectionPool;
+    }
+
+    @Override
+    public void succeeded()
+    {
+        send();
+    }
+
+    @Override
+    public void failed(Throwable x)
+    {
+        abort(x);
+    }
+
     protected void send(HttpRequest request, List<Response.ResponseListener> listeners)
     {
         if (!getScheme().equalsIgnoreCase(request.getScheme()))
@@ -217,7 +267,78 @@
         return queue.offer(exchange);
     }
 
-    public abstract void send();
+    public void send()
+    {
+        if (getHttpExchanges().isEmpty())
+            return;
+        process();
+    }
+
+    private void process()
+    {
+        while (true)
+        {
+            Connection connection = connectionPool.acquire();
+            if (connection == null)
+                break;
+            boolean proceed = process(connection);
+            if (!proceed)
+                break;
+        }
+    }
+
+    public boolean process(final Connection connection)
+    {
+        HttpClient client = getHttpClient();
+        final HttpExchange exchange = getHttpExchanges().poll();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Processing exchange {} on {} of {}", exchange, connection, this);
+        if (exchange == null)
+        {
+            if (!connectionPool.release(connection))
+                connection.close();
+            if (!client.isRunning())
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("{} is stopping", client);
+                connection.close();
+            }
+            return false;
+        }
+        else
+        {
+            final Request request = exchange.getRequest();
+            Throwable cause = request.getAbortCause();
+            if (cause != null)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Aborted before processing {}: {}", exchange, cause);
+                // It may happen that the request is aborted before the exchange
+                // is created. Aborting the exchange a second time will result in
+                // a no-operation, so we just abort here to cover that edge case.
+                exchange.abort(cause);
+            }
+            else
+            {
+                SendFailure result = send(connection, exchange);
+                if (result != null)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Send failed {} for {}", result, exchange);
+                    if (result.retry)
+                    {
+                        if (enqueue(getHttpExchanges(), exchange))
+                            return true;
+                    }
+
+                    request.abort(result.failure);
+                }
+            }
+            return getHttpExchanges().peek() != null;
+        }
+    }
+
+    protected abstract SendFailure send(Connection connection, HttpExchange exchange);
 
     public void newConnection(Promise<Connection> promise)
     {
@@ -239,14 +360,67 @@
         abort(new AsynchronousCloseException());
         if (LOG.isDebugEnabled())
             LOG.debug("Closed {}", this);
+        connectionPool.close();
     }
 
     public void release(Connection connection)
     {
+        if (LOG.isDebugEnabled())
+            LOG.debug("Released {}", connection);
+        HttpClient client = getHttpClient();
+        if (client.isRunning())
+        {
+            if (connectionPool.isActive(connection))
+            {
+                if (connectionPool.release(connection))
+                    send();
+                else
+                    connection.close();
+            }
+            else
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Released explicit {}", connection);
+            }
+        }
+        else
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("{} is stopped", client);
+            connection.close();
+        }
+    }
+
+    public boolean remove(Connection connection)
+    {
+        return connectionPool.remove(connection);
     }
 
     public void close(Connection connection)
     {
+        boolean removed = remove(connection);
+
+        if (getHttpExchanges().isEmpty())
+        {
+            if (getHttpClient().isRemoveIdleDestinations() && connectionPool.isEmpty())
+            {
+                // There is a race condition between this thread removing the destination
+                // and another thread queueing a request to this same destination.
+                // If this destination is removed, but the request queued, a new connection
+                // will be opened, the exchange will be executed and eventually the connection
+                // will idle timeout and be closed. Meanwhile a new destination will be created
+                // in HttpClient and will be used for other requests.
+                getHttpClient().removeDestination(this);
+            }
+        }
+        else
+        {
+            // We need to execute queued requests even if this connection failed.
+            // We may create a connection that is not needed, but it will eventually
+            // idle timeout, so no worries.
+            if (removed)
+                process();
+        }
     }
 
     /**
@@ -274,6 +448,7 @@
     public void dump(Appendable out, String indent) throws IOException
     {
         ContainerLifeCycle.dumpObject(out, toString());
+        ContainerLifeCycle.dump(out, indent, Collections.singletonList(connectionPool));
     }
 
     public String asString()
@@ -284,11 +459,12 @@
     @Override
     public String toString()
     {
-        return String.format("%s[%s]%x%s,queue=%d",
+        return String.format("%s[%s]%x%s,queue=%d,pool=%s",
                 HttpDestination.class.getSimpleName(),
                 asString(),
                 hashCode(),
                 proxy == null ? "" : "(via " + proxy + ")",
-                exchanges.size());
+                exchanges.size(),
+                connectionPool);
     }
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpProxy.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpProxy.java
index cd11b53..acf9a9b 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpProxy.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpProxy.java
@@ -107,7 +107,7 @@
             public void succeeded(Connection connection)
             {
                 HttpDestination destination = (HttpDestination)context.get(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY);
-                if (HttpScheme.HTTPS.is(destination.getScheme()))
+                if (destination.isSecure())
                 {
                     SslContextFactory sslContextFactory = destination.getHttpClient().getSslContextFactory();
                     if (sslContextFactory != null)
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java
index 9a950a3..a126771 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java
@@ -444,6 +444,7 @@
 
         if (result != null)
         {
+            result = channel.exchangeTerminating(exchange, result);
             boolean ordered = getHttpDestination().getHttpClient().isStrictEventOrdering();
             if (!ordered)
                 channel.exchangeTerminated(exchange, result);
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java
index c930740..de1cbd6 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java
@@ -376,6 +376,7 @@
         }
         else
         {
+            result = channel.exchangeTerminating(exchange, result);
             HttpDestination destination = getHttpChannel().getHttpDestination();
             boolean ordered = destination.getHttpClient().isStrictEventOrdering();
             if (!ordered)
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/LeakTrackingConnectionPool.java b/jetty-client/src/main/java/org/eclipse/jetty/client/LeakTrackingConnectionPool.java
index 47f7613..ece808f 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/LeakTrackingConnectionPool.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/LeakTrackingConnectionPool.java
@@ -25,7 +25,7 @@
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
-public class LeakTrackingConnectionPool extends ConnectionPool
+public class LeakTrackingConnectionPool extends DuplexConnectionPool
 {
     private static final Logger LOG = Log.getLogger(LeakTrackingConnectionPool.class);
 
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexConnectionPool.java b/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexConnectionPool.java
new file mode 100644
index 0000000..2bd8b48
--- /dev/null
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexConnectionPool.java
@@ -0,0 +1,328 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.client;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+import org.eclipse.jetty.client.api.Connection;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.component.ContainerLifeCycle;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+
+public class MultiplexConnectionPool extends AbstractConnectionPool
+{
+    private static final Logger LOG = Log.getLogger(MultiplexConnectionPool.class);
+
+    private final ReentrantLock lock = new ReentrantLock();
+    private final Deque<Holder> idleConnections;
+    private final Map<Connection, Holder> muxedConnections;
+    private final Map<Connection, Holder> busyConnections;
+    private int maxMultiplex;
+
+    public MultiplexConnectionPool(HttpDestination destination, int maxConnections, Callback requester, int maxMultiplex)
+    {
+        super(destination, maxConnections, requester);
+        this.idleConnections = new ArrayDeque<>(maxConnections);
+        this.muxedConnections = new HashMap<>(maxConnections);
+        this.busyConnections = new HashMap<>(maxConnections);
+        this.maxMultiplex = maxMultiplex;
+    }
+
+    protected void lock()
+    {
+        lock.lock();
+    }
+
+    protected void unlock()
+    {
+        lock.unlock();
+    }
+
+    public int getMaxMultiplex()
+    {
+        lock();
+        try
+        {
+            return maxMultiplex;
+        }
+        finally
+        {
+            unlock();
+        }
+    }
+
+    public void setMaxMultiplex(int maxMultiplex)
+    {
+        lock();
+        try
+        {
+            this.maxMultiplex = maxMultiplex;
+        }
+        finally
+        {
+            unlock();
+        }
+    }
+
+    @Override
+    public boolean isActive(Connection connection)
+    {
+        lock();
+        try
+        {
+            if (muxedConnections.containsKey(connection))
+                return true;
+            if (busyConnections.containsKey(connection))
+                return true;
+            return false;
+        }
+        finally
+        {
+            unlock();
+        }
+    }
+
+    @Override
+    protected void onCreated(Connection connection)
+    {
+        lock();
+        try
+        {
+            // Use "cold" connections as last.
+            idleConnections.offer(new Holder(connection));
+        }
+        finally
+        {
+            unlock();
+        }
+
+        idle(connection, false);
+    }
+
+    @Override
+    protected Connection activate()
+    {
+        Holder holder;
+        lock();
+        try
+        {
+            while (true)
+            {
+                if (muxedConnections.isEmpty())
+                {
+                    holder = idleConnections.poll();
+                    if (holder == null)
+                        return null;
+                    muxedConnections.put(holder.connection, holder);
+                }
+                else
+                {
+                    holder = muxedConnections.values().iterator().next();
+                }
+
+                if (holder.count < maxMultiplex)
+                {
+                    ++holder.count;
+                    break;
+                }
+                else
+                {
+                    muxedConnections.remove(holder.connection);
+                    busyConnections.put(holder.connection, holder);
+                }
+            }
+        }
+        finally
+        {
+            unlock();
+        }
+
+        return active(holder.connection);
+    }
+
+    @Override
+    public boolean release(Connection connection)
+    {
+        boolean closed = isClosed();
+        boolean idle = false;
+        Holder holder;
+        lock();
+        try
+        {
+            holder = muxedConnections.get(connection);
+            if (holder != null)
+            {
+                int count = --holder.count;
+                if (count == 0)
+                {
+                    muxedConnections.remove(connection);
+                    if (!closed)
+                    {
+                        idleConnections.offerFirst(holder);
+                        idle = true;
+                    }
+                }
+            }
+            else
+            {
+                holder = busyConnections.remove(connection);
+                if (holder != null)
+                {
+                    int count = --holder.count;
+                    if (!closed)
+                    {
+                        if (count == 0)
+                        {
+                            idleConnections.offerFirst(holder);
+                            idle = true;
+                        }
+                        else
+                        {
+                            muxedConnections.put(connection, holder);
+                        }
+                    }
+                }
+            }
+        }
+        finally
+        {
+            unlock();
+        }
+
+        if (holder == null)
+            return false;
+
+        released(connection);
+        if (idle || closed)
+            return idle(connection, closed);
+        return true;
+    }
+
+    @Override
+    public boolean remove(Connection connection)
+    {
+        return remove(connection, false);
+    }
+
+    protected boolean remove(Connection connection, boolean force)
+    {
+        boolean activeRemoved = true;
+        boolean idleRemoved = false;
+        lock();
+        try
+        {
+            Holder holder = muxedConnections.remove(connection);
+            if (holder == null)
+                holder = busyConnections.remove(connection);
+            if (holder == null)
+            {
+                activeRemoved = false;
+                for (Iterator<Holder> iterator = idleConnections.iterator(); iterator.hasNext();)
+                {
+                    holder = iterator.next();
+                    if (holder.connection == connection)
+                    {
+                        idleRemoved = true;
+                        iterator.remove();
+                        break;
+                    }
+                }
+            }
+        }
+        finally
+        {
+            unlock();
+        }
+
+        if (activeRemoved || force)
+            released(connection);
+        boolean removed = activeRemoved || idleRemoved || force;
+        if (removed)
+            removed(connection);
+        return removed;
+    }
+
+    @Override
+    public void close()
+    {
+        super.close();
+
+        List<Connection> connections;
+        lock();
+        try
+        {
+            connections = idleConnections.stream().map(holder -> holder.connection).collect(Collectors.toList());
+            connections.addAll(muxedConnections.keySet());
+            connections.addAll(busyConnections.keySet());
+        }
+        finally
+        {
+            unlock();
+        }
+
+        close(connections);
+    }
+
+    @Override
+    public void dump(Appendable out, String indent) throws IOException
+    {
+        List<Holder> connections = new ArrayList<>();
+        lock();
+        try
+        {
+            connections.addAll(busyConnections.values());
+            connections.addAll(muxedConnections.values());
+            connections.addAll(idleConnections);
+        }
+        finally
+        {
+            unlock();
+        }
+
+        ContainerLifeCycle.dumpObject(out, this);
+        ContainerLifeCycle.dump(out, indent, connections);
+    }
+
+    private static class Holder
+    {
+        private final Connection connection;
+        private int count;
+
+        private Holder(Connection connection)
+        {
+            this.connection = connection;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s[%d]", connection, count);
+        }
+    }
+}
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexHttpDestination.java b/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexHttpDestination.java
index c4b84e3..c114f46 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexHttpDestination.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexHttpDestination.java
@@ -18,183 +18,31 @@
 
 package org.eclipse.jetty.client;
 
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.util.Promise;
-
-public abstract class MultiplexHttpDestination<C extends Connection> extends HttpDestination implements Promise<Connection>
+public abstract class MultiplexHttpDestination extends HttpDestination
 {
-    private final AtomicReference<ConnectState> connect = new AtomicReference<>(ConnectState.DISCONNECTED);
-    private final AtomicInteger requestsPerConnection = new AtomicInteger();
-    private int maxRequestsPerConnection = 1024;
-    private C connection;
-
     protected MultiplexHttpDestination(HttpClient client, Origin origin)
     {
         super(client, origin);
     }
 
+    protected ConnectionPool newConnectionPool(HttpClient client)
+    {
+        return new MultiplexConnectionPool(this, client.getMaxConnectionsPerDestination(), this,
+                client.getMaxRequestsQueuedPerDestination());
+    }
+
     public int getMaxRequestsPerConnection()
     {
-        return maxRequestsPerConnection;
+        ConnectionPool connectionPool = getConnectionPool();
+        if (connectionPool instanceof MultiplexConnectionPool)
+            return ((MultiplexConnectionPool)connectionPool).getMaxMultiplex();
+        return 1;
     }
 
     public void setMaxRequestsPerConnection(int maxRequestsPerConnection)
     {
-        this.maxRequestsPerConnection = maxRequestsPerConnection;
-    }
-
-    @Override
-    public void send()
-    {
-        while (true)
-        {
-            ConnectState current = connect.get();
-            switch (current)
-            {
-                case DISCONNECTED:
-                {
-                    if (!connect.compareAndSet(current, ConnectState.CONNECTING))
-                        break;
-                    newConnection(this);
-                    return;
-                }
-                case CONNECTING:
-                {
-                    // Waiting to connect, just return
-                    return;
-                }
-                case CONNECTED:
-                {
-                    if (process(connection))
-                        break;
-                    return;
-                }
-                default:
-                {
-                    abort(new IllegalStateException("Invalid connection state " + current));
-                    return;
-                }
-            }
-        }
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public void succeeded(Connection result)
-    {
-        C connection = this.connection = (C)result;
-        if (connect.compareAndSet(ConnectState.CONNECTING, ConnectState.CONNECTED))
-        {
-            send();
-        }
-        else
-        {
-            connection.close();
-            failed(new IllegalStateException("Invalid connection state " + connect));
-        }
-    }
-
-    @Override
-    public void failed(Throwable x)
-    {
-        connect.set(ConnectState.DISCONNECTED);
-        abort(x);
-    }
-
-    protected boolean process(final C connection)
-    {
-        while (true)
-        {
-            int max = getMaxRequestsPerConnection();
-            int count = requestsPerConnection.get();
-            int next = count + 1;
-            if (next > max)
-                return false;
-
-            if (requestsPerConnection.compareAndSet(count, next))
-            {
-                HttpExchange exchange = getHttpExchanges().poll();
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Processing {}/{} {} on {}", next, max, exchange, connection);
-                if (exchange == null)
-                {
-                    requestsPerConnection.decrementAndGet();
-                    return false;
-                }
-
-                final Request request = exchange.getRequest();
-                Throwable cause = request.getAbortCause();
-                if (cause != null)
-                {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Aborted before processing {}: {}", exchange, cause);
-                    // It may happen that the request is aborted before the exchange
-                    // is created. Aborting the exchange a second time will result in
-                    // a no-operation, so we just abort here to cover that edge case.
-                    exchange.abort(cause);
-                    requestsPerConnection.decrementAndGet();
-                }
-                else
-                {
-                    SendFailure result = send(connection, exchange);
-                    if (result != null)
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Send failed {} for {}", result, exchange);
-                        if (result.retry)
-                        {
-                            if (enqueue(getHttpExchanges(), exchange))
-                                return true;
-                        }
-
-                        request.abort(result.failure);
-                    }
-                }
-                return getHttpExchanges().peek() != null;
-            }
-        }
-    }
-
-    @Override
-    public void release(Connection connection)
-    {
-        requestsPerConnection.decrementAndGet();
-        send();
-    }
-
-    @Override
-    public void close()
-    {
-        super.close();
-        C connection = this.connection;
-        if (connection != null)
-            connection.close();
-    }
-
-    @Override
-    public void close(Connection connection)
-    {
-        super.close(connection);
-        while (true)
-        {
-            ConnectState current = connect.get();
-            if (connect.compareAndSet(current, ConnectState.DISCONNECTED))
-            {
-                if (getHttpClient().isRemoveIdleDestinations())
-                    getHttpClient().removeDestination(this);
-                break;
-            }
-        }
-    }
-
-    protected abstract SendFailure send(C connection, HttpExchange exchange);
-
-    private enum ConnectState
-    {
-        DISCONNECTED, CONNECTING, CONNECTED
+        ConnectionPool connectionPool = getConnectionPool();
+        if (connectionPool instanceof MultiplexConnectionPool)
+            ((MultiplexConnectionPool)connectionPool).setMaxMultiplex(maxRequestsPerConnection);
     }
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/PoolingHttpDestination.java b/jetty-client/src/main/java/org/eclipse/jetty/client/PoolingHttpDestination.java
index 8416832..f6a94e3 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/PoolingHttpDestination.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/PoolingHttpDestination.java
@@ -18,244 +18,15 @@
 
 package org.eclipse.jetty.client;
 
-import java.io.IOException;
-import java.util.Collections;
-
-import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.annotation.ManagedAttribute;
-import org.eclipse.jetty.util.annotation.ManagedObject;
-import org.eclipse.jetty.util.component.ContainerLifeCycle;
-import org.eclipse.jetty.util.thread.Sweeper;
-
-@ManagedObject
-public abstract class PoolingHttpDestination<C extends Connection> extends HttpDestination implements Callback
+public abstract class PoolingHttpDestination extends HttpDestination
 {
-    private DuplexConnectionPool connectionPool;
-
     public PoolingHttpDestination(HttpClient client, Origin origin)
     {
         super(client, origin);
-        this.connectionPool = newConnectionPool(client);
-        addBean(connectionPool);
-        Sweeper sweeper = client.getBean(Sweeper.class);
-        if (sweeper != null)
-            sweeper.offer(connectionPool);
     }
 
-    @Override
-    protected void doStart() throws Exception
-    {
-        HttpClient client = getHttpClient();
-        this.connectionPool = newConnectionPool(client);
-        addBean(connectionPool);
-        super.doStart();
-        Sweeper sweeper = client.getBean(Sweeper.class);
-        if (sweeper != null)
-            sweeper.offer(connectionPool);
-    }
-
-    @Override
-    protected void doStop() throws Exception
-    {
-        HttpClient client = getHttpClient();
-        Sweeper sweeper = client.getBean(Sweeper.class);
-        if (sweeper != null)
-            sweeper.remove(connectionPool);
-        super.doStop();
-        removeBean(connectionPool);
-    }
-
-    protected DuplexConnectionPool newConnectionPool(HttpClient client)
+    protected ConnectionPool newConnectionPool(HttpClient client)
     {
         return new DuplexConnectionPool(this, client.getMaxConnectionsPerDestination(), this);
     }
-
-    @ManagedAttribute(value = "The connection pool", readonly = true)
-    public DuplexConnectionPool getConnectionPool()
-    {
-        return connectionPool;
-    }
-
-    @Override
-    public void succeeded()
-    {
-        send();
-    }
-
-    @Override
-    public void failed(final Throwable x)
-    {
-        abort(x);
-    }
-
-    public void send()
-    {
-        if (getHttpExchanges().isEmpty())
-            return;
-        process();
-    }
-
-    @SuppressWarnings("unchecked")
-    public C acquire()
-    {
-        return (C)connectionPool.acquire();
-    }
-
-    private void process()
-    {
-        while (true)
-        {
-            C connection = acquire();
-            if (connection == null)
-                break;
-            boolean proceed = process(connection);
-            if (!proceed)
-                break;
-        }
-    }
-
-    /**
-     * <p>Processes a new connection making it idle or active depending on whether requests are waiting to be sent.</p>
-     * <p>A new connection is created when a request needs to be executed; it is possible that the request that
-     * triggered the request creation is executed by another connection that was just released, so the new connection
-     * may become idle.</p>
-     * <p>If a request is waiting to be executed, it will be dequeued and executed by the new connection.</p>
-     *
-     * @param connection the new connection
-     * @return whether to perform more processing
-     */
-    public boolean process(final C connection)
-    {
-        HttpClient client = getHttpClient();
-        final HttpExchange exchange = getHttpExchanges().poll();
-        if (LOG.isDebugEnabled())
-            LOG.debug("Processing exchange {} on {} of {}", exchange, connection, this);
-        if (exchange == null)
-        {
-            if (!connectionPool.release(connection))
-                connection.close();
-            if (!client.isRunning())
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("{} is stopping", client);
-                connection.close();
-            }
-            return false;
-        }
-        else
-        {
-            final Request request = exchange.getRequest();
-            Throwable cause = request.getAbortCause();
-            if (cause != null)
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Aborted before processing {}: {}", exchange, cause);
-                // It may happen that the request is aborted before the exchange
-                // is created. Aborting the exchange a second time will result in
-                // a no-operation, so we just abort here to cover that edge case.
-                exchange.abort(cause);
-            }
-            else
-            {
-                SendFailure result = send(connection, exchange);
-                if (result != null)
-                {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Send failed {} for {}", result, exchange);
-                    if (result.retry)
-                    {
-                        if (enqueue(getHttpExchanges(), exchange))
-                            return true;
-                    }
-
-                    request.abort(result.failure);
-                }
-            }
-            return getHttpExchanges().peek() != null;
-        }
-    }
-
-    protected abstract SendFailure send(C connection, HttpExchange exchange);
-
-    @Override
-    public void release(Connection c)
-    {
-        @SuppressWarnings("unchecked")
-        C connection = (C)c;
-        if (LOG.isDebugEnabled())
-            LOG.debug("Released {}", connection);
-        HttpClient client = getHttpClient();
-        if (client.isRunning())
-        {
-            if (connectionPool.isActive(connection))
-            {
-                if (connectionPool.release(connection))
-                    send();
-                else
-                    connection.close();
-            }
-            else
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Released explicit {}", connection);
-            }
-        }
-        else
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("{} is stopped", client);
-            connection.close();
-        }
-    }
-
-    @Override
-    public void close(Connection connection)
-    {
-        super.close(connection);
-
-        boolean removed = connectionPool.remove(connection);
-
-        if (getHttpExchanges().isEmpty())
-        {
-            if (getHttpClient().isRemoveIdleDestinations() && connectionPool.isEmpty())
-            {
-                // There is a race condition between this thread removing the destination
-                // and another thread queueing a request to this same destination.
-                // If this destination is removed, but the request queued, a new connection
-                // will be opened, the exchange will be executed and eventually the connection
-                // will idle timeout and be closed. Meanwhile a new destination will be created
-                // in HttpClient and will be used for other requests.
-                getHttpClient().removeDestination(this);
-            }
-        }
-        else
-        {
-            // We need to execute queued requests even if this connection failed.
-            // We may create a connection that is not needed, but it will eventually
-            // idle timeout, so no worries.
-            if (removed)
-                process();
-        }
-    }
-
-    public void close()
-    {
-        super.close();
-        connectionPool.close();
-    }
-
-    @Override
-    public void dump(Appendable out, String indent) throws IOException
-    {
-        super.dump(out, indent);
-        ContainerLifeCycle.dump(out, indent, Collections.singletonList(connectionPool));
-    }
-
-    @Override
-    public String toString()
-    {
-        return String.format("%s,pool=%s", super.toString(), connectionPool);
-    }
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/ResponseNotifier.java b/jetty-client/src/main/java/org/eclipse/jetty/client/ResponseNotifier.java
index 0f27789..1d73dc0 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/ResponseNotifier.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/ResponseNotifier.java
@@ -210,7 +210,7 @@
         notifyHeaders(listeners, response);
         if (response instanceof ContentResponse)
             // TODO: handle callback
-            notifyContent(listeners, response, ByteBuffer.wrap(((ContentResponse)response).getContent()), new Callback.Adapter());
+            notifyContent(listeners, response, ByteBuffer.wrap(((ContentResponse)response).getContent()), Callback.NOOP);
         notifySuccess(listeners, response);
     }
 
@@ -232,7 +232,7 @@
         notifyHeaders(listeners, response);
         if (response instanceof ContentResponse)
             // TODO: handle callback
-            notifyContent(listeners, response, ByteBuffer.wrap(((ContentResponse)response).getContent()), new Callback.Adapter());
+            notifyContent(listeners, response, ByteBuffer.wrap(((ContentResponse)response).getContent()), Callback.NOOP);
         notifyFailure(listeners, response, failure);
     }
 
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/Socks4Proxy.java b/jetty-client/src/main/java/org/eclipse/jetty/client/Socks4Proxy.java
index a309fe3..fddcd3c 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/Socks4Proxy.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/Socks4Proxy.java
@@ -27,7 +27,6 @@
 import java.util.regex.Pattern;
 
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.io.AbstractConnection;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.EndPoint;
@@ -196,7 +195,7 @@
                 HttpDestination destination = (HttpDestination)context.get(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY);
                 HttpClient client = destination.getHttpClient();
                 ClientConnectionFactory connectionFactory = this.connectionFactory;
-                if (HttpScheme.HTTPS.is(destination.getScheme()))
+                if (destination.isSecure())
                     connectionFactory = new SslClientConnectionFactory(client.getSslContextFactory(), client.getByteBufferPool(), client.getExecutor(), connectionFactory);
                 org.eclipse.jetty.io.Connection newConnection = connectionFactory.newConnection(getEndPoint(), context);
                 getEndPoint().upgrade(newConnection);
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/ValidatingConnectionPool.java b/jetty-client/src/main/java/org/eclipse/jetty/client/ValidatingConnectionPool.java
index 95d1446..516781a 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/ValidatingConnectionPool.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/ValidatingConnectionPool.java
@@ -56,7 +56,7 @@
  * tuning the idle timeout of the servers to be larger than
  * that of the client.</p>
  */
-public class ValidatingConnectionPool extends ConnectionPool
+public class ValidatingConnectionPool extends DuplexConnectionPool
 {
     private static final Logger LOG = Log.getLogger(ValidatingConnectionPool.class);
 
@@ -154,7 +154,7 @@
     private class Holder implements Runnable
     {
         private final long timestamp = System.nanoTime();
-        private final AtomicBoolean latch = new AtomicBoolean();
+        private final AtomicBoolean done = new AtomicBoolean();
         private final Connection connection;
         public Scheduler.Task task;
 
@@ -166,30 +166,31 @@
         @Override
         public void run()
         {
-            if (latch.compareAndSet(false, true))
+            if (done.compareAndSet(false, true))
             {
-                boolean idle;
+                boolean closed = isClosed();
                 lock();
                 try
                 {
-                    quarantine.remove(connection);
-                    idle = offerIdle(connection);
                     if (LOG.isDebugEnabled())
                         LOG.debug("Validated {}", connection);
+                    quarantine.remove(connection);
+                    if (!closed)
+                        deactivate(connection);
                 }
                 finally
                 {
                     unlock();
                 }
 
-                if (idle(connection, idle))
-                    proceed();
+                idle(connection, closed);
+                proceed();
             }
         }
 
         public boolean cancel()
         {
-            if (latch.compareAndSet(false, true))
+            if (done.compareAndSet(false, true))
             {
                 task.cancel();
                 return true;
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/api/Result.java b/jetty-client/src/main/java/org/eclipse/jetty/client/api/Result.java
index 0ae336e..034053e 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/api/Result.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/api/Result.java
@@ -52,6 +52,14 @@
         this.responseFailure = responseFailure;
     }
 
+    public Result(Result result, Throwable responseFailure)
+    {
+        this.request = result.request;
+        this.requestFailure = result.requestFailure;
+        this.response = result.response;
+        this.responseFailure = responseFailure;
+    }
+
     /**
      * @return the request object
      */
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpChannelOverHTTP.java b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpChannelOverHTTP.java
index cdbf1fa..a922f17 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpChannelOverHTTP.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpChannelOverHTTP.java
@@ -18,16 +18,20 @@
 
 package org.eclipse.jetty.client.http;
 
+import java.util.Locale;
+
 import org.eclipse.jetty.client.HttpChannel;
 import org.eclipse.jetty.client.HttpExchange;
-import org.eclipse.jetty.client.HttpReceiver;
-import org.eclipse.jetty.client.HttpSender;
+import org.eclipse.jetty.client.HttpRequest;
+import org.eclipse.jetty.client.HttpResponse;
+import org.eclipse.jetty.client.HttpResponseException;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpVersion;
 
 public class HttpChannelOverHTTP extends HttpChannel
@@ -55,13 +59,13 @@
     }
 
     @Override
-    protected HttpSender getHttpSender()
+    protected HttpSenderOverHTTP getHttpSender()
     {
         return sender;
     }
 
     @Override
-    protected HttpReceiver getHttpReceiver()
+    protected HttpReceiverOverHTTP getHttpReceiver()
     {
         return receiver;
     }
@@ -85,6 +89,42 @@
         connection.release();
     }
 
+    @Override
+    public Result exchangeTerminating(HttpExchange exchange, Result result)
+    {
+        if (result.isFailed())
+            return result;
+
+        HttpResponse response = exchange.getResponse();
+        
+        if ((response.getVersion() == HttpVersion.HTTP_1_1) && 
+            (response.getStatus() == HttpStatus.SWITCHING_PROTOCOLS_101))
+        {
+            String connection = response.getHeaders().get(HttpHeader.CONNECTION);
+            if ((connection == null) || !connection.toLowerCase(Locale.US).contains("upgrade"))
+            {
+                return new Result(result,new HttpResponseException("101 Switching Protocols without Connection: Upgrade not supported",response));
+            }
+            
+            // Upgrade Response
+            HttpRequest request = exchange.getRequest();
+            if (request instanceof HttpConnectionUpgrader)
+            {
+                HttpConnectionUpgrader listener = (HttpConnectionUpgrader)request;
+                try
+                {
+                    listener.upgrade(response,getHttpConnection());
+                }
+                catch (Throwable x)
+                {
+                    return new Result(result,x);
+                }
+            }
+        }
+
+        return result;
+    }
+
     public void receive()
     {
         receiver.receive();
@@ -131,7 +171,10 @@
         }
         else
         {
-            release();
+            if (response.getStatus() == HttpStatus.SWITCHING_PROTOCOLS_101)
+                connection.remove();
+            else
+                release();
         }
     }
 
@@ -143,4 +186,5 @@
                 sender,
                 receiver);
     }
+
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java
index f6371f4..2b308a6 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java
@@ -18,6 +18,7 @@
 
 package org.eclipse.jetty.client.http;
 
+import java.nio.ByteBuffer;
 import java.nio.channels.AsynchronousCloseException;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -37,7 +38,7 @@
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.Sweeper;
 
-public class HttpConnectionOverHTTP extends AbstractConnection implements Connection, Sweeper.Sweepable
+public class HttpConnectionOverHTTP extends AbstractConnection implements Connection, org.eclipse.jetty.io.Connection.UpgradeFrom, Sweeper.Sweepable
 {
     private static final Logger LOG = Log.getLogger(HttpConnectionOverHTTP.class);
 
@@ -120,6 +121,13 @@
         }
     }
 
+    @Override
+    public ByteBuffer onUpgradeFrom()
+    {
+        HttpReceiverOverHTTP receiver = channel.getHttpReceiver();
+        return receiver.onUpgradeFrom();
+    }
+
     public void release()
     {
         // Restore idle timeout
@@ -167,6 +175,11 @@
         return true;
     }
 
+    public void remove()
+    {
+        getHttpDestination().remove(this);
+    }
+
     @Override
     public String toString()
     {
diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/graph/Predicate.java b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionUpgrader.java
similarity index 78%
copy from jetty-start/src/main/java/org/eclipse/jetty/start/graph/Predicate.java
copy to jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionUpgrader.java
index 0f974cd..2a5dac8 100644
--- a/jetty-start/src/main/java/org/eclipse/jetty/start/graph/Predicate.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionUpgrader.java
@@ -16,12 +16,11 @@
 //  ========================================================================
 //
 
-package org.eclipse.jetty.start.graph;
+package org.eclipse.jetty.client.http;
 
-/**
- * Matcher of Nodes
- */
-public interface Predicate
+import org.eclipse.jetty.client.HttpResponse;
+
+public interface HttpConnectionUpgrader
 {
-    public boolean match(Node<?> input);
+    public void upgrade(HttpResponse response, HttpConnectionOverHTTP connection);
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTP.java b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTP.java
index 37ff0ea..b936529 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTP.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTP.java
@@ -23,8 +23,9 @@
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.PoolingHttpDestination;
 import org.eclipse.jetty.client.SendFailure;
+import org.eclipse.jetty.client.api.Connection;
 
-public class HttpDestinationOverHTTP extends PoolingHttpDestination<HttpConnectionOverHTTP>
+public class HttpDestinationOverHTTP extends PoolingHttpDestination
 {
     public HttpDestinationOverHTTP(HttpClient client, Origin origin)
     {
@@ -32,8 +33,8 @@
     }
 
     @Override
-    protected SendFailure send(HttpConnectionOverHTTP connection, HttpExchange exchange)
+    protected SendFailure send(Connection connection, HttpExchange exchange)
     {
-        return connection.send(exchange);
+        return ((HttpConnectionOverHTTP)connection).send(exchange);
     }
 }
diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java
index a201b31..d414fab 100644
--- a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java
+++ b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java
@@ -88,6 +88,17 @@
         buffer = null;
     }
 
+    protected ByteBuffer onUpgradeFrom()
+    {
+        if (BufferUtil.hasContent(buffer))
+        {
+            ByteBuffer upgradeBuffer = ByteBuffer.allocate(buffer.remaining());
+            upgradeBuffer.put(buffer);
+            return upgradeBuffer;
+        }
+        return null;
+    }
+
     private void process()
     {
         try
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientExplicitConnectionTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientExplicitConnectionTest.java
index b68d614..81455cb 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientExplicitConnectionTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientExplicitConnectionTest.java
@@ -59,7 +59,7 @@
             Assert.assertEquals(200, response.getStatus());
 
             HttpDestinationOverHTTP httpDestination = (HttpDestinationOverHTTP)destination;
-            DuplexConnectionPool connectionPool = httpDestination.getConnectionPool();
+            DuplexConnectionPool connectionPool = (DuplexConnectionPool)httpDestination.getConnectionPool();
             Assert.assertTrue(connectionPool.getActiveConnections().isEmpty());
             Assert.assertTrue(connectionPool.getIdleConnections().isEmpty());
         }
@@ -94,7 +94,7 @@
         Assert.assertFalse(httpConnection.getEndPoint().isOpen());
 
         HttpDestinationOverHTTP httpDestination = (HttpDestinationOverHTTP)destination;
-        DuplexConnectionPool connectionPool = httpDestination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)httpDestination.getConnectionPool();
         Assert.assertTrue(connectionPool.getActiveConnections().isEmpty());
         Assert.assertTrue(connectionPool.getIdleConnections().isEmpty());
     }
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientFailureTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientFailureTest.java
index 48eea5a..0a9c2da 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientFailureTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientFailureTest.java
@@ -25,9 +25,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.client.api.Response;
-import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
 import org.eclipse.jetty.client.util.DeferredContentProvider;
@@ -89,14 +86,7 @@
         try
         {
             client.newRequest("localhost", connector.getLocalPort())
-                    .onRequestHeaders(new Request.HeadersListener()
-                    {
-                        @Override
-                        public void onHeaders(Request request)
-                        {
-                            connectionRef.get().getEndPoint().close();
-                        }
-                    })
+                    .onRequestHeaders(request -> connectionRef.get().getEndPoint().close())
                     .timeout(5, TimeUnit.SECONDS)
                     .send();
             Assert.fail();
@@ -106,7 +96,7 @@
             // Expected.
         }
 
-        DuplexConnectionPool connectionPool = connectionRef.get().getHttpDestination().getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)connectionRef.get().getHttpDestination().getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -134,25 +124,17 @@
         final CountDownLatch completeLatch = new CountDownLatch(1);
         DeferredContentProvider content = new DeferredContentProvider();
         client.newRequest("localhost", connector.getLocalPort())
-                .onRequestCommit(new Request.CommitListener()
+                .onRequestCommit(request ->
                 {
-                    @Override
-                    public void onCommit(Request request)
-                    {
-                        connectionRef.get().getEndPoint().close();
-                        commitLatch.countDown();
-                    }
+                    connectionRef.get().getEndPoint().close();
+                    commitLatch.countDown();
                 })
                 .content(content)
                 .idleTimeout(2, TimeUnit.SECONDS)
-                .send(new Response.CompleteListener()
+                .send(result ->
                 {
-                    @Override
-                    public void onComplete(Result result)
-                    {
-                        if (result.isFailed())
-                            completeLatch.countDown();
-                    }
+                    if (result.isFailed())
+                        completeLatch.countDown();
                 });
 
         Assert.assertTrue(commitLatch.await(5, TimeUnit.SECONDS));
@@ -170,7 +152,7 @@
         Assert.assertTrue(contentLatch.await(5, TimeUnit.SECONDS));
         Assert.assertTrue(completeLatch.await(5, TimeUnit.SECONDS));
 
-        DuplexConnectionPool connectionPool = connectionRef.get().getHttpDestination().getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)connectionRef.get().getHttpDestination().getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java
index 2155bde..96c2b10 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java
@@ -114,7 +114,7 @@
         Assert.assertEquals(200, response.getStatus());
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
         long start = System.nanoTime();
         HttpConnectionOverHTTP connection = null;
@@ -633,7 +633,8 @@
                 .onRequestBegin(request ->
                 {
                     HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-                    destination.getConnectionPool().getActiveConnections().peek().close();
+                    DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+                    connectionPool.getActiveConnections().iterator().next().close();
                 })
                 .send(new Response.Listener.Adapter()
                 {
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTimeoutTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTimeoutTest.java
index d65e99c..3457050 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTimeoutTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTimeoutTest.java
@@ -448,7 +448,7 @@
         start(new EmptyServerHandler());
 
         long timeout = 1000;
-        Request request = client.newRequest("badscheme://localhost:" + connector.getLocalPort());
+        Request request = client.newRequest("badscheme://localhost:badport");
 
         try
         {
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientUploadDuringServerShutdown.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientUploadDuringServerShutdown.java
index 941532a..3a73f39 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientUploadDuringServerShutdown.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientUploadDuringServerShutdown.java
@@ -31,8 +31,6 @@
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.Response;
-import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.http.HttpChannelOverHTTP;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
@@ -121,14 +119,7 @@
                     int length = 16 * 1024 * 1024 + random.nextInt(16 * 1024 * 1024);
                     client.newRequest("localhost", 8888)
                             .content(new BytesContentProvider(new byte[length]))
-                            .send(new Response.CompleteListener()
-                            {
-                                @Override
-                                public void onComplete(Result result)
-                                {
-                                    latch.countDown();
-                                }
-                            });
+                            .send(result -> latch.countDown());
                     long sleep = 1 + random.nextInt(10);
                     TimeUnit.MILLISECONDS.sleep(sleep);
                 }
@@ -244,35 +235,24 @@
         final CountDownLatch completeLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
                 .timeout(10, TimeUnit.SECONDS)
-                .onRequestBegin(new org.eclipse.jetty.client.api.Request.BeginListener()
+                .onRequestBegin(request ->
                 {
-                    @Override
-                    public void onBegin(org.eclipse.jetty.client.api.Request request)
+                    try
                     {
-                        try
-                        {
-                            beginLatch.countDown();
-                            completeLatch.await(5, TimeUnit.SECONDS);
-                        }
-                        catch (InterruptedException x)
-                        {
-                            x.printStackTrace();
-                        }
+                        beginLatch.countDown();
+                        completeLatch.await(5, TimeUnit.SECONDS);
+                    }
+                    catch (InterruptedException x)
+                    {
+                        x.printStackTrace();
                     }
                 })
-                .send(new Response.CompleteListener()
-                {
-                    @Override
-                    public void onComplete(Result result)
-                    {
-                        completeLatch.countDown();
-                    }
-                });
+                .send(result -> completeLatch.countDown());
 
         Assert.assertTrue(completeLatch.await(5, TimeUnit.SECONDS));
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination("http", "localhost", connector.getLocalPort());
-        DuplexConnectionPool pool = destination.getConnectionPool();
+        DuplexConnectionPool pool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, pool.getConnectionCount());
         Assert.assertEquals(0, pool.getIdleConnections().size());
         Assert.assertEquals(0, pool.getActiveConnections().size());
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java
index 2d1c25f..6949484 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Queue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -69,35 +70,24 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         final CountDownLatch headersLatch = new CountDownLatch(1);
         final CountDownLatch successLatch = new CountDownLatch(3);
         client.newRequest(host, port)
                 .scheme(scheme)
-                .onRequestSuccess(new Request.SuccessListener()
+                .onRequestSuccess(request -> successLatch.countDown())
+                .onResponseHeaders(response ->
                 {
-                    @Override
-                    public void onSuccess(Request request)
-                    {
-                        successLatch.countDown();
-                    }
-                })
-                .onResponseHeaders(new Response.HeadersListener()
-                {
-                    @Override
-                    public void onHeaders(Response response)
-                    {
-                        Assert.assertEquals(0, idleConnections.size());
-                        Assert.assertEquals(1, activeConnections.size());
-                        headersLatch.countDown();
-                    }
+                    Assert.assertEquals(0, idleConnections.size());
+                    Assert.assertEquals(1, activeConnections.size());
+                    headersLatch.countDown();
                 })
                 .send(new Response.Listener.Adapter()
                 {
@@ -130,12 +120,12 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         final CountDownLatch beginLatch = new CountDownLatch(1);
@@ -145,7 +135,7 @@
             @Override
             public void onBegin(Request request)
             {
-                activeConnections.peek().close();
+                activeConnections.iterator().next().close();
                 beginLatch.countDown();
             }
 
@@ -181,12 +171,12 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
         final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         final CountDownLatch successLatch = new CountDownLatch(3);
@@ -241,12 +231,12 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         final long delay = 1000;
@@ -314,12 +304,12 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         server.stop();
@@ -327,22 +317,11 @@
         final CountDownLatch failureLatch = new CountDownLatch(2);
         client.newRequest(host, port)
                 .scheme(scheme)
-                .onRequestFailure(new Request.FailureListener()
+                .onRequestFailure((request, failure) -> failureLatch.countDown())
+                .send(result ->
                 {
-                    @Override
-                    public void onFailure(Request request, Throwable failure)
-                    {
-                        failureLatch.countDown();
-                    }
-                })
-                .send(new Response.Listener.Adapter()
-                {
-                    @Override
-                    public void onComplete(Result result)
-                    {
-                        Assert.assertTrue(result.isFailed());
-                        failureLatch.countDown();
-                    }
+                    Assert.assertTrue(result.isFailed());
+                    failureLatch.countDown();
                 });
 
         Assert.assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
@@ -367,12 +346,12 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         final CountDownLatch latch = new CountDownLatch(1);
@@ -417,12 +396,12 @@
             String host = "localhost";
             int port = connector.getLocalPort();
             HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-            DuplexConnectionPool connectionPool = destination.getConnectionPool();
+            DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-            final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+            final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
             Assert.assertEquals(0, idleConnections.size());
 
-            final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+            final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
             Assert.assertEquals(0, activeConnections.size());
 
             Log.getLogger(HttpConnection.class).info("Expecting java.lang.IllegalStateException: HttpParser{s=CLOSED,...");
@@ -467,12 +446,12 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         ContentResponse response = client.newRequest(host, port)
@@ -499,25 +478,21 @@
         String host = "localhost";
         int port = connector.getLocalPort();
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, host, port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
 
-        final Queue<Connection> idleConnections = connectionPool.getIdleConnections();
+        final Collection<Connection> idleConnections = connectionPool.getIdleConnections();
         Assert.assertEquals(0, idleConnections.size());
 
-        final Queue<Connection> activeConnections = connectionPool.getActiveConnections();
+        final Collection<Connection> activeConnections = connectionPool.getActiveConnections();
         Assert.assertEquals(0, activeConnections.size());
 
         client.setStrictEventOrdering(false);
         ContentResponse response = client.newRequest(host, port)
                 .scheme(scheme)
-                .onResponseBegin(new Response.BeginListener()
+                .onResponseBegin(response1 ->
                 {
-                    @Override
-                    public void onBegin(Response response)
-                    {
-                        // Simulate a HTTP 1.0 response has been received.
-                        ((HttpResponse)response).version(HttpVersion.HTTP_1_0);
-                    }
+                    // Simulate a HTTP 1.0 response has been received.
+                    ((HttpResponse)response1).version(HttpVersion.HTTP_1_0);
                 })
                 .send();
 
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpRequestAbortTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpRequestAbortTest.java
index ef1f390..c7ee37b 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpRequestAbortTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpRequestAbortTest.java
@@ -25,12 +25,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.http.HttpDestinationOverHTTP;
 import org.eclipse.jetty.client.util.ByteBufferContentProvider;
@@ -88,7 +88,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -135,7 +135,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -182,7 +182,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -204,14 +204,10 @@
         {
             client.newRequest("localhost", connector.getLocalPort())
                     .scheme(scheme)
-                    .onRequestCommit(new Request.CommitListener()
+                    .onRequestCommit(request ->
                     {
-                        @Override
-                        public void onCommit(Request request)
-                        {
-                            aborted.set(request.abort(cause));
-                            latch.countDown();
-                        }
+                        aborted.set(request.abort(cause));
+                        latch.countDown();
                     })
                     .timeout(5, TimeUnit.SECONDS)
                     .send();
@@ -225,7 +221,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -260,14 +256,10 @@
         {
             client.newRequest("localhost", connector.getLocalPort())
                     .scheme(scheme)
-                    .onRequestCommit(new Request.CommitListener()
+                    .onRequestCommit(request ->
                     {
-                        @Override
-                        public void onCommit(Request request)
-                        {
-                            aborted.set(request.abort(cause));
-                            latch.countDown();
-                        }
+                        aborted.set(request.abort(cause));
+                        latch.countDown();
                     })
                     .content(new ByteBufferContentProvider(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
                     {
@@ -289,7 +281,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -315,14 +307,10 @@
         {
             client.newRequest("localhost", connector.getLocalPort())
                     .scheme(scheme)
-                    .onRequestContent(new Request.ContentListener()
+                    .onRequestContent((request, content) ->
                     {
-                        @Override
-                        public void onContent(Request request, ByteBuffer content)
-                        {
-                            aborted.set(request.abort(cause));
-                            latch.countDown();
-                        }
+                        aborted.set(request.abort(cause));
+                        latch.countDown();
                     })
                     .content(new ByteBufferContentProvider(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
                     {
@@ -344,7 +332,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -454,7 +442,7 @@
         }
 
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination(scheme, "localhost", connector.getLocalPort());
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnections().size());
         Assert.assertEquals(0, connectionPool.getIdleConnections().size());
@@ -486,15 +474,11 @@
         Request request = client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scheme)
                 .timeout(3 * delay, TimeUnit.MILLISECONDS);
-        request.send(new Response.CompleteListener()
+        request.send(result ->
         {
-            @Override
-            public void onComplete(Result result)
-            {
-                Assert.assertTrue(result.isFailed());
-                Assert.assertSame(cause, result.getFailure());
-                latch.countDown();
-            }
+            Assert.assertTrue(result.isFailed());
+            Assert.assertSame(cause, result.getFailure());
+            latch.countDown();
         });
 
         TimeUnit.MILLISECONDS.sleep(delay);
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/ServerConnectionCloseTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/ServerConnectionCloseTest.java
index 639984d..3263381 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/ServerConnectionCloseTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/ServerConnectionCloseTest.java
@@ -151,7 +151,7 @@
 
         // Connection should have been removed from pool.
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination("http", "localhost", port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getIdleConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnectionCount());
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/TLSServerConnectionCloseTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/TLSServerConnectionCloseTest.java
index 7000eb3..be3ab09 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/TLSServerConnectionCloseTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/TLSServerConnectionCloseTest.java
@@ -183,7 +183,7 @@
 
         // Connection should have been removed from pool.
         HttpDestinationOverHTTP destination = (HttpDestinationOverHTTP)client.getDestination("http", "localhost", port);
-        DuplexConnectionPool connectionPool = destination.getConnectionPool();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
         Assert.assertEquals(0, connectionPool.getConnectionCount());
         Assert.assertEquals(0, connectionPool.getIdleConnectionCount());
         Assert.assertEquals(0, connectionPool.getActiveConnectionCount());
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTPTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTPTest.java
index 4e9d095..6d4867c 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTPTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTPTest.java
@@ -24,6 +24,7 @@
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.client.AbstractHttpClientServerTest;
+import org.eclipse.jetty.client.ConnectionPool;
 import org.eclipse.jetty.client.DuplexConnectionPool;
 import org.eclipse.jetty.client.EmptyServerHandler;
 import org.eclipse.jetty.client.HttpClient;
@@ -31,9 +32,6 @@
 import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Destination;
-import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.client.api.Response;
-import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
@@ -59,11 +57,13 @@
     public void test_FirstAcquire_WithEmptyQueue() throws Exception
     {
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", connector.getLocalPort()));
-        Connection connection = destination.acquire();
+        destination.start();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        Connection connection = connectionPool.acquire();
         if (connection == null)
         {
             // There are no queued requests, so the newly created connection will be idle
-            connection = timedPoll(destination.getConnectionPool().getIdleConnections(), 5, TimeUnit.SECONDS);
+            connection = timedPoll(connectionPool.getIdleConnections(), 5, TimeUnit.SECONDS);
         }
         Assert.assertNotNull(connection);
     }
@@ -72,7 +72,9 @@
     public void test_SecondAcquire_AfterFirstAcquire_WithEmptyQueue_ReturnsSameConnection() throws Exception
     {
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", connector.getLocalPort()));
-        Connection connection1 = destination.acquire();
+        destination.start();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        Connection connection1 = connectionPool.acquire();
         if (connection1 == null)
         {
             // There are no queued requests, so the newly created connection will be idle
@@ -80,11 +82,11 @@
             while (connection1 == null && TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start) < 5)
             {
                 TimeUnit.MILLISECONDS.sleep(50);
-                connection1 = destination.getConnectionPool().getIdleConnections().peek();
+                connection1 = connectionPool.getIdleConnections().peek();
             }
             Assert.assertNotNull(connection1);
 
-            Connection connection2 = destination.acquire();
+            Connection connection2 = connectionPool.acquire();
             Assert.assertSame(connection1, connection2);
         }
     }
@@ -97,18 +99,18 @@
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", connector.getLocalPort()))
         {
             @Override
-            protected DuplexConnectionPool newConnectionPool(HttpClient client)
+            protected ConnectionPool newConnectionPool(HttpClient client)
             {
                 return new DuplexConnectionPool(this, client.getMaxConnectionsPerDestination(), this)
                 {
                     @Override
-                    protected void idleCreated(Connection connection)
+                    protected void onCreated(Connection connection)
                     {
                         try
                         {
                             idleLatch.countDown();
                             latch.await(5, TimeUnit.SECONDS);
-                            super.idleCreated(connection);
+                            super.onCreated(connection);
                         }
                         catch (InterruptedException x)
                         {
@@ -118,7 +120,9 @@
                 };
             }
         };
-        Connection connection1 = destination.acquire();
+        destination.start();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        Connection connection1 = connectionPool.acquire();
 
         // Make sure we entered idleCreated().
         Assert.assertTrue(idleLatch.await(5, TimeUnit.SECONDS));
@@ -128,13 +132,13 @@
         Assert.assertNull(connection1);
 
         // Second attempt also returns null because we delayed idleCreated() above.
-        Connection connection2 = destination.acquire();
+        Connection connection2 = connectionPool.acquire();
         Assert.assertNull(connection2);
 
         latch.countDown();
 
         // There must be 2 idle connections.
-        Queue<Connection> idleConnections = destination.getConnectionPool().getIdleConnections();
+        Queue<Connection> idleConnections = connectionPool.getIdleConnections();
         Connection connection = timedPoll(idleConnections, 5, TimeUnit.SECONDS);
         Assert.assertNotNull(connection);
         connection = timedPoll(idleConnections, 5, TimeUnit.SECONDS);
@@ -145,23 +149,25 @@
     public void test_Acquire_Process_Release_Acquire_ReturnsSameConnection() throws Exception
     {
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", connector.getLocalPort()));
-        HttpConnectionOverHTTP connection1 = destination.acquire();
+        destination.start();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        HttpConnectionOverHTTP connection1 = (HttpConnectionOverHTTP)connectionPool.acquire();
 
         long start = System.nanoTime();
         while (connection1 == null && TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start) < 5)
         {
             TimeUnit.MILLISECONDS.sleep(50);
-            connection1 = (HttpConnectionOverHTTP)destination.getConnectionPool().getIdleConnections().peek();
+            connection1 = (HttpConnectionOverHTTP)connectionPool.getIdleConnections().peek();
         }
         Assert.assertNotNull(connection1);
 
         // Acquire the connection to make it active
-        Assert.assertSame(connection1, destination.acquire());
+        Assert.assertSame(connection1, connectionPool.acquire());
 
         destination.process(connection1);
         destination.release(connection1);
 
-        Connection connection2 = destination.acquire();
+        Connection connection2 = connectionPool.acquire();
         Assert.assertSame(connection1, connection2);
     }
 
@@ -172,7 +178,9 @@
         client.setIdleTimeout(idleTimeout);
 
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", connector.getLocalPort()));
-        Connection connection1 = destination.acquire();
+        destination.start();
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        Connection connection1 = connectionPool.acquire();
         if (connection1 == null)
         {
             // There are no queued requests, so the newly created connection will be idle
@@ -180,13 +188,13 @@
             while (connection1 == null && TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start) < 5)
             {
                 TimeUnit.MILLISECONDS.sleep(50);
-                connection1 = destination.getConnectionPool().getIdleConnections().peek();
+                connection1 = connectionPool.getIdleConnections().peek();
             }
             Assert.assertNotNull(connection1);
 
             TimeUnit.MILLISECONDS.sleep(2 * idleTimeout);
 
-            connection1 = destination.getConnectionPool().getIdleConnections().poll();
+            connection1 = connectionPool.getIdleConnections().poll();
             Assert.assertNull(connection1);
         }
     }
@@ -210,35 +218,23 @@
         client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scheme)
                 .path("/one")
-                .onRequestQueued(new Request.QueuedListener()
+                .onRequestQueued(request ->
                 {
-                    @Override
-                    public void onQueued(Request request)
-                    {
-                        // This request exceeds the maximum queued, should fail
-                        client.newRequest("localhost", connector.getLocalPort())
-                                .scheme(scheme)
-                                .path("/two")
-                                .send(new Response.CompleteListener()
-                                {
-                                    @Override
-                                    public void onComplete(Result result)
-                                    {
-                                        Assert.assertTrue(result.isFailed());
-                                        Assert.assertThat(result.getRequestFailure(), Matchers.instanceOf(RejectedExecutionException.class));
-                                        failureLatch.countDown();
-                                    }
-                                });
-                    }
+                    // This request exceeds the maximum queued, should fail
+                    client.newRequest("localhost", connector.getLocalPort())
+                            .scheme(scheme)
+                            .path("/two")
+                            .send(result ->
+                            {
+                                Assert.assertTrue(result.isFailed());
+                                Assert.assertThat(result.getRequestFailure(), Matchers.instanceOf(RejectedExecutionException.class));
+                                failureLatch.countDown();
+                            });
                 })
-                .send(new Response.CompleteListener()
+                .send(result ->
                 {
-                    @Override
-                    public void onComplete(Result result)
-                    {
-                        if (result.isSucceeded())
-                            successLatch.countDown();
-                    }
+                    if (result.isSucceeded())
+                        successLatch.countDown();
                 });
 
         Assert.assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTPTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTPTest.java
index 051208d..4c32f87 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTPTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTPTest.java
@@ -30,6 +30,7 @@
 import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpResponseException;
 import org.eclipse.jetty.client.Origin;
+import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.util.FutureResponseListener;
 import org.eclipse.jetty.http.HttpFields;
@@ -60,6 +61,7 @@
         client = new HttpClient();
         client.start();
         destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         endPoint = new ByteArrayEndPoint();
         connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<>());
         endPoint.setConnection(connection);
@@ -235,7 +237,7 @@
             }
         };
         endPoint.setConnection(connection);
-
+        
         // Partial response to trigger the call to fillInterested().
         endPoint.addInput("" +
                 "HTTP/1.1 200 OK\r\n" +
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java
index b32f6db..301867f 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java
@@ -67,6 +67,7 @@
     {
         ByteArrayEndPoint endPoint = new ByteArrayEndPoint();
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         final CountDownLatch headersLatch = new CountDownLatch(1);
@@ -100,6 +101,7 @@
     {
         ByteArrayEndPoint endPoint = new ByteArrayEndPoint("", 16);
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         connection.send(request, null);
@@ -129,6 +131,7 @@
         // Shutdown output to trigger the exception on write
         endPoint.shutdownOutput();
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         final CountDownLatch failureLatch = new CountDownLatch(2);
@@ -158,6 +161,7 @@
     {
         ByteArrayEndPoint endPoint = new ByteArrayEndPoint("", 16);
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         final CountDownLatch failureLatch = new CountDownLatch(2);
@@ -193,6 +197,7 @@
     {
         ByteArrayEndPoint endPoint = new ByteArrayEndPoint();
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content = "abcdef";
@@ -227,6 +232,7 @@
     {
         ByteArrayEndPoint endPoint = new ByteArrayEndPoint();
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content1 = "0123456789";
@@ -262,6 +268,7 @@
     {
         ByteArrayEndPoint endPoint = new ByteArrayEndPoint();
         HttpDestinationOverHTTP destination = new HttpDestinationOverHTTP(client, new Origin("http", "localhost", 8080));
+        destination.start();
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content1 = "0123456789";
diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/ssl/SslBytesServerTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/ssl/SslBytesServerTest.java
index 62699c9..b5f1690 100644
--- a/jetty-client/src/test/java/org/eclipse/jetty/client/ssl/SslBytesServerTest.java
+++ b/jetty-client/src/test/java/org/eclipse/jetty/client/ssl/SslBytesServerTest.java
@@ -54,10 +54,10 @@
 
 import org.eclipse.jetty.client.ssl.SslBytesTest.TLSRecord.Type;
 import org.eclipse.jetty.http.HttpParser;
+import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
-import org.eclipse.jetty.io.SelectChannelEndPoint;
 import org.eclipse.jetty.io.ssl.SslConnection;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConnection;
@@ -173,9 +173,9 @@
         ServerConnector connector = new ServerConnector(server, null,null,null,1,1,sslFactory, httpFactory)
         {
             @Override
-            protected SelectChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+            protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
             {
-                SelectChannelEndPoint endp = super.newEndPoint(channel,selectSet,key);
+                ChannelEndPoint endp = super.newEndPoint(channel,selectSet,key);
                 serverEndPoint.set(endp);
                 return endp;
             }
@@ -367,11 +367,19 @@
         System.arraycopy(doneBytes, 0, chunk, recordBytes.length, doneBytes.length);
         System.arraycopy(closeRecordBytes, 0, chunk, recordBytes.length + doneBytes.length, closeRecordBytes.length);
         proxy.flushToServer(0, chunk);
+        
         // Close the raw socket
         proxy.flushToServer(null);
 
         // Expect the server to send a FIN as well
         record = proxy.readFromServer();
+        if (record!=null)
+        {
+            // Close alert snuck out  // TODO check if this is acceptable
+            Assert.assertEquals(Type.ALERT,record.getType());
+            record = proxy.readFromServer();
+        }
+        
         Assert.assertNull(record);
 
         // Check that we did not spin
diff --git a/jetty-client/src/test/resources/jetty-logging.properties b/jetty-client/src/test/resources/jetty-logging.properties
index 1c19e53..5f8794e 100644
--- a/jetty-client/src/test/resources/jetty-logging.properties
+++ b/jetty-client/src/test/resources/jetty-logging.properties
@@ -1,3 +1,4 @@
 org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
 #org.eclipse.jetty.LEVEL=DEBUG
 #org.eclipse.jetty.client.LEVEL=DEBUG
+#org.eclipse.jetty.io.ChannelEndPoint.LEVEL=DEBUG
\ No newline at end of file
diff --git a/jetty-continuation/pom.xml b/jetty-continuation/pom.xml
index 55860f3..edd16b1 100644
--- a/jetty-continuation/pom.xml
+++ b/jetty-continuation/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-continuation</artifactId>
diff --git a/jetty-deploy/pom.xml b/jetty-deploy/pom.xml
index 42c6675..9f94d9e 100644
--- a/jetty-deploy/pom.xml
+++ b/jetty-deploy/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-deploy</artifactId>
diff --git a/jetty-deploy/src/main/config/modules/deploy.mod b/jetty-deploy/src/main/config/modules/deploy.mod
index f567a20..794868b 100644
--- a/jetty-deploy/src/main/config/modules/deploy.mod
+++ b/jetty-deploy/src/main/config/modules/deploy.mod
@@ -1,6 +1,5 @@
-#
-# Deploy Feature
-#
+[description]
+Enables webapplication deployment from the webapps directory.
 
 [depend]
 webapp
diff --git a/jetty-distribution/pom.xml b/jetty-distribution/pom.xml
index 49770cc..f68e729 100644
--- a/jetty-distribution/pom.xml
+++ b/jetty-distribution/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <artifactId>jetty-distribution</artifactId>
   <name>Jetty :: Distribution Assemblies</name>
@@ -711,6 +711,11 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-unixsocket</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.eclipse.jetty.fcgi</groupId>
       <artifactId>fcgi-server</artifactId>
       <version>${project.version}</version>
diff --git a/jetty-distribution/src/main/resources/modules/hawtio.mod b/jetty-distribution/src/main/resources/modules/hawtio.mod
index f6d0d9d..fcc34d1 100644
--- a/jetty-distribution/src/main/resources/modules/hawtio.mod
+++ b/jetty-distribution/src/main/resources/modules/hawtio.mod
@@ -1,6 +1,5 @@
-#
-# Hawtio x module
-#
+[description]
+Deploys the Hawtio console as a webapplication.
 
 [depend]
 stats
diff --git a/jetty-distribution/src/main/resources/modules/jamon.mod b/jetty-distribution/src/main/resources/modules/jamon.mod
index 2d1f144..77cc3d1 100644
--- a/jetty-distribution/src/main/resources/modules/jamon.mod
+++ b/jetty-distribution/src/main/resources/modules/jamon.mod
@@ -1,6 +1,5 @@
-#
-# JAMon Jetty module
-#
+[description]
+Deploys the JAMon webapplication
 
 [depend]
 stats
diff --git a/jetty-distribution/src/main/resources/modules/jminix.mod b/jetty-distribution/src/main/resources/modules/jminix.mod
index 05788f0..81a75c7 100644
--- a/jetty-distribution/src/main/resources/modules/jminix.mod
+++ b/jetty-distribution/src/main/resources/modules/jminix.mod
@@ -1,6 +1,5 @@
-#
-# JaMON Jetty module
-#
+[description]
+Deploys the Jminix JMX Console within the server
 
 [depend]
 stats
diff --git a/jetty-distribution/src/main/resources/modules/jolokia.mod b/jetty-distribution/src/main/resources/modules/jolokia.mod
index da8ac8f..efe8a59 100644
--- a/jetty-distribution/src/main/resources/modules/jolokia.mod
+++ b/jetty-distribution/src/main/resources/modules/jolokia.mod
@@ -1,6 +1,5 @@
-#
-# Jolokia Jetty module
-#
+[description]
+Deploys the Jolokia console as a web application.
 
 [depend]
 stats
diff --git a/jetty-distribution/src/main/resources/modules/jsp.mod b/jetty-distribution/src/main/resources/modules/jsp.mod
index a16cc93..2bc7ba8 100644
--- a/jetty-distribution/src/main/resources/modules/jsp.mod
+++ b/jetty-distribution/src/main/resources/modules/jsp.mod
@@ -1,6 +1,5 @@
-#
-# Jetty JSP Module
-#
+[description]
+Enables JSP for all webapplications deployed on the server.
 
 [depend]
 servlet
diff --git a/jetty-distribution/src/main/resources/modules/jstl.mod b/jetty-distribution/src/main/resources/modules/jstl.mod
index efc310a..dedb2c0 100644
--- a/jetty-distribution/src/main/resources/modules/jstl.mod
+++ b/jetty-distribution/src/main/resources/modules/jstl.mod
@@ -1,6 +1,5 @@
-#
-# Jetty JSTL Module
-#
+[description]
+Enables JSTL for all webapplications deployed on the server
 
 [depend]
 jsp
diff --git a/jetty-distribution/src/main/resources/modules/setuid.mod b/jetty-distribution/src/main/resources/modules/setuid.mod
index 41ef757..c1174cc 100644
--- a/jetty-distribution/src/main/resources/modules/setuid.mod
+++ b/jetty-distribution/src/main/resources/modules/setuid.mod
@@ -1,6 +1,7 @@
-#
-# Set UID Feature
-#
+[description]
+Enables the unix setUID configuration so that the server
+may be started as root to open privileged ports/files before
+changing to a restricted user (eg jetty).
 
 [depend]
 server
diff --git a/jetty-fcgi/fcgi-client/pom.xml b/jetty-fcgi/fcgi-client/pom.xml
index 79b4c0b..8cf9e9a 100644
--- a/jetty-fcgi/fcgi-client/pom.xml
+++ b/jetty-fcgi/fcgi-client/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.eclipse.jetty.fcgi</groupId>
         <artifactId>fcgi-parent</artifactId>
-        <version>9.3.8-SNAPSHOT</version>
+        <version>9.4.0-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpDestinationOverFCGI.java b/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpDestinationOverFCGI.java
index a6ac2cc..3b4297b 100644
--- a/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpDestinationOverFCGI.java
+++ b/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpDestinationOverFCGI.java
@@ -23,8 +23,9 @@
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.PoolingHttpDestination;
 import org.eclipse.jetty.client.SendFailure;
+import org.eclipse.jetty.client.api.Connection;
 
-public class HttpDestinationOverFCGI extends PoolingHttpDestination<HttpConnectionOverFCGI>
+public class HttpDestinationOverFCGI extends PoolingHttpDestination
 {
     public HttpDestinationOverFCGI(HttpClient client, Origin origin)
     {
@@ -32,8 +33,8 @@
     }
 
     @Override
-    protected SendFailure send(HttpConnectionOverFCGI connection, HttpExchange exchange)
+    protected SendFailure send(Connection connection, HttpExchange exchange)
     {
-        return connection.send(exchange);
+        return ((HttpConnectionOverFCGI)connection).send(exchange);
     }
 }
diff --git a/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/MultiplexHttpDestinationOverFCGI.java b/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/MultiplexHttpDestinationOverFCGI.java
index bf7f403..4f16e71 100644
--- a/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/MultiplexHttpDestinationOverFCGI.java
+++ b/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/MultiplexHttpDestinationOverFCGI.java
@@ -23,8 +23,9 @@
 import org.eclipse.jetty.client.MultiplexHttpDestination;
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.SendFailure;
+import org.eclipse.jetty.client.api.Connection;
 
-public class MultiplexHttpDestinationOverFCGI extends MultiplexHttpDestination<HttpConnectionOverFCGI>
+public class MultiplexHttpDestinationOverFCGI extends MultiplexHttpDestination
 {
     public MultiplexHttpDestinationOverFCGI(HttpClient client, Origin origin)
     {
@@ -32,8 +33,8 @@
     }
 
     @Override
-    protected SendFailure send(HttpConnectionOverFCGI connection, HttpExchange exchange)
+    protected SendFailure send(Connection connection, HttpExchange exchange)
     {
-        return connection.send(exchange);
+        return ((HttpConnectionOverFCGI)connection).send(exchange);
     }
 }
diff --git a/jetty-fcgi/fcgi-server/pom.xml b/jetty-fcgi/fcgi-server/pom.xml
index b3d1b95..271ef70 100644
--- a/jetty-fcgi/fcgi-server/pom.xml
+++ b/jetty-fcgi/fcgi-server/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty.fcgi</groupId>
     <artifactId>fcgi-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-fcgi/fcgi-server/src/main/config/modules/fcgi.mod b/jetty-fcgi/fcgi-server/src/main/config/modules/fcgi.mod
index 14152d5..6a4beaf 100644
--- a/jetty-fcgi/fcgi-server/src/main/config/modules/fcgi.mod
+++ b/jetty-fcgi/fcgi-server/src/main/config/modules/fcgi.mod
@@ -1,6 +1,5 @@
-#
-# FastCGI Module
-#
+[description]
+Adds the FastCGI implementation to the classpath.
 
 [depend]
 servlet
diff --git a/jetty-fcgi/pom.xml b/jetty-fcgi/pom.xml
index e8b1511..d3f278c 100644
--- a/jetty-fcgi/pom.xml
+++ b/jetty-fcgi/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-project</artifactId>
-        <version>9.3.8-SNAPSHOT</version>
+        <version>9.4.0-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-gcloud/jetty-gcloud-session-manager/pom.xml b/jetty-gcloud/jetty-gcloud-session-manager/pom.xml
index 120c85c..3953a5a 100644
--- a/jetty-gcloud/jetty-gcloud-session-manager/pom.xml
+++ b/jetty-gcloud/jetty-gcloud-session-manager/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty.gcloud</groupId>
     <artifactId>gcloud-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/etc/jetty-gcloud-sessions.xml b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/etc/jetty-gcloud-sessions.xml
index 72f9da6..b1b9a84 100644
--- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/etc/jetty-gcloud-sessions.xml
+++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/etc/jetty-gcloud-sessions.xml
@@ -7,11 +7,12 @@
   <!-- GCloud configuration.                                                                          -->
   <!-- Note: passwords can use jetty obfuscation.  See                                                -->
   <!-- https://www.eclipse.org/jetty/documentation/current/configuring-security-secure-passwords.html -->
+  <!-- See your start.ini or gcloud-sessions.ini file for more configuration information.             -->
   <!-- ============================================================================================== -->
   <New id="gconf" class="org.eclipse.jetty.gcloud.session.GCloudConfiguration">
-    <!-- To contact remote gclouddatastore set the following properties in start.ini -->
     <!-- Either set jetty.gcloudSession.projectId or use system property/env var DATASTORE_DATASET-->
     <Set name="projectId"><Property name="jetty.gcloudSession.projectId"/></Set>
+    <!-- To contact remote gclouddatastore set the following properties in start.ini -->
     <Set name="p12File"><Property name="jetty.gcloudSession.p12File"/></Set>
     <Set name="serviceAccount"><Property name="jetty.gcloudSession.serviceAccount"/></Set>
     <Set name="password"><Property name="jetty.gcloudSession.password"/></Set>
diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/modules/gcloud-sessions.mod b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/modules/gcloud-sessions.mod
index 14671f7..6bd5e67 100644
--- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/modules/gcloud-sessions.mod
+++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config/modules/gcloud-sessions.mod
@@ -1,6 +1,5 @@
-#
-# Jetty GCloudDatastore Session Manager module
-#
+[description]
+Enables the GCloudDatastore Session Mananger module.
 
 [depend]
 annotations
diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java
new file mode 100644
index 0000000..56371fa
--- /dev/null
+++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java
@@ -0,0 +1,389 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+
+package org.eclipse.jetty.gcloud.session;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.server.session.AbstractSessionDataStore;
+import org.eclipse.jetty.server.session.SessionContext;
+import org.eclipse.jetty.server.session.SessionData;
+import org.eclipse.jetty.util.ClassLoadingObjectInputStream;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+
+import com.google.gcloud.datastore.Blob;
+import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreFactory;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+import com.google.gcloud.datastore.ProjectionEntity;
+import com.google.gcloud.datastore.Query;
+import com.google.gcloud.datastore.QueryResults;
+import com.google.gcloud.datastore.StructuredQuery;
+import com.google.gcloud.datastore.StructuredQuery.CompositeFilter;
+import com.google.gcloud.datastore.StructuredQuery.KeyQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.Projection;
+import com.google.gcloud.datastore.StructuredQuery.ProjectionEntityQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
+
+/**
+ * GCloudSessionDataStore
+ *
+ *
+ */
+public class GCloudSessionDataStore extends AbstractSessionDataStore
+{
+    private  final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
+
+    public  static final String ID = "id";
+    public  static final String CONTEXTPATH = "contextPath";
+    public  static final String VHOST = "vhost";
+    public  static final String ACCESSED = "accessed";
+    public  static final String LASTACCESSED = "lastAccessed";
+    public  static final String CREATETIME = "createTime";
+    public  static final  String COOKIESETTIME = "cookieSetTime";
+    public  static final String LASTNODE = "lastNode";
+    public  static final String EXPIRY = "expiry";
+    public  static final  String MAXINACTIVE = "maxInactive";
+    public  static final  String ATTRIBUTES = "attributes";
+
+    public static final String KIND = "GCloudSession";
+    public static final int DEFAULT_MAX_QUERY_RESULTS = 100;
+
+    private GCloudConfiguration _config;
+    private Datastore _datastore;
+    private KeyFactory _keyFactory;
+    private int _maxResults = DEFAULT_MAX_QUERY_RESULTS;
+    
+
+    
+    
+    
+    
+    
+    
+    
+    
+
+    /** 
+     * @see org.eclipse.jetty.server.session.AbstractSessionDataStore#doStart()
+     */
+    @Override
+    protected void doStart() throws Exception
+    {
+        
+        if (_config == null)
+            throw new IllegalStateException("No DataStore configuration");
+        
+        _datastore = DatastoreFactory.instance().get(_config.getDatastoreOptions());
+        _keyFactory = _datastore.newKeyFactory().kind(KIND);
+        
+        super.doStart();
+    }
+
+    /** 
+     * @see org.eclipse.jetty.util.component.AbstractLifeCycle#doStop()
+     */
+    @Override
+    protected void doStop() throws Exception
+    {
+        super.doStop();
+    }
+    
+    
+    /**
+     * @param cfg
+     */
+    public void setGCloudConfiguration (GCloudConfiguration cfg)
+    {
+        _config = cfg;
+    }
+    
+    /**
+     * @return
+     */
+    public GCloudConfiguration getGCloudConfiguration ()
+    {
+        return _config;
+    }
+
+    
+    /**
+     * @return
+     */
+    public int getMaxResults()
+    {
+        return _maxResults;
+    }
+
+
+    /**
+     * @param maxResults
+     */
+    public void setMaxResults(int maxResults)
+    {
+        if (_maxResults <= 0)
+            _maxResults = DEFAULT_MAX_QUERY_RESULTS;
+        else
+            _maxResults = maxResults;
+    }
+    
+    
+    
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#load(java.lang.String)
+     */
+    @Override
+    public SessionData load(String id) throws Exception
+    {
+        if (LOG.isDebugEnabled()) LOG.debug("Loading session {} from DataStore", id);
+
+        Entity entity = _datastore.get(makeKey(id, _context));
+        if (entity == null)
+        {
+            if (LOG.isDebugEnabled()) LOG.debug("No session {} in DataStore ", id);
+            return null;
+        }
+        else
+        {
+            SessionData data = sessionFromEntity(entity);
+            return data;
+        }
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#delete(java.lang.String)
+     */
+    @Override
+    public boolean delete(String id) throws Exception
+    {
+        if (LOG.isDebugEnabled()) LOG.debug("Removing session {} from DataStore", id);
+        _datastore.delete(makeKey(id, _context));
+        return true;
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#getExpired(java.util.Set)
+     */
+    @Override
+    public Set<String> getExpired(Set<String> candidates)
+    {
+       long now = System.currentTimeMillis();
+       Set<String> expired = new HashSet<String>();
+        
+       //get up to maxResult number of sessions that have expired
+       ProjectionEntityQueryBuilder pbuilder = Query.projectionEntityQueryBuilder();
+       pbuilder.addProjection(Projection.property(ID));
+       pbuilder.filter(CompositeFilter.and(PropertyFilter.gt(EXPIRY, 0), PropertyFilter.le(EXPIRY, now)));
+       pbuilder.limit(_maxResults);
+       pbuilder.kind(KIND);
+       StructuredQuery<ProjectionEntity> pquery = pbuilder.build();
+       QueryResults<ProjectionEntity> presults = _datastore.run(pquery);
+       
+       while (presults.hasNext())
+       {
+           ProjectionEntity pe = presults.next();
+           String id = pe.getString(ID);
+           expired.add(id);
+       }
+      
+       //reconcile against ids that the SessionStore thinks are expired
+       Set<String> tmp = new HashSet<String>(candidates);
+       tmp.removeAll(expired);       
+       if (!tmp.isEmpty())
+       {
+           //sessionstore thinks these are expired, but they are either no
+           //longer in the db or not expired in the db, or we exceeded the
+           //number of records retrieved by the expiry query, so check them
+           //individually
+           for (String s:tmp)
+           {
+               try
+               {
+                   KeyQueryBuilder kbuilder = Query.keyQueryBuilder();
+                   kbuilder.filter(PropertyFilter.eq(ID, s));
+                   kbuilder.kind(KIND);
+                   StructuredQuery<Key> kq = kbuilder.build();
+                   QueryResults<Key> kresults = _datastore.run(kq);
+                   if (!kresults.hasNext())
+                       expired.add(s); //not in db, can be expired
+               }
+               catch (Exception e)
+               {
+                   LOG.warn(e);
+               }
+           }
+       }
+       
+       return expired;
+        
+    }
+
+ 
+    
+    /** 
+     * @see org.eclipse.jetty.server.session.AbstractSessionDataStore#doStore(java.lang.String, org.eclipse.jetty.server.session.SessionData, boolean)
+     */
+    @Override
+    public void doStore(String id, SessionData data, boolean isNew) throws Exception
+    {
+        if (LOG.isDebugEnabled()) LOG.debug("Writing session {} to DataStore", data.getId());
+    
+        Entity entity = entityFromSession(data, makeKey(id, _context));
+        _datastore.put(entity);
+    }
+
+    /**
+     * Make a unique key for this session.
+     * As the same session id can be used across multiple contexts, to
+     * make it unique, the key must be composed of:
+     * <ol>
+     * <li>the id</li>
+     * <li>the context path</li>
+     * <li>the virtual hosts</li>
+     * </ol>
+     * 
+     *
+     * @param session
+     * @return
+     */
+    private Key makeKey (String id, SessionContext context)
+    {
+        String key = context.getCanonicalContextPath()+"_"+context.getVhost()+"_"+id;
+        return _keyFactory.newKey(key);
+    }
+    
+    
+    /**
+     * Generate a gcloud datastore Entity from SessionData
+     * @param session
+     * @param key
+     * @return
+     * @throws Exception
+     */
+    private Entity entityFromSession (SessionData session, Key key) throws Exception
+    {
+        if (session == null)
+            return null;
+        
+        Entity entity = null;
+        
+        //serialize the attribute map
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos = new ObjectOutputStream(baos);
+        oos.writeObject(session.getAllAttributes());
+        oos.flush();
+        
+        //turn a session into an entity
+        entity = Entity.builder(key)
+                .set(ID, session.getId())
+                .set(CONTEXTPATH, session.getContextPath())
+                .set(VHOST, session.getVhost())
+                .set(ACCESSED, session.getAccessed())
+                .set(LASTACCESSED, session.getLastAccessed())
+                .set(CREATETIME, session.getCreated())
+                .set(COOKIESETTIME, session.getCookieSet())
+                .set(LASTNODE,session.getLastNode())
+                .set(EXPIRY, session.getExpiry())
+                .set(MAXINACTIVE, session.getMaxInactiveMs())
+                .set(ATTRIBUTES, Blob.copyFrom(baos.toByteArray())).build();
+                 
+        return entity;
+    }
+    
+    /**
+     * Generate SessionData from an Entity retrieved from gcloud datastore.
+     * @param entity
+     * @return
+     * @throws Exception
+     */
+    private SessionData sessionFromEntity (Entity entity) throws Exception
+    {
+        if (entity == null)
+            return null;
+
+        final AtomicReference<SessionData> reference = new AtomicReference<SessionData>();
+        final AtomicReference<Exception> exception = new AtomicReference<Exception>();
+        Runnable load = new Runnable()
+        {
+            public void run ()
+            {
+                try
+                {
+                    //turn an Entity into a Session
+                    String id = entity.getString(ID);
+                    String contextPath = entity.getString(CONTEXTPATH);
+                    String vhost = entity.getString(VHOST);
+                    long accessed = entity.getLong(ACCESSED);
+                    long lastAccessed = entity.getLong(LASTACCESSED);
+                    long createTime = entity.getLong(CREATETIME);
+                    long cookieSet = entity.getLong(COOKIESETTIME);
+                    String lastNode = entity.getString(LASTNODE);
+                    long expiry = entity.getLong(EXPIRY);
+                    long maxInactive = entity.getLong(MAXINACTIVE);
+                    Blob blob = (Blob) entity.getBlob(ATTRIBUTES);
+
+                    SessionData session = newSessionData (id, createTime, accessed, lastAccessed, maxInactive);
+                    session.setLastNode(lastNode);
+                    session.setContextPath(contextPath);
+                    session.setVhost(vhost);
+                    session.setCookieSet(cookieSet);
+                    session.setLastNode(lastNode);
+                    session.setExpiry(expiry);
+                    try (ClassLoadingObjectInputStream ois = new ClassLoadingObjectInputStream(blob.asInputStream()))
+                    {
+                        Object o = ois.readObject();
+                        session.putAllAttributes((Map<String,Object>)o);
+                    }
+                    reference.set(session);
+                }
+                catch (Exception e)
+                {
+                    exception.set(e);
+                }
+            }
+        };
+        
+        //ensure this runs in the context classloader
+       _context.run(load);
+    
+        if (exception.get() != null)
+            throw exception.get();
+        
+        return reference.get();
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#isPassivating()
+     */
+    @Override
+    public boolean isPassivating()
+    {
+       return true;
+    }
+    
+    
+}
diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionIdManager.java b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionIdManager.java
index 7c2a81e..2f8da7a 100644
--- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionIdManager.java
+++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionIdManager.java
@@ -20,16 +20,9 @@
 
 import java.util.Random;
 
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpSession;
-
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.SessionManager;
-import org.eclipse.jetty.server.handler.ContextHandler;
-import org.eclipse.jetty.server.session.AbstractSession;
 import org.eclipse.jetty.server.session.AbstractSessionIdManager;
-import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.server.session.Session;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
@@ -52,7 +45,6 @@
     private  final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
     public static final int DEFAULT_IDLE_EXPIRY_MULTIPLE = 2;
     public static final String KIND = "GCloudSessionId";
-    private Server _server;
     private Datastore _datastore;
     private KeyFactory _keyFactory;
     private GCloudConfiguration _config;
@@ -65,8 +57,7 @@
      */
     public GCloudSessionIdManager(Server server)
     {
-        super();
-        _server = server;
+        super(server);
     }
 
     /**
@@ -75,8 +66,7 @@
      */
     public GCloudSessionIdManager(Server server, Random random)
     {
-       super(random);
-       _server = server;
+       super(server,random);
     }
 
 
@@ -111,56 +101,7 @@
     }
 
     
-   
-
-    
-    /** 
-     * Check to see if the given session id is being
-     * used by a session in any context.
-     * 
-     * This method will consult the cluster.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#idInUse(java.lang.String)
-     */
-    @Override
-    public boolean idInUse(String id)
-    {
-        if (id == null)
-            return false;
-        
-        String clusterId = getClusterId(id);
-        
-        //ask the cluster - this should also tickle the idle expiration timer on the sessionid entry
-        //keeping it valid
-        try
-        {
-            return exists(clusterId);
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Problem checking inUse for id="+clusterId, e);
-            return false;
-        }
-        
-    }
-
-    /** 
-     * Remember a new in-use session id.
-     * 
-     * This will save the in-use session id to the cluster.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#addSession(javax.servlet.http.HttpSession)
-     */
-    @Override
-    public void addSession(HttpSession session)
-    {
-        if (session == null)
-            return;
-
-        //insert into the store
-        insert (((AbstractSession)session).getClusterId());
-    }
-
+ 
   
 
     
@@ -174,91 +115,8 @@
         _config = config;
     }
 
-    
-    
-    /** 
-     * Remove a session id from the list of in-use ids.
-     * 
-     * This will remvove the corresponding session id from the cluster.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#removeSession(javax.servlet.http.HttpSession)
-     */
-    @Override
-    public void removeSession(HttpSession session)
-    {
-        if (session == null)
-            return;
+   
 
-        //delete from the cache
-        delete (((AbstractSession)session).getClusterId());
-    }
-
-    /** 
-     * Remove a session id. This compels all other contexts who have a session
-     * with the same id to also remove it.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#invalidateAll(java.lang.String)
-     */
-    @Override
-    public void invalidateAll(String id)
-    {
-        //delete the session id from list of in-use sessions
-        delete (id);
-
-
-        //tell all contexts that may have a session object with this id to
-        //get rid of them
-        Handler[] contexts = _server.getChildHandlersByClass(ContextHandler.class);
-        for (int i=0; contexts!=null && i<contexts.length; i++)
-        {
-            SessionHandler sessionHandler = ((ContextHandler)contexts[i]).getChildHandlerByClass(SessionHandler.class);
-            if (sessionHandler != null)
-            {
-                SessionManager manager = sessionHandler.getSessionManager();
-
-                if (manager != null && manager instanceof GCloudSessionManager)
-                {
-                    ((GCloudSessionManager)manager).invalidateSession(id);
-                }
-            }
-        }
-
-    }
-
-    /** 
-     * Change a session id. 
-     * 
-     * Typically this occurs when a previously existing session has passed through authentication.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionIdManager#renewSessionId(java.lang.String, java.lang.String, javax.servlet.http.HttpServletRequest)
-     */
-    @Override
-    public void renewSessionId(String oldClusterId, String oldNodeId, HttpServletRequest request)
-    {
-        //generate a new id
-        String newClusterId = newSessionId(request.hashCode());
-
-        delete(oldClusterId);
-        insert(newClusterId);
-
-
-        //tell all contexts to update the id 
-        Handler[] contexts = _server.getChildHandlersByClass(ContextHandler.class);
-        for (int i=0; contexts!=null && i<contexts.length; i++)
-        {
-            SessionHandler sessionHandler = ((ContextHandler)contexts[i]).getChildHandlerByClass(SessionHandler.class);
-            if (sessionHandler != null) 
-            {
-                SessionManager manager = sessionHandler.getSessionManager();
-
-                if (manager != null && manager instanceof GCloudSessionManager)
-                {
-                    ((GCloudSessionManager)manager).renewSessionId(oldClusterId, oldNodeId, newClusterId, getNodeId(newClusterId, request));
-                }
-            }
-        }
-
-    }
 
     
     
@@ -300,12 +158,13 @@
      * 
      * @param id
      */
-    protected void delete (String id)
+    protected boolean delete (String id)
     {
         if (_datastore == null)
             throw new IllegalStateException ("No DataStore");
         
         _datastore.delete(makeKey(id));
+        return true; //gcloud does not distinguish between first and subsequent removes
     }
     
     
@@ -320,4 +179,53 @@
     {
         return _keyFactory.newKey(id);
     }
+
+    /** 
+     * @see org.eclipse.jetty.server.SessionIdManager#isIdInUse(java.lang.String)
+     */
+    @Override
+    public boolean isIdInUse(String id)
+    {
+        if (id == null)
+            return false;
+        
+        
+        //ask the cluster - this should also tickle the idle expiration timer on the sessionid entry
+        //keeping it valid
+        try
+        {
+            return exists(id);
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Problem checking inUse for id="+id, e);
+            return false;
+        }
+        
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.SessionIdManager#useId(org.eclipse.jetty.server.session.Session)
+     */
+    @Override
+    public void useId(Session session)
+    {
+        if (session == null)
+            return;
+
+        //insert into the store
+        insert (session.getId());
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.SessionIdManager#removeId(java.lang.String)
+     */
+    @Override
+    public boolean removeId(String id)
+    {
+       if (id == null)
+           return false;
+       
+       return delete(id);
+    }
 }
diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionManager.java b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionManager.java
index 640aec5..dd50b94 100644
--- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionManager.java
+++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionManager.java
@@ -18,43 +18,16 @@
 
 package org.eclipse.jetty.gcloud.session;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReentrantLock;
 
 import javax.servlet.http.HttpServletRequest;
 
-import org.eclipse.jetty.server.handler.ContextHandler;
-import org.eclipse.jetty.server.handler.ContextHandler.Context;
-import org.eclipse.jetty.server.session.AbstractSession;
-import org.eclipse.jetty.server.session.AbstractSessionManager;
-import org.eclipse.jetty.server.session.MemSession;
-import org.eclipse.jetty.util.ClassLoadingObjectInputStream;
+import org.eclipse.jetty.server.session.AbstractSessionStore;
+import org.eclipse.jetty.server.session.MemorySessionStore;
+import org.eclipse.jetty.server.session.SessionManager;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-import org.eclipse.jetty.util.thread.Scheduler;
-
-import com.google.gcloud.datastore.Blob;
-import com.google.gcloud.datastore.Datastore;
-import com.google.gcloud.datastore.DatastoreFactory;
-import com.google.gcloud.datastore.Entity;
-import com.google.gcloud.datastore.GqlQuery;
-import com.google.gcloud.datastore.Key;
-import com.google.gcloud.datastore.KeyFactory;
-import com.google.gcloud.datastore.Query;
-import com.google.gcloud.datastore.Query.ResultType;
-import com.google.gcloud.datastore.QueryResults;
 
 
 
@@ -63,360 +36,50 @@
  * 
  * 
  */
-public class GCloudSessionManager extends AbstractSessionManager
+public class GCloudSessionManager extends SessionManager
 {
     private  final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
-    
-    
-    public static final String KIND = "GCloudSession";
-    public static final int DEFAULT_MAX_QUERY_RESULTS = 100;
-    public static final long DEFAULT_SCAVENGE_SEC = 600; 
-    
-    /**
-     * Sessions known to this node held in memory
-     */
-    private ConcurrentHashMap<String, GCloudSessionManager.Session> _sessions;
+
+  
+ 
+
+
+    private GCloudSessionDataStore _sessionDataStore = null;
+
+
 
     
-    /**
-     * The length of time a session can be in memory without being checked against
-     * the cluster. A value of 0 indicates that the session is never checked against
-     * the cluster - the current node is considered to be the master for the session.
-     *
-     */
-    private long _staleIntervalSec = 0;
+/*
     
-    protected Scheduler.Task _task; //scavenge task
-    protected Scheduler _scheduler;
-    protected Scavenger _scavenger;
-    protected long _scavengeIntervalMs = 1000L * DEFAULT_SCAVENGE_SEC; //10mins
-    protected boolean _ownScheduler;
-    
-    private Datastore _datastore;
-    private KeyFactory _keyFactory;
-
-
-    private SessionEntityConverter _converter;
-
-
-    private int _maxResults = DEFAULT_MAX_QUERY_RESULTS;
-
-
-    /**
-     * Scavenger
-     *
-     */
-    protected class Scavenger implements Runnable
-    {
-
-        @Override
-        public void run()
-        {
-           try
-           {
-               scavenge();
-           }
-           finally
-           {
-               if (_scheduler != null && _scheduler.isRunning())
-                   _task = _scheduler.schedule(this, _scavengeIntervalMs, TimeUnit.MILLISECONDS);
-           }
-        }
-    }
-
-    /**
-     * SessionEntityConverter
-     *
-     *
-     */
-    public class SessionEntityConverter
-    {
-        public  final String CLUSTERID = "clusterId";
-        public  final String CONTEXTPATH = "contextPath";
-        public  final String VHOST = "vhost";
-        public  final String ACCESSED = "accessed";
-        public  final String LASTACCESSED = "lastAccessed";
-        public  final String CREATETIME = "createTime";
-        public  final  String COOKIESETTIME = "cookieSetTime";
-        public  final String LASTNODE = "lastNode";
-        public  final String EXPIRY = "expiry";
-        public  final  String MAXINACTIVE = "maxInactive";
-        public  final  String ATTRIBUTES = "attributes";
-
-      
-        
-        public Entity entityFromSession (Session session, Key key) throws Exception
-        {
-            if (session == null)
-                return null;
-            
-            Entity entity = null;
-            
-            //serialize the attribute map
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();
-            ObjectOutputStream oos = new ObjectOutputStream(baos);
-            oos.writeObject(session.getAttributeMap());
-            oos.flush();
-            
-            //turn a session into an entity
-            entity = Entity.builder(key)
-                    .set(CLUSTERID, session.getId())
-                    .set(CONTEXTPATH, session.getContextPath())
-                    .set(VHOST, session.getVHost())
-                    .set(ACCESSED, session.getAccessed())
-                    .set(LASTACCESSED, session.getLastAccessedTime())
-                    .set(CREATETIME, session.getCreationTime())
-                    .set(COOKIESETTIME, session.getCookieSetTime())
-                    .set(LASTNODE,session.getLastNode())
-                    .set(EXPIRY, session.getExpiry())
-                    .set(MAXINACTIVE, session.getMaxInactiveInterval())
-                    .set(ATTRIBUTES, Blob.copyFrom(baos.toByteArray())).build();
-                     
-            return entity;
-        }
-        
-        public Session sessionFromEntity (Entity entity) throws Exception
-        {
-            if (entity == null)
-                return null;
-
-            final AtomicReference<Session> reference = new AtomicReference<Session>();
-            final AtomicReference<Exception> exception = new AtomicReference<Exception>();
-            Runnable load = new Runnable()
-            {
-                public void run ()
-                {
-                    try
-                    {
-                        //turn an entity into a Session
-                        String clusterId = entity.getString(CLUSTERID);
-                        String contextPath = entity.getString(CONTEXTPATH);
-                        String vhost = entity.getString(VHOST);
-                        long accessed = entity.getLong(ACCESSED);
-                        long lastAccessed = entity.getLong(LASTACCESSED);
-                        long createTime = entity.getLong(CREATETIME);
-                        long cookieSetTime = entity.getLong(COOKIESETTIME);
-                        String lastNode = entity.getString(LASTNODE);
-                        long expiry = entity.getLong(EXPIRY);
-                        long maxInactive = entity.getLong(MAXINACTIVE);
-                        Blob blob = (Blob) entity.getBlob(ATTRIBUTES);
-
-                        Session session = new Session (clusterId, createTime, accessed, maxInactive);
-                        session.setLastNode(lastNode);
-                        session.setContextPath(contextPath);
-                        session.setVHost(vhost);
-                        session.setCookieSetTime(cookieSetTime);
-                        session.setLastAccessedTime(lastAccessed);
-                        session.setLastNode(lastNode);
-                        session.setExpiry(expiry);
-                        try (ClassLoadingObjectInputStream ois = new ClassLoadingObjectInputStream(blob.asInputStream()))
-                        {
-                            Object o = ois.readObject();
-                            session.addAttributes((Map<String,Object>)o);
-                        }
-                        reference.set(session);
-                    }
-                    catch (Exception e)
-                    {
-                        exception.set(e);
-                    }
-                }
-            };
-            
-            if (_context==null)
-                load.run();
-            else
-                _context.getContextHandler().handle(null,load);
-   
-           
-            if (exception.get() != null)
-            {
-                exception.get().printStackTrace();
-                throw exception.get();
-            }
-            
-            return reference.get();
-        }
-    }
-    
-    /*
-     * Every time a Session is put into the cache one of these objects
-     * is created to copy the data out of the in-memory session, and 
-     * every time an object is read from the cache one of these objects
-     * a fresh Session object is created based on the data held by this
-     * object.
-     */
-    public class SerializableSessionData implements Serializable
-    {
-        /**
-         * 
-         */
-        private static final long serialVersionUID = -7779120106058533486L;
-        String clusterId;
-        String contextPath;
-        String vhost;
-        long accessed;
-        long lastAccessed;
-        long createTime;
-        long cookieSetTime;
-        String lastNode;
-        long expiry;
-        long maxInactive;
-        Map<String, Object> attributes;
-
-        public SerializableSessionData()
-        {
-
-        }
-
-       
-       public SerializableSessionData(Session s)
-       {
-           clusterId = s.getClusterId();
-           contextPath = s.getContextPath();
-           vhost = s.getVHost();
-           accessed = s.getAccessed();
-           lastAccessed = s.getLastAccessedTime();
-           createTime = s.getCreationTime();
-           cookieSetTime = s.getCookieSetTime();
-           lastNode = s.getLastNode();
-           expiry = s.getExpiry();
-           maxInactive = s.getMaxInactiveInterval();
-           attributes = s.getAttributeMap(); // TODO pointer, not a copy
-       }
-        
-        private void writeObject(java.io.ObjectOutputStream out) throws IOException
-        {  
-            out.writeUTF(clusterId); //session id
-            out.writeUTF(contextPath); //context path
-            out.writeUTF(vhost); //first vhost
-
-            out.writeLong(accessed);//accessTime
-            out.writeLong(lastAccessed); //lastAccessTime
-            out.writeLong(createTime); //time created
-            out.writeLong(cookieSetTime);//time cookie was set
-            out.writeUTF(lastNode); //name of last node managing
-      
-            out.writeLong(expiry); 
-            out.writeLong(maxInactive);
-            out.writeObject(attributes);
-        }
-        
-        private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException
-        {
-            clusterId = in.readUTF();
-            contextPath = in.readUTF();
-            vhost = in.readUTF();
-            
-            accessed = in.readLong();//accessTime
-            lastAccessed = in.readLong(); //lastAccessTime
-            createTime = in.readLong(); //time created
-            cookieSetTime = in.readLong();//time cookie was set
-            lastNode = in.readUTF(); //last managing node
-            expiry = in.readLong(); 
-            maxInactive = in.readLong();
-            attributes = (HashMap<String,Object>)in.readObject();
-        }
-        
-    }
-    
-
-    
-    /**
+    *//**
      * Session
      *
      * Representation of a session in local memory.
-     */
+     *//*
     public class Session extends MemSession
     {
         
         private ReentrantLock _lock = new ReentrantLock();
         
-        /**
-         * The (canonical) context path for with which this session is associated
-         */
-        private String _contextPath;
-        
-        
-        
-        /**
-         * The time in msec since the epoch at which this session should expire
-         */
-        private long _expiryTime; 
-        
-        
-        /**
-         * Time in msec since the epoch at which this session was last read from cluster
-         */
-        private long _lastSyncTime;
-        
-        
-        /**
-         * The workername of last node known to be managing the session
-         */
-        private String _lastNode;
-        
-        
-        /**
-         * If dirty, session needs to be (re)sent to cluster
-         */
-        protected boolean _dirty=false;
-        
-        
      
+        private long _lastSyncTime;
 
-        /**
-         * Any virtual hosts for the context with which this session is associated
-         */
-        private String _vhost;
-
-        
-        /**
-         * Count of how many threads are active in this session
-         */
         private AtomicInteger _activeThreads = new AtomicInteger(0);
         
         
-        
-        
-        /**
-         * A new session.
-         * 
-         * @param request
-         */
+       
         protected Session (HttpServletRequest request)
         {
-            super(GCloudSessionManager.this,request);
-            long maxInterval = getMaxInactiveInterval();
-            _expiryTime = (maxInterval <= 0 ? 0 : (System.currentTimeMillis() + maxInterval*1000L));
-            _lastNode = getSessionIdManager().getWorkerName();
-           setVHost(GCloudSessionManager.getVirtualHost(_context));
-           setContextPath(GCloudSessionManager.getContextPath(_context));
            _activeThreads.incrementAndGet(); //access will not be called on a freshly created session so increment here
         }
         
         
     
-        
-        /**
-         * A restored session.
-         * 
-         * @param sessionId
-         * @param created
-         * @param accessed
-         * @param maxInterval
-         */
-        protected Session (String sessionId, long created, long accessed, long maxInterval)
-        {
-            super(GCloudSessionManager.this, created, accessed, sessionId);
-            _expiryTime = (maxInterval <= 0 ? 0 : (System.currentTimeMillis() + maxInterval*1000L));
-        }
-        
-        /** 
+        *//** 
          * Called on entry to the session.
          * 
          * @see org.eclipse.jetty.server.session.AbstractSession#access(long)
-         */
+         *//*
         @Override
         protected boolean access(long time)
         {
@@ -459,10 +122,10 @@
         }
 
 
-        /**
+        *//**
          * Exit from session
          * @see org.eclipse.jetty.server.session.AbstractSession#complete()
-         */
+         *//*
         @Override
         protected void complete()
         {
@@ -511,46 +174,24 @@
             }
         }
         
-        /** Test if the session is stale
+        *//** Test if the session is stale
          * @param atTime
          * @return
-         */
+         *//*
         protected boolean isStale (long atTime)
         {
             return (getStaleIntervalSec() > 0) && (atTime - getLastSyncTime() >= (getStaleIntervalSec()*1000L));
         }
-        
-        
-        /** Test if the session is dirty
-         * @return
-         */
-        protected boolean isDirty ()
-        {
-            return _dirty;
-        }
-
-        /** 
-         * Expire the session.
-         * 
-         * @see org.eclipse.jetty.server.session.AbstractSession#timeout()
-         */
-        @Override
-        protected void timeout()
-        {
-            if (LOG.isDebugEnabled()) LOG.debug("Timing out session {}", getId());
-            super.timeout();
-        }
-        
       
         
-        /**
+        *//**
          * Reload the session from the cluster. If the node that
          * last managed the session from the cluster is ourself,
          * then the session does not need refreshing.
          * NOTE: this method MUST be called with sufficient locks
          * in place to prevent 2 or more concurrent threads from
          * simultaneously updating the session.
-         */
+         *//*
         private void refresh () 
         throws Exception
         {
@@ -612,25 +253,6 @@
         }
 
 
-        public void setExpiry (long expiry)
-        {
-            _expiryTime = expiry;
-        }
-        
-
-        public long getExpiry ()
-        {
-            return _expiryTime;
-        }
-        
-        public boolean isExpiredAt (long time)
-        {
-            if (_expiryTime <= 0)
-                return false; //never expires
-            
-            return  (_expiryTime <= time);
-        }
-        
         public void swapId (String newId, String newNodeId)
         {
             //TODO probably synchronize rather than use the access/complete lock?
@@ -639,69 +261,37 @@
             setNodeId(newNodeId);
             _lock.unlock();
         }
-        
-        @Override
-        public void setAttribute (String name, Object value)
-        {
-            Object old = changeAttribute(name, value);
-            if (value == null && old == null)
-                return; //if same as remove attribute but attribute was already removed, no change
-            
-           _dirty = true;
-        }
-        
-        
-        public String getContextPath()
-        {
-            return _contextPath;
-        }
 
 
-        public void setContextPath(String contextPath)
-        {
-            this._contextPath = contextPath;
-        }
+    }
 
+*/
 
-        public String getVHost()
-        {
-            return _vhost;
-        }
+    
+    /**
+     * 
+     */
+    public GCloudSessionManager()
+    {
+        _sessionDataStore = new GCloudSessionDataStore();
+        _sessionStore = new MemorySessionStore();
+    }
 
+    
+    
 
-        public void setVHost(String vhost)
-        {
-            this._vhost = vhost;
-        }
-        
-        public String getLastNode()
-        {
-            return _lastNode;
-        }
-
-
-        public void setLastNode(String lastNode)
-        {
-            _lastNode = lastNode;
-        }
-
-
-        public long getLastSyncTime()
-        {
-            return _lastSyncTime;
-        }
-
-
-        public void setLastSyncTime(long lastSyncTime)
-        {
-            _lastSyncTime = lastSyncTime;
-        }
-
+    /**
+     * @return
+     */
+    public GCloudSessionDataStore getSessionDataStore()
+    {
+        return _sessionDataStore;
     }
 
 
 
-    
+
+
     /**
      * Start the session manager.
      *
@@ -710,32 +300,7 @@
     @Override
     public void doStart() throws Exception
     {
-        if (_sessionIdManager == null)
-            throw new IllegalStateException("No session id manager defined");
-        
-        GCloudConfiguration config = ((GCloudSessionIdManager)_sessionIdManager).getConfig();
-        if (config == null)
-            throw new IllegalStateException("No gcloud configuration");
-        
-        
-        _datastore = DatastoreFactory.instance().get(config.getDatastoreOptions());
-        _keyFactory = _datastore.newKeyFactory().kind(KIND);
-        _converter = new SessionEntityConverter();       
-        _sessions = new ConcurrentHashMap<String, Session>();
-
-        //try and use a common scheduler, fallback to own
-        _scheduler = getSessionHandler().getServer().getBean(Scheduler.class);
-        if (_scheduler == null)
-        {
-            _scheduler = new ScheduledExecutorScheduler();
-            _ownScheduler = true;
-            _scheduler.start();
-        }
-        else if (!_scheduler.isStarted())
-            throw new IllegalStateException("Shared scheduler not started");
- 
-        setScavengeIntervalSec(getScavengeIntervalSec());
-        
+        ((AbstractSessionStore)_sessionStore).setSessionDataStore(_sessionDataStore);
         super.doStart();
     }
 
@@ -749,550 +314,16 @@
     public void doStop() throws Exception
     {
         super.doStop();
-
-        if (_task!=null)
-            _task.cancel();
-        _task=null;
-        if (_ownScheduler && _scheduler !=null)
-            _scheduler.stop();
-        _scheduler = null;
-
-        _sessions.clear();
-        _sessions = null;
     }
 
 
 
-    /**
-     * Look for sessions in local memory that have expired.
-     */
-    public void scavenge ()
-    {
-        try
-        {
-            //scavenge in the database every so often
-            scavengeGCloudDataStore();
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Problem scavenging", e);
-        }
-    }
-
  
     
     protected void scavengeGCloudDataStore()
     throws Exception
     {
        
-        //query the datastore for sessions that have expired
-        long now = System.currentTimeMillis();
-        
-        //give a bit of leeway so we don't immediately something that has only just expired a nanosecond ago
-        now = now - (_scavengeIntervalMs/2);
-        
-        if (LOG.isDebugEnabled())
-            LOG.debug("Scavenging for sessions expired before "+now);
-
-
-        GqlQuery.Builder builder = Query.gqlQueryBuilder(ResultType.ENTITY, "select * from "+KIND+" where expiry < @1 limit "+_maxResults);
-        builder.allowLiteral(true);
-        builder.addBinding(now);
-        Query<Entity> query = builder.build();
-        QueryResults<Entity> results = _datastore.run(query);
-        
-        while (results.hasNext())
-        {          
-            Entity sessionEntity = results.next();
-            scavengeSession(sessionEntity);        
-        }
-
+     
     }
-
-    /**
-     * Scavenge a session that has expired
-     * @param e
-     * @throws Exception
-     */
-    protected void scavengeSession (Entity e)
-            throws Exception
-    {
-        long now = System.currentTimeMillis();
-        Session session = _converter.sessionFromEntity(e);
-        if (session == null)
-            return;
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("Scavenging session: {}",session.getId());
-        //if the session isn't in memory already, put it there so we can do a normal timeout call
-         Session memSession =  _sessions.putIfAbsent(session.getId(), session);
-         if (memSession == null)
-         {
-             memSession = session;
-         }
-
-        //final check
-        if (memSession.isExpiredAt(now))
-        {
-            if (LOG.isDebugEnabled()) LOG.debug("Session {} is definitely expired", memSession.getId());
-            memSession.timeout();   
-        }
-    }
-
-    public long getScavengeIntervalSec ()
-    {
-        return _scavengeIntervalMs/1000;
-    }
-
-    
-    
-    /**
-     * Set the interval between runs of the scavenger. It should not be run too
-     * often.
-     * 
-     * 
-     * @param sec
-     */
-    public void setScavengeIntervalSec (long sec)
-    {
-
-        long old_period=_scavengeIntervalMs;
-        long period=sec*1000L;
-
-        _scavengeIntervalMs=period;
-
-        if (_scavengeIntervalMs > 0)
-        {
-            //add a bit of variability into the scavenge time so that not all
-            //nodes with the same scavenge time sync up
-            long tenPercent = _scavengeIntervalMs/10;
-            if ((System.currentTimeMillis()%2) == 0)
-                _scavengeIntervalMs += tenPercent;
-            if (LOG.isDebugEnabled())
-                LOG.debug("Scavenging every "+_scavengeIntervalMs+" ms");
-        }
-        else
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Scavenging disabled"); 
-        }
-
- 
-        
-        synchronized (this)
-        {
-            if (_scheduler != null && (period!=old_period || _task==null))
-            {
-                //clean up any previously scheduled scavenger
-                if (_task!=null)
-                    _task.cancel();
-
-                //start a new one
-                if (_scavengeIntervalMs > 0)
-                {
-                    if (_scavenger == null)
-                        _scavenger = new Scavenger();
-
-                    _task = _scheduler.schedule(_scavenger,_scavengeIntervalMs,TimeUnit.MILLISECONDS);
-                }
-            }
-        }
-    }
-    
-    
-    public long getStaleIntervalSec()
-    {
-        return _staleIntervalSec;
-    }
-
-
-    public void setStaleIntervalSec(long staleIntervalSec)
-    {
-        _staleIntervalSec = staleIntervalSec;
-    }
-    
-    
-    public int getMaxResults()
-    {
-        return _maxResults;
-    }
-
-
-    public void setMaxResults(int maxResults)
-    {
-        if (_maxResults <= 0)
-            _maxResults = DEFAULT_MAX_QUERY_RESULTS;
-        else
-            _maxResults = maxResults;
-    }
-
-
-    /** 
-     * Add a new session for the context related to this session manager
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#addSession(org.eclipse.jetty.server.session.AbstractSession)
-     */
-    @Override
-    protected void addSession(AbstractSession session)
-    {
-        if (session==null)
-            return;
-        
-        if (LOG.isDebugEnabled()) LOG.debug("Adding session({}) to session manager for context {} on worker {}",session.getClusterId(), getContextPath(getContext()),getSessionIdManager().getWorkerName() + " with lastnode="+((Session)session).getLastNode());
-        _sessions.put(session.getClusterId(), (Session)session);
-        
-        try
-        {     
-                session.willPassivate();
-                save(((GCloudSessionManager.Session)session));
-                session.didActivate();
-            
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Unable to store new session id="+session.getId() , e);
-        }
-    }
-
-    /** 
-     * Ask the cluster for the session.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#getSession(java.lang.String)
-     */
-    @Override
-    public AbstractSession getSession(String idInCluster)
-    {
-        Session session = null;
-
-        //try and find the session in this node's memory
-        Session memSession = (Session)_sessions.get(idInCluster);
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("getSession({}) {} in session map",idInCluster,(memSession==null?"not":""));
-
-        long now = System.currentTimeMillis();
-        try
-        {
-            //if the session is not in this node's memory, then load it from the datastore
-            if (memSession == null)
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("getSession({}): loading session data from cluster", idInCluster);
-
-                session = load(makeKey(idInCluster, _context));
-                if (session != null)
-                {
-                    //Check that it wasn't expired
-                    if (session.getExpiry() > 0 && session.getExpiry() <= now)
-                    {
-                        if (LOG.isDebugEnabled()) LOG.debug("getSession ({}): Session expired", idInCluster);
-                        //ensure that the session id for the expired session is deleted so that a new session with the 
-                        //same id cannot be created (because the idInUse() test would succeed)
-                        ((GCloudSessionIdManager)getSessionIdManager()).removeSession(session);
-                        return null;  
-                    }
-
-                    //Update the last worker node to me
-                    session.setLastNode(getSessionIdManager().getWorkerName());                            
-                    //TODO consider saving session here if lastNode was not this node
-
-                    //Check that another thread hasn't loaded the same session
-                    Session existingSession = _sessions.putIfAbsent(idInCluster, session);
-                    if (existingSession != null)
-                    {
-                        //use the one that the other thread inserted
-                        session = existingSession;
-                        LOG.debug("getSession({}): using session loaded by another request thread ", idInCluster);
-                    }
-                    else
-                    {
-                        //indicate that the session was reinflated
-                        session.didActivate();
-                        LOG.debug("getSession({}): loaded session from cluster", idInCluster);
-                    }
-                    return session;
-                }
-                else
-                {
-                    //The requested session does not exist anywhere in the cluster
-                    LOG.debug("getSession({}): No session in cluster matching",idInCluster);
-                    return null;
-                }
-            }
-            else
-            {
-               //The session exists in this node's memory
-               LOG.debug("getSession({}): returning session from local memory ", memSession.getClusterId());
-                return memSession;
-            }
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Unable to load session="+idInCluster, e);
-            return null;
-        }
-    }
-    
-    
-
-    /** 
-     * The session manager is stopping.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#shutdownSessions()
-     */
-    @Override
-    protected void shutdownSessions() throws Exception
-    {
-        Set<String> keys = new HashSet<String>(_sessions.keySet());
-        for (String key:keys)
-        {
-            Session session = _sessions.remove(key); //take the session out of the session list
-            //If the session is dirty, then write it to the cluster.
-            //If the session is simply stale do NOT write it to the cluster, as some other node
-            //may have started managing that session - this means that the last accessed/expiry time
-            //will not be updated, meaning it may look like it can expire sooner than it should.
-            try
-            {
-                if (session.isDirty())
-                {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Saving dirty session {} before exiting ", session.getId());
-                    save(session);
-                }
-            }
-            catch (Exception e)
-            {
-                LOG.warn(e);
-            }
-        }
-    }
-
-
-    @Override
-    protected AbstractSession newSession(HttpServletRequest request)
-    {
-        return new Session(request);
-    }
-
-    /** 
-     * Remove a session from local memory, and delete it from
-     * the cluster cache.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#removeSession(java.lang.String)
-     */
-    @Override
-    protected boolean removeSession(String idInCluster)
-    {
-        Session session = (Session)_sessions.remove(idInCluster);
-        try
-        {
-            if (session != null)
-            {
-                delete(session);
-            }
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Problem deleting session id="+idInCluster, e);
-        }
-        return session!=null;
-    }
-    
-    
-    
-    
-    @Override
-    public void renewSessionId(String oldClusterId, String oldNodeId, String newClusterId, String newNodeId)
-    {
-        Session session = null;
-        try
-        {
-            //take the session with that id out of our managed list
-            session = (Session)_sessions.remove(oldClusterId);
-            if (session != null)
-            {
-                //TODO consider transactionality and ramifications if the session is live on another node
-                delete(session); //delete the old session from the cluster  
-                session.swapId(newClusterId, newNodeId); //update the session
-                _sessions.put(newClusterId, session); //put it into managed list under new key
-                save(session); //put the session under the new id into the cluster
-            }
-        }
-        catch (Exception e)
-        {
-            LOG.warn(e);
-        }
-
-        super.renewSessionId(oldClusterId, oldNodeId, newClusterId, newNodeId);
-    }
-
-
-    /**
-     * Load a session from the clustered cache.
-     * 
-     * @param key
-     * @return
-     */
-    protected Session load (Key key)
-    throws Exception
-    {
-        if (_datastore == null)
-            throw new IllegalStateException("No DataStore");
-        
-        if (LOG.isDebugEnabled()) LOG.debug("Loading session {} from DataStore", key);
-
-        Entity entity = _datastore.get(key);
-        if (entity == null)
-        {
-            if (LOG.isDebugEnabled()) LOG.debug("No session {} in DataStore ",key);
-            return null;
-        }
-        else
-        {
-            Session session = _converter.sessionFromEntity(entity);
-            session.setLastSyncTime(System.currentTimeMillis());
-            return session;
-        }
-    }
-    
-    
-    
-    /**
-     * Save or update the session to the cluster cache
-     * 
-     * @param session
-     * @throws Exception
-     */
-    protected void save (GCloudSessionManager.Session session)
-    throws Exception
-    {
-        if (_datastore == null)
-            throw new IllegalStateException("No DataStore");
-        
-        if (LOG.isDebugEnabled()) LOG.debug("Writing session {} to DataStore", session.getId());
-    
-        Entity entity = _converter.entityFromSession(session, makeKey(session, _context));
-        _datastore.put(entity);
-        session.setLastSyncTime(System.currentTimeMillis());
-    }
-    
-    
-    
-    /**
-     * Remove the session from the cluster cache.
-     * 
-     * @param session
-     */
-    protected void delete (GCloudSessionManager.Session session)
-    {  
-        if (_datastore == null)
-            throw new IllegalStateException("No DataStore");
-        if (LOG.isDebugEnabled()) LOG.debug("Removing session {} from DataStore", session.getId());
-        _datastore.delete(makeKey(session, _context));
-    }
-
-    
-    /**
-     * Invalidate a session for this context with the given id
-     * 
-     * @param idInCluster
-     */
-    public void invalidateSession (String idInCluster)
-    {
-        Session session = (Session)_sessions.get(idInCluster);
-
-        if (session != null)
-        {
-            session.invalidate();
-        }
-    }
-
-    
-    /**
-     * Make a unique key for this session.
-     * As the same session id can be used across multiple contexts, to
-     * make it unique, the key must be composed of:
-     * <ol>
-     * <li>the id</li>
-     * <li>the context path</li>
-     * <li>the virtual hosts</li>
-     * </ol>
-     * 
-     *TODO consider the difference between getClusterId and getId
-     * @param session
-     * @return
-     */
-    private Key makeKey (Session session, Context context)
-    {
-       return makeKey(session.getId(), context);
-    }
-    
-    /**
-     * Make a unique key for this session.
-     * As the same session id can be used across multiple contexts, to
-     * make it unique, the key must be composed of:
-     * <ol>
-     * <li>the id</li>
-     * <li>the context path</li>
-     * <li>the virtual hosts</li>
-     * </ol>
-     * 
-     *TODO consider the difference between getClusterId and getId
-     * @param session
-     * @return
-     */
-    private Key makeKey (String id, Context context)
-    {
-        String key = getContextPath(context);
-        key = key + "_" + getVirtualHost(context);
-        key = key+"_"+id;
-        return _keyFactory.newKey(key);
-    }
-    
-    /**
-     * Turn the context path into an acceptable string
-     * 
-     * @param context
-     * @return
-     */
-    private static String getContextPath (ContextHandler.Context context)
-    {
-        return canonicalize (context.getContextPath());
-    }
-
-    /**
-     * Get the first virtual host for the context.
-     *
-     * Used to help identify the exact session/contextPath.
-     *
-     * @return 0.0.0.0 if no virtual host is defined
-     */
-    private static String getVirtualHost (ContextHandler.Context context)
-    {
-        String vhost = "0.0.0.0";
-
-        if (context==null)
-            return vhost;
-
-        String [] vhosts = context.getContextHandler().getVirtualHosts();
-        if (vhosts==null || vhosts.length==0 || vhosts[0]==null)
-            return vhost;
-
-        return vhosts[0];
-    }
-
-    /**
-     * Make an acceptable name from a context path.
-     *
-     * @param path
-     * @return
-     */
-    private static String canonicalize (String path)
-    {
-        if (path==null)
-            return "";
-
-        return path.replace('/', '_').replace('.','_').replace('\\','_');
-    }
-
 }
diff --git a/jetty-gcloud/pom.xml b/jetty-gcloud/pom.xml
index 7703328..3d571b8 100644
--- a/jetty-gcloud/pom.xml
+++ b/jetty-gcloud/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>jetty-project</artifactId>
     <groupId>org.eclipse.jetty</groupId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http-spi/pom.xml b/jetty-http-spi/pom.xml
index f280ccb..03b9cd7 100644
--- a/jetty-http-spi/pom.xml
+++ b/jetty-http-spi/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-http-spi</artifactId>
diff --git a/jetty-http/pom.xml b/jetty-http/pom.xml
index 4551cfa..8455a9d 100644
--- a/jetty-http/pom.xml
+++ b/jetty-http/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>jetty-project</artifactId>
     <groupId>org.eclipse.jetty</groupId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-http</artifactId>
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/HttpHeader.java b/jetty-http/src/main/java/org/eclipse/jetty/http/HttpHeader.java
index b57a5fa..b28aefd 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/HttpHeader.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/HttpHeader.java
@@ -101,6 +101,16 @@
     WWW_AUTHENTICATE("WWW-Authenticate"),
 
     /* ------------------------------------------------------------ */
+    /** WebSocket Fields.
+     */
+    ORIGIN("Origin"),
+    SEC_WEBSOCKET_KEY("Sec-WebSocket-Key"),
+    SEC_WEBSOCKET_VERSION("Sec-WebSocket-Version"),
+    SEC_WEBSOCKET_EXTENSIONS("Sec-WebSocket-Extensions"),
+    SEC_WEBSOCKET_SUBPROTOCOL("Sec-WebSocket-Protocol"),
+    SEC_WEBSOCKET_ACCEPT("Sec-WebSocket-Accept"),
+
+    /* ------------------------------------------------------------ */
     /** Other Fields.
      */
     COOKIE("Cookie"),
@@ -127,7 +137,7 @@
 
 
     /* ------------------------------------------------------------ */
-    public final static Trie<HttpHeader> CACHE= new ArrayTrie<>(560);
+    public final static Trie<HttpHeader> CACHE= new ArrayTrie<>(630);
     static
     {
         for (HttpHeader header : HttpHeader.values())
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/HttpURI.java b/jetty-http/src/main/java/org/eclipse/jetty/http/HttpURI.java
index fe3768b..6b0a39a 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/HttpURI.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/HttpURI.java
@@ -119,6 +119,7 @@
     public HttpURI(HttpURI uri)
     {
         this(uri._scheme,uri._host,uri._port,uri._path,uri._param,uri._query,uri._fragment);
+        _uri=uri._uri;
     }
     
     /* ------------------------------------------------------------ */
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/MetaData.java b/jetty-http/src/main/java/org/eclipse/jetty/http/MetaData.java
index 725870d..18dc231 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/MetaData.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/MetaData.java
@@ -158,7 +158,6 @@
             this(request.getMethod(),new HttpURI(request.getURI()), request.getVersion(), new HttpFields(request.getFields()), request.getContentLength());
         }
 
-        // TODO MetaData should be immuttable!!! 
         public void recycle()
         {
             super.recycle();
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java b/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java
index 61c047c..e20bf83 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.function.Predicate;
+import java.util.function.Predicate;
 
 import org.eclipse.jetty.util.ArrayTernaryTrie;
 import org.eclipse.jetty.util.Trie;
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathMappings.java b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathMappings.java
index d1c36cf..c6500f3 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathMappings.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathMappings.java
@@ -20,10 +20,13 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
+import org.eclipse.jetty.util.ArrayTernaryTrie;
+import org.eclipse.jetty.util.Trie;
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
 import org.eclipse.jetty.util.annotation.ManagedObject;
 import org.eclipse.jetty.util.component.ContainerLifeCycle;
@@ -42,10 +45,12 @@
 public class PathMappings<E> implements Iterable<MappedResource<E>>, Dumpable
 {
     private static final Logger LOG = Log.getLogger(PathMappings.class);
-    private List<MappedResource<E>> mappings = new ArrayList<MappedResource<E>>();
-    private MappedResource<E> defaultResource = null;
-    private MappedResource<E> rootResource = null;
-
+    private final Set<MappedResource<E>> _mappings = new TreeSet<>();
+    
+    private Trie<MappedResource<E>> _exactMap=new ArrayTernaryTrie<>(false);
+    private Trie<MappedResource<E>> _prefixMap=new ArrayTernaryTrie<>(false);
+    private Trie<MappedResource<E>> _suffixMap=new ArrayTernaryTrie<>(false);
+    
     @Override
     public String dump()
     {
@@ -55,18 +60,25 @@
     @Override
     public void dump(Appendable out, String indent) throws IOException
     {
-        ContainerLifeCycle.dump(out,indent,mappings);
+        ContainerLifeCycle.dump(out,indent,_mappings);
     }
 
     @ManagedAttribute(value = "mappings", readonly = true)
     public List<MappedResource<E>> getMappings()
     {
-        return mappings;
+        return new ArrayList<>(_mappings);
     }
 
+    public int size()
+    {
+        return _mappings.size();
+    }
+    
     public void reset()
     {
-        mappings.clear();
+        _mappings.clear();
+        _prefixMap.clear();
+        _suffixMap.clear();
     }
     
     /**
@@ -77,22 +89,19 @@
      */
     public List<MappedResource<E>> getMatches(String path)
     {
-        boolean matchRoot = "/".equals(path);
+        boolean isRootPath = "/".equals(path);
         
         List<MappedResource<E>> ret = new ArrayList<>();
-        int len = mappings.size();
-        for (int i = 0; i < len; i++)
+        for (MappedResource<E> mr :_mappings)
         {
-            MappedResource<E> mr = mappings.get(i);
-
             switch (mr.getPathSpec().group)
             {
                 case ROOT:
-                    if (matchRoot)
+                    if (isRootPath)
                         ret.add(mr);
                     break;
                 case DEFAULT:
-                    if (matchRoot || mr.getPathSpec().matches(path))
+                    if (isRootPath || mr.getPathSpec().matches(path))
                         ret.add(mr);
                     break;
                 default:
@@ -106,54 +115,160 @@
 
     public MappedResource<E> getMatch(String path)
     {
-        if (path.equals("/") && rootResource != null)
+        PathSpecGroup last_group=null;
+        
+        // Search all the mappings
+        for (MappedResource<E> mr : _mappings)
         {
-            return rootResource;
+            PathSpecGroup group=mr.getPathSpec().getGroup();
+            if (group!=last_group)
+            {
+                // New group in list, so let's look for an optimization
+                switch(group)
+                {
+                    case EXACT:
+                    {
+                        int i= path.length();
+                        final Trie<MappedResource<E>> exact_map=_exactMap;
+                        while(i>=0)
+                        {
+                            MappedResource<E> candidate=exact_map.getBest(path,0,i);
+                            if (candidate==null)
+                                break;
+                            if (candidate.getPathSpec().matches(path))
+                                return candidate;
+                            i=candidate.getPathSpec().getPrefix().length()-1;
+                        }
+                        break;
+                    }
+                        
+                    case PREFIX_GLOB:
+                    {
+                        int i= path.length();
+                        final Trie<MappedResource<E>> prefix_map=_prefixMap;
+                        while(i>=0)
+                        {
+                            MappedResource<E> candidate=prefix_map.getBest(path,0,i);
+                            if (candidate==null)
+                                break;
+                            if (candidate.getPathSpec().matches(path))
+                                return candidate;
+                            i=candidate.getPathSpec().getPrefix().length()-1;
+                        }
+                        break;
+                    }
+                        
+                    case SUFFIX_GLOB:
+                    {
+                        int i=0;
+                        final Trie<MappedResource<E>> suffix_map=_suffixMap;
+                        while ((i=path.indexOf('.',i+1))>0)
+                        {
+                            MappedResource<E> candidate=suffix_map.get(path,i+1,path.length()-i-1);
+                            if (candidate!=null && candidate.getPathSpec().matches(path))
+                                return candidate;
+                        }
+                        break;
+                    }
+                    
+                    default:
+                }   
+            }
+            
+            if (mr.getPathSpec().matches(path))
+                return mr;
+            
+            last_group=group;
         }
         
-        int len = mappings.size();
-        for (int i = 0; i < len; i++)
-        {
-            MappedResource<E> mr = mappings.get(i);
-            if (mr.getPathSpec().matches(path))
-            {
-                return mr;
-            }
-        }
-        return defaultResource;
+        return null;
     }
 
     @Override
     public Iterator<MappedResource<E>> iterator()
     {
-        return mappings.iterator();
+        return _mappings.iterator();
     }
 
-    @SuppressWarnings("incomplete-switch")
-    public void put(PathSpec pathSpec, E resource)
+    public static PathSpec asPathSpec(String pathSpecString)
+    {
+        if ((pathSpecString == null) || (pathSpecString.length() < 1))
+        {
+            throw new RuntimeException("Path Spec String must start with '^', '/', or '*.': got [" + pathSpecString + "]");
+        }
+        return pathSpecString.charAt(0) == '^' ? new RegexPathSpec(pathSpecString):new ServletPathSpec(pathSpecString);
+    }
+    
+    public boolean put(String pathSpecString, E resource)
+    {
+        return put(asPathSpec(pathSpecString),resource);
+    }
+    
+    public boolean put(PathSpec pathSpec, E resource)
     {
         MappedResource<E> entry = new MappedResource<>(pathSpec,resource);
         switch (pathSpec.group)
         {
-            case DEFAULT:
-                defaultResource = entry;
+            case EXACT:
+                String exact = pathSpec.getPrefix();
+                while (exact!=null && !_exactMap.put(exact,entry))
+                    _exactMap=new ArrayTernaryTrie<>((ArrayTernaryTrie<MappedResource<E>>)_exactMap,1.5);
                 break;
-            case ROOT:
-                rootResource = entry;
+            case PREFIX_GLOB:
+                String prefix = pathSpec.getPrefix();
+                while (prefix!=null && !_prefixMap.put(prefix,entry))
+                    _prefixMap=new ArrayTernaryTrie<>((ArrayTernaryTrie<MappedResource<E>>)_prefixMap,1.5);
+                break;
+            case SUFFIX_GLOB:
+                String suffix = pathSpec.getSuffix();
+                while (suffix!=null && !_suffixMap.put(suffix,entry))
+                    _suffixMap=new ArrayTernaryTrie<>((ArrayTernaryTrie<MappedResource<E>>)_prefixMap,1.5);
+                break;
+            default:
+        }
+        
+        boolean added =_mappings.add(entry);
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} {} to {}",added?"Added":"Ignored",entry,this);
+        return added;
+    }
+    
+    @SuppressWarnings("incomplete-switch")
+    public boolean remove(PathSpec pathSpec)
+    {
+        switch (pathSpec.group)
+        {
+            case EXACT:
+                _exactMap.remove(pathSpec.getPrefix());
+                break;
+            case PREFIX_GLOB:
+                _prefixMap.remove(pathSpec.getPrefix());
+                break;
+            case SUFFIX_GLOB:
+                _suffixMap.remove(pathSpec.getSuffix());
                 break;
         }
         
-        // TODO: add warning when replacing an existing pathspec?
-        
-        mappings.add(entry);
+        Iterator<MappedResource<E>> iter = _mappings.iterator();
+        boolean removed=false;
+        while (iter.hasNext())
+        {
+            if (iter.next().getPathSpec().equals(pathSpec))
+            {
+                removed=true;
+                iter.remove();
+                break;
+            }
+        }
         if (LOG.isDebugEnabled())
-            LOG.debug("Added {} to {}",entry,this);
-        Collections.sort(mappings);
+            LOG.debug("{} {} to {}",removed?"Removed":"Ignored",pathSpec,this);
+        return removed;
     }
 
     @Override
     public String toString()
     {
-        return String.format("%s[size=%d]",this.getClass().getSimpleName(),mappings.size());
+        return String.format("%s[size=%d]",this.getClass().getSimpleName(),_mappings.size());
     }
+
 }
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpec.java b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpec.java
index a2b8ea5..8a1f82b 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpec.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpec.java
@@ -27,6 +27,8 @@
     protected PathSpecGroup group;
     protected int pathDepth;
     protected int specLength;
+    protected String prefix;
+    protected String suffix;
 
     @Override
     public int compareTo(PathSpec other)
@@ -125,6 +127,24 @@
     }
 
     /**
+     * A simple prefix match for the pathspec or null
+     * @return A simple prefix match for the pathspec or null
+     */
+    public String getPrefix()
+    {
+        return prefix;
+    }
+
+    /**
+     * A simple suffix match for the pathspec or null
+     * @return A simple suffix match for the pathspec or null
+     */
+    public String getSuffix()
+    {
+        return suffix;
+    }
+    
+    /**
      * Get the relative path.
      * 
      * @param base
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecGroup.java b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecGroup.java
index f9d96ce..e03a035 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecGroup.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecGroup.java
@@ -35,6 +35,17 @@
     // NOTE: Order of enums determines order of Groups.
 
     /**
+     * The root spec for accessing the Root behavior.
+     * 
+     * <pre>
+     *   ""           - servlet spec       (Root Servlet)
+     *   null         - servlet spec       (Root Servlet)
+     * </pre>
+     * 
+     * Note: there is no known uri-template spec variant of this kind of path spec
+     */
+    ROOT,
+    /**
      * For exactly defined path specs, no glob.
      */
     EXACT,
@@ -75,17 +86,6 @@
      */
     SUFFIX_GLOB,
     /**
-     * The root spec for accessing the Root behavior.
-     * 
-     * <pre>
-     *   ""           - servlet spec       (Root Servlet)
-     *   null         - servlet spec       (Root Servlet)
-     * </pre>
-     * 
-     * Note: there is no known uri-template spec variant of this kind of path spec
-     */
-    ROOT,
-    /**
      * The default spec for accessing the Default path behavior.
      * 
      * <pre>
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecSet.java b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecSet.java
index c1a3235..b898cf5 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecSet.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/PathSpecSet.java
@@ -18,12 +18,8 @@
 
 package org.eclipse.jetty.http.pathmap;
 
-import java.util.ArrayList;
-import java.util.Collection;
+import java.util.AbstractSet;
 import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
 import java.util.function.Predicate;
 
 /**
@@ -31,60 +27,16 @@
  * <p>
  * Used by {@link org.eclipse.jetty.util.IncludeExclude} logic
  */
-public class PathSpecSet implements Set<String>, Predicate<String>
+public class PathSpecSet extends AbstractSet<String> implements Predicate<String>
 {
-    private final Set<PathSpec> specs = new TreeSet<>();
+    private final PathMappings<Boolean> specs = new PathMappings<>();
 
     @Override
     public boolean test(String s)
     {
-        for (PathSpec spec : specs)
-        {
-            if (spec.matches(s))
-            {
-                return true;
-            }
-        }
-        return false;
+        return specs.getMatch(s)!=null;
     }
 
-    @Override
-    public boolean isEmpty()
-    {
-        return specs.isEmpty();
-    }
-
-    @Override
-    public Iterator<String> iterator()
-    {
-        return new Iterator<String>()
-        {
-            private Iterator<PathSpec> iter = specs.iterator();
-
-            @Override
-            public boolean hasNext()
-            {
-                return iter.hasNext();
-            }
-
-            @Override
-            public String next()
-            {
-                PathSpec spec = iter.next();
-                if (spec == null)
-                {
-                    return null;
-                }
-                return spec.getDeclaration();
-            }
-
-            @Override
-            public void remove()
-            {
-                throw new UnsupportedOperationException("Remove not supported by this Iterator");
-            }
-        };
-    }
 
     @Override
     public int size()
@@ -92,20 +44,6 @@
         return specs.size();
     }
 
-    @Override
-    public boolean contains(Object o)
-    {
-        if (o instanceof PathSpec)
-        {
-            return specs.contains(o);
-        }
-        if (o instanceof String)
-        {
-            return specs.contains(toPathSpec((String)o));
-        }
-        return false;
-    }
-
     private PathSpec asPathSpec(Object o)
     {
         if (o == null)
@@ -118,48 +56,15 @@
         }
         if (o instanceof String)
         {
-            return toPathSpec((String)o);
+            return PathMappings.asPathSpec((String)o);
         }
-        return toPathSpec(o.toString());
-    }
-
-    private PathSpec toPathSpec(String rawSpec)
-    {
-        if ((rawSpec == null) || (rawSpec.length() < 1))
-        {
-            throw new RuntimeException("Path Spec String must start with '^', '/', or '*.': got [" + rawSpec + "]");
-        }
-        if (rawSpec.charAt(0) == '^')
-        {
-            return new RegexPathSpec(rawSpec);
-        }
-        else
-        {
-            return new ServletPathSpec(rawSpec);
-        }
+        return PathMappings.asPathSpec(o.toString());
     }
 
     @Override
-    public Object[] toArray()
+    public boolean add(String s)
     {
-        return toArray(new String[specs.size()]);
-    }
-
-    @Override
-    public <T> T[] toArray(T[] a)
-    {
-        int i = 0;
-        for (PathSpec spec : specs)
-        {
-            a[i++] = (T)spec.getDeclaration();
-        }
-        return a;
-    }
-
-    @Override
-    public boolean add(String e)
-    {
-        return specs.add(toPathSpec(e));
+        return specs.put(PathMappings.asPathSpec(s),Boolean.TRUE);
     }
 
     @Override
@@ -169,54 +74,29 @@
     }
 
     @Override
-    public boolean containsAll(Collection<?> coll)
-    {
-        for (Object o : coll)
-        {
-            if (!specs.contains(asPathSpec(o)))
-                return false;
-        }
-        return true;
-    }
-
-    @Override
-    public boolean addAll(Collection<? extends String> coll)
-    {
-        boolean ret = false;
-
-        for (String s : coll)
-        {
-            ret |= add(s);
-        }
-
-        return ret;
-    }
-
-    @Override
-    public boolean retainAll(Collection<?> coll)
-    {
-        List<PathSpec> collSpecs = new ArrayList<>();
-        for (Object o : coll)
-        {
-            collSpecs.add(asPathSpec(o));
-        }
-        return specs.retainAll(collSpecs);
-    }
-
-    @Override
-    public boolean removeAll(Collection<?> coll)
-    {
-        List<PathSpec> collSpecs = new ArrayList<>();
-        for (Object o : coll)
-        {
-            collSpecs.add(asPathSpec(o));
-        }
-        return specs.removeAll(collSpecs);
-    }
-
-    @Override
     public void clear()
     {
-        specs.clear();
+        specs.reset();
+    }
+
+
+    @Override
+    public Iterator<String> iterator()
+    {
+        final Iterator<MappedResource<Boolean>> iterator = specs.iterator();
+        return new Iterator<String>()
+        {
+            @Override
+            public boolean hasNext()
+            {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public String next()
+            {
+                return iterator.next().getPathSpec().getDeclaration();
+            }
+        };
     }
 }
diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/ServletPathSpec.java b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/ServletPathSpec.java
index 4563305..9f0732e 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/ServletPathSpec.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/pathmap/ServletPathSpec.java
@@ -54,15 +54,18 @@
         if ((servletPathSpec.charAt(0) == '/') && (specLength > 1) && (lastChar == '*'))
         {
             this.group = PathSpecGroup.PREFIX_GLOB;
+            this.prefix = servletPathSpec.substring(0,specLength-2);
         }
         // suffix based
         else if (servletPathSpec.charAt(0) == '*')
         {
             this.group = PathSpecGroup.SUFFIX_GLOB;
+            this.suffix = servletPathSpec.substring(2,specLength);
         }
         else
         {
             this.group = PathSpecGroup.EXACT;
+            this.prefix = servletPathSpec;
         }
 
         for (int i = 0; i < specLength; i++)
@@ -109,6 +112,11 @@
             {
                 throw new IllegalArgumentException("Servlet Spec 12.2 violation: glob '*' can only exist at end of prefix based matches: bad spec \""+ servletPathSpec +"\"");
             }
+            
+            if (idx<1 || servletPathSpec.charAt(idx-1)!='/')
+            {
+                throw new IllegalArgumentException("Servlet Spec 12.2 violation: suffix glob '*' can only exist after '/': bad spec \""+ servletPathSpec +"\"");
+            }
         }
         else if (servletPathSpec.startsWith("*."))
         {
diff --git a/jetty-http/src/test/java/org/eclipse/jetty/http/pathmap/PathMappingsTest.java b/jetty-http/src/test/java/org/eclipse/jetty/http/pathmap/PathMappingsTest.java
index f3f2ef2..7b1c864 100644
--- a/jetty-http/src/test/java/org/eclipse/jetty/http/pathmap/PathMappingsTest.java
+++ b/jetty-http/src/test/java/org/eclipse/jetty/http/pathmap/PathMappingsTest.java
@@ -278,4 +278,14 @@
         assertEquals("suffix",p.getMatch("/foo/something.txt").getResource());
         assertEquals("prefix",p.getMatch("/dump/gzip/something.txt").getResource());
     }
+    
+    @Test
+    public void testBadPathSpecs()
+    {
+        try{new ServletPathSpec("*");Assert.fail();}catch(IllegalArgumentException e){}
+        try{new ServletPathSpec("/foo/*/bar");Assert.fail();}catch(IllegalArgumentException e){}
+        try{new ServletPathSpec("/foo*");Assert.fail();}catch(IllegalArgumentException e){}
+        try{new ServletPathSpec("*/foo");Assert.fail();}catch(IllegalArgumentException e){}
+        try{new ServletPathSpec("*.foo/*");Assert.fail();}catch(IllegalArgumentException e){}
+    }
 }
diff --git a/jetty-http2/http2-alpn-tests/pom.xml b/jetty-http2/http2-alpn-tests/pom.xml
index b7e8af4..8a1521e 100644
--- a/jetty-http2/http2-alpn-tests/pom.xml
+++ b/jetty-http2/http2-alpn-tests/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.eclipse.jetty.http2</groupId>
         <artifactId>http2-parent</artifactId>
-        <version>9.3.8-SNAPSHOT</version>
+        <version>9.4.0-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http2/http2-client/pom.xml b/jetty-http2/http2-client/pom.xml
index 9520bea..56fb4dd 100644
--- a/jetty-http2/http2-client/pom.xml
+++ b/jetty-http2/http2-client/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty.http2</groupId>
     <artifactId>http2-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2Client.java b/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2Client.java
index 05a8621..55c3dca 100644
--- a/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2Client.java
+++ b/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2Client.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.SocketChannel;
 import java.util.Arrays;
@@ -38,8 +39,8 @@
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.MappedByteBufferPool;
-import org.eclipse.jetty.io.SelectChannelEndPoint;
 import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.io.SocketChannelEndPoint;
 import org.eclipse.jetty.io.ssl.SslClientConnectionFactory;
 import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
@@ -376,13 +377,15 @@
         }
 
         @Override
-        protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
+        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
         {
-            return new SelectChannelEndPoint(channel, selector, selectionKey, getScheduler(), getIdleTimeout());
+            SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, selectionKey, getScheduler());
+            endp.setIdleTimeout(getIdleTimeout());
+            return endp;
         }
 
         @Override
-        public Connection newConnection(SocketChannel channel, EndPoint endpoint, Object attachment) throws IOException
+        public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment) throws IOException
         {
             @SuppressWarnings("unchecked")
             Map<String, Object> context = (Map<String, Object>)attachment;
@@ -393,7 +396,7 @@
         }
 
         @Override
-        protected void connectionFailed(SocketChannel channel, Throwable failure, Object attachment)
+        protected void connectionFailed(SelectableChannel channel, Throwable failure, Object attachment)
         {
             @SuppressWarnings("unchecked")
             Map<String, Object> context = (Map<String, Object>)attachment;
diff --git a/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2ClientConnectionFactory.java b/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2ClientConnectionFactory.java
index 28aad7e..bf5688c 100644
--- a/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2ClientConnectionFactory.java
+++ b/jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2ClientConnectionFactory.java
@@ -51,7 +51,6 @@
     public static final String SESSION_PROMISE_CONTEXT_KEY = "http2.client.sessionPromise";
 
     private final Connection.Listener connectionListener = new ConnectionListener();
-    private int initialSessionRecvWindow = FlowControlStrategy.DEFAULT_WINDOW_SIZE;
 
     @Override
     public Connection newConnection(EndPoint endPoint, Map<String, Object> context) throws IOException
@@ -65,9 +64,7 @@
         Promise<Session> promise = (Promise<Session>)context.get(SESSION_PROMISE_CONTEXT_KEY);
 
         Generator generator = new Generator(byteBufferPool);
-        FlowControlStrategy flowControl = newFlowControlStrategy();
-        if (flowControl == null)
-            flowControl = client.getFlowControlStrategyFactory().newFlowControlStrategy();
+        FlowControlStrategy flowControl = client.getFlowControlStrategyFactory().newFlowControlStrategy();
         HTTP2ClientSession session = new HTTP2ClientSession(scheduler, endPoint, generator, listener, flowControl);
         Parser parser = new Parser(byteBufferPool, session, 4096, 8192);
         HTTP2ClientConnection connection = new HTTP2ClientConnection(client, byteBufferPool, executor, endPoint, parser, session, client.getInputBufferSize(), promise, listener);
@@ -75,33 +72,6 @@
         return connection;
     }
 
-    /**
-     * @deprecated use {@link HTTP2Client#setFlowControlStrategyFactory(FlowControlStrategy.Factory)} instead
-     */
-    @Deprecated
-    protected FlowControlStrategy newFlowControlStrategy()
-    {
-        return null;
-    }
-
-    /**
-     * @deprecated use {@link HTTP2Client#getInitialSessionRecvWindow()} instead
-     */
-    @Deprecated
-    public int getInitialSessionRecvWindow()
-    {
-        return initialSessionRecvWindow;
-    }
-
-    /**
-     * @deprecated use {@link HTTP2Client#setInitialSessionRecvWindow(int)} instead
-     */
-    @Deprecated
-    public void setInitialSessionRecvWindow(int initialSessionRecvWindow)
-    {
-        this.initialSessionRecvWindow = initialSessionRecvWindow;
-    }
-
     private class HTTP2ClientConnection extends HTTP2Connection implements Callback
     {
         private final HTTP2Client client;
@@ -128,11 +98,7 @@
 
             ISession session = getSession();
 
-            int sessionRecv = client.getInitialSessionRecvWindow();
-            if (sessionRecv == FlowControlStrategy.DEFAULT_WINDOW_SIZE)
-                sessionRecv = initialSessionRecvWindow;
-
-            int windowDelta = sessionRecv - FlowControlStrategy.DEFAULT_WINDOW_SIZE;
+            int windowDelta = client.getInitialSessionRecvWindow() - FlowControlStrategy.DEFAULT_WINDOW_SIZE;
             if (windowDelta > 0)
             {
                 session.updateRecvWindow(windowDelta);
diff --git a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/FlowControlStrategyTest.java b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/FlowControlStrategyTest.java
index eaeaa09..9345ca5 100644
--- a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/FlowControlStrategyTest.java
+++ b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/FlowControlStrategyTest.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Exchanger;
 import java.util.concurrent.TimeUnit;
@@ -64,6 +65,7 @@
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 
@@ -281,11 +283,16 @@
         Stream stream = promise.get(5, TimeUnit.SECONDS);
 
         // Send first chunk that exceeds the window.
-        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(size * 2), false), Callback.NOOP);
+        Callback.Completable completable = new Callback.Completable();
+        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(size * 2), false), completable);
         settingsLatch.await(5, TimeUnit.SECONDS);
 
-        // Send the second chunk of data, must not arrive since we're flow control stalled on the client.
-        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(size * 2), true), Callback.NOOP);
+        completable.thenRun(() ->
+        {
+            // Send the second chunk of data, must not arrive since we're flow control stalled on the client.
+            stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(size * 2), true), Callback.NOOP);
+        });
+
         Assert.assertFalse(dataLatch.await(1, TimeUnit.SECONDS));
 
         // Consume the data arrived to server, this will resume flow control on the client.
@@ -313,10 +320,13 @@
             {
                 MetaData.Response metaData = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
                 HeadersFrame responseFrame = new HeadersFrame(stream.getId(), metaData, null, false);
-                stream.headers(responseFrame, Callback.NOOP);
-
-                DataFrame dataFrame = new DataFrame(stream.getId(), ByteBuffer.allocate(length), true);
-                stream.data(dataFrame, Callback.NOOP);
+                CompletableFuture<Void> completable = new CompletableFuture<>();
+                stream.headers(responseFrame, Callback.from(completable));
+                completable.thenRun(() ->
+                {
+                    DataFrame dataFrame = new DataFrame(stream.getId(), ByteBuffer.allocate(length), true);
+                    stream.data(dataFrame, Callback.NOOP);
+                });
                 return null;
             }
         });
@@ -404,7 +414,7 @@
             public Stream.Listener onNewStream(Stream stream, HeadersFrame requestFrame)
             {
                 MetaData.Response metaData = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
-                HeadersFrame responseFrame = new HeadersFrame(stream.getId(), metaData, null, false);
+                HeadersFrame responseFrame = new HeadersFrame(stream.getId(), metaData, null, true);
                 stream.headers(responseFrame, Callback.NOOP);
                 return new Stream.Listener.Adapter()
                 {
@@ -515,9 +525,13 @@
                     // For every stream, send down half the window size of data.
                     MetaData.Response metaData = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
                     HeadersFrame responseFrame = new HeadersFrame(stream.getId(), metaData, null, false);
-                    stream.headers(responseFrame, Callback.NOOP);
-                    DataFrame dataFrame = new DataFrame(stream.getId(), ByteBuffer.allocate(windowSize / 2), true);
-                    stream.data(dataFrame, Callback.NOOP);
+                    Callback.Completable completable = new Callback.Completable();
+                    stream.headers(responseFrame, completable);
+                    completable.thenRun(() ->
+                    {
+                        DataFrame dataFrame = new DataFrame(stream.getId(), ByteBuffer.allocate(windowSize / 2), true);
+                        stream.data(dataFrame, Callback.NOOP);
+                    });
                     return null;
                 }
             }
@@ -603,9 +617,13 @@
             {
                 MetaData.Response metaData = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
                 HeadersFrame responseFrame = new HeadersFrame(stream.getId(), metaData, null, false);
-                stream.headers(responseFrame, Callback.NOOP);
-                DataFrame dataFrame = new DataFrame(stream.getId(), ByteBuffer.wrap(data), true);
-                stream.data(dataFrame, Callback.NOOP);
+                Callback.Completable completable = new Callback.Completable();
+                stream.headers(responseFrame, completable);
+                completable.thenRun(() ->
+                {
+                    DataFrame dataFrame = new DataFrame(stream.getId(), ByteBuffer.wrap(data), true);
+                    stream.data(dataFrame, Callback.NOOP);
+                });
                 return null;
             }
         });
@@ -635,6 +653,11 @@
         Assert.assertArrayEquals(data, bytes);
     }
 
+    // TODO
+    // Since we changed the API to disallow consecutive data() calls without waiting
+    // for the callback, it is now not possible to have DATA1, DATA2 in the queue for
+    // the same stream. Perhaps this test should just be deleted.
+    @Ignore
     @Test
     public void testServerTwoDataFramesWithStalledStream() throws Exception
     {
@@ -722,7 +745,8 @@
             {
                 MetaData metaData = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
                 HeadersFrame responseFrame = new HeadersFrame(stream.getId(), metaData, null, false);
-                stream.headers(responseFrame, Callback.NOOP);
+                Callback.Completable completable = new Callback.Completable();
+                stream.headers(responseFrame, completable);
                 return new Stream.Listener.Adapter()
                 {
                     @Override
@@ -733,7 +757,8 @@
                         ByteBuffer data = frame.getData();
                         ByteBuffer copy = ByteBuffer.allocateDirect(data.remaining());
                         copy.put(data).flip();
-                        stream.data(new DataFrame(stream.getId(), copy, frame.isEndStream()), callback);
+                        completable.thenRun(() ->
+                                stream.data(new DataFrame(stream.getId(), copy, frame.isEndStream()), callback));
                     }
                 };
             }
@@ -758,9 +783,9 @@
         final ByteBuffer responseContent = ByteBuffer.wrap(responseData);
         MetaData.Request metaData = newRequest("GET", new HttpFields());
         HeadersFrame requestFrame = new HeadersFrame(metaData, null, false);
-        FuturePromise<Stream> streamPromise = new FuturePromise<>();
+        Promise.Completable<Stream> completable = new Promise.Completable<>();
         final CountDownLatch latch = new CountDownLatch(1);
-        session.newStream(requestFrame, streamPromise, new Stream.Listener.Adapter()
+        session.newStream(requestFrame, completable, new Stream.Listener.Adapter()
         {
             @Override
             public void onData(Stream stream, DataFrame frame, Callback callback)
@@ -771,11 +796,12 @@
                     latch.countDown();
             }
         });
-        Stream stream = streamPromise.get(5, TimeUnit.SECONDS);
-
-        ByteBuffer requestContent = ByteBuffer.wrap(requestData);
-        DataFrame dataFrame = new DataFrame(stream.getId(), requestContent, true);
-        stream.data(dataFrame, Callback.NOOP);
+        completable.thenAccept(stream ->
+        {
+            ByteBuffer requestContent = ByteBuffer.wrap(requestData);
+            DataFrame dataFrame = new DataFrame(stream.getId(), requestContent, true);
+            stream.data(dataFrame, Callback.NOOP);
+        });
 
         Assert.assertTrue(latch.await(5, TimeUnit.SECONDS));
 
@@ -803,9 +829,9 @@
         // Consume the whole session and stream window.
         MetaData.Request metaData = newRequest("POST", new HttpFields());
         HeadersFrame requestFrame = new HeadersFrame(metaData, null, false);
-        FuturePromise<Stream> streamPromise = new FuturePromise<>();
-        session.newStream(requestFrame, streamPromise, new Stream.Listener.Adapter());
-        Stream stream = streamPromise.get(5, TimeUnit.SECONDS);
+        CompletableFuture<Stream> completable = new CompletableFuture<>();
+        session.newStream(requestFrame, Promise.from(completable), new Stream.Listener.Adapter());
+        Stream stream = completable.get(5, TimeUnit.SECONDS);
         ByteBuffer data = ByteBuffer.allocate(FlowControlStrategy.DEFAULT_WINDOW_SIZE);
         final CountDownLatch dataLatch = new CountDownLatch(1);
         stream.data(new DataFrame(stream.getId(), data, false), new Callback.NonBlocking()
diff --git a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/HTTP2Test.java b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/HTTP2Test.java
index 6abca17..5ec0177 100644
--- a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/HTTP2Test.java
+++ b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/HTTP2Test.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.channels.WritePendingException;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
@@ -264,7 +265,7 @@
             }
         });
 
-        Thread.sleep(1000);
+        sleep(1000);
 
         server.stop();
 
@@ -286,7 +287,7 @@
 
         newClient(new Session.Listener.Adapter());
 
-        Thread.sleep(1000);
+        sleep(1000);
 
         client.stop();
 
@@ -419,6 +420,173 @@
     }
 
     @Test
+    public void testInvalidAPIUsageOnClient() throws Exception
+    {
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                Callback.Completable completable = new Callback.Completable();
+                MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, new HttpFields());
+                stream.headers(new HeadersFrame(stream.getId(), response, null, false), completable);
+                return new Stream.Listener.Adapter()
+                {
+                    @Override
+                    public void onData(Stream stream, DataFrame frame, Callback callback)
+                    {
+                        callback.succeeded();
+                        if (frame.isEndStream())
+                        {
+                            completable.thenRun(() ->
+                            {
+                                DataFrame endFrame = new DataFrame(stream.getId(), BufferUtil.EMPTY_BUFFER, true);
+                                stream.data(endFrame, Callback.NOOP);
+                            });
+                        }
+                    }
+                };
+            }
+        });
+
+        Session session = newClient(new Session.Listener.Adapter());
+
+        MetaData.Request metaData = newRequest("GET", new HttpFields());
+        HeadersFrame frame = new HeadersFrame(metaData, null, false);
+        Promise.Completable<Stream> completable = new Promise.Completable<>();
+        CountDownLatch completeLatch = new CountDownLatch(2);
+        session.newStream(frame, completable, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onData(Stream stream, DataFrame frame, Callback callback)
+            {
+                callback.succeeded();
+                if (frame.isEndStream())
+                    completeLatch.countDown();
+            }
+        });
+        Stream stream = completable.get(5, TimeUnit.SECONDS);
+
+        long sleep = 1000;
+        DataFrame data1 = new DataFrame(stream.getId(), ByteBuffer.allocate(1024), false)
+        {
+            @Override
+            public ByteBuffer getData()
+            {
+                sleep(2 * sleep);
+                return super.getData();
+            }
+        };
+        DataFrame data2 = new DataFrame(stream.getId(), BufferUtil.EMPTY_BUFFER, true);
+
+        new Thread(() ->
+        {
+            // The first data() call is legal, but slow.
+            stream.data(data1, new Callback()
+            {
+                @Override
+                public void succeeded()
+                {
+                    stream.data(data2, NOOP);
+                }
+            });
+        }).start();
+
+        // Wait for the first data() call to happen.
+        sleep(sleep);
+
+        // This data call is illegal because it does not
+        // wait for the previous callback to complete.
+        stream.data(data2, new Callback()
+        {
+            @Override
+            public void failed(Throwable x)
+            {
+                if (x instanceof WritePendingException)
+                {
+                    // Expected.
+                    completeLatch.countDown();
+                }
+            }
+        });
+
+        Assert.assertTrue(completeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testInvalidAPIUsageOnServer() throws Exception
+    {
+        long sleep = 1000;
+        CountDownLatch completeLatch = new CountDownLatch(2);
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, new HttpFields());
+                DataFrame dataFrame = new DataFrame(stream.getId(), BufferUtil.EMPTY_BUFFER, true);
+                // The call to headers() is legal, but slow.
+                new Thread(() ->
+                {
+                    stream.headers(new HeadersFrame(stream.getId(), response, null, false)
+                    {
+                        @Override
+                        public MetaData getMetaData()
+                        {
+                            sleep(2 * sleep);
+                            return super.getMetaData();
+                        }
+                    }, new Callback()
+                    {
+                        @Override
+                        public void succeeded()
+                        {
+                            stream.data(dataFrame, NOOP);
+                        }
+                    });
+                }).start();
+
+                // Wait for the headers() call to happen.
+                sleep(sleep);
+
+                // This data call is illegal because it does not
+                // wait for the previous callback to complete.
+                stream.data(dataFrame, new Callback()
+                {
+                    @Override
+                    public void failed(Throwable x)
+                    {
+                        if (x instanceof WritePendingException)
+                        {
+                            // Expected.
+                            completeLatch.countDown();
+                        }
+                    }
+                });
+
+                return null;
+            }
+        });
+
+        Session session = newClient(new Session.Listener.Adapter());
+
+        MetaData.Request metaData = newRequest("GET", new HttpFields());
+        HeadersFrame frame = new HeadersFrame(metaData, null, true);
+        session.newStream(frame, new Promise.Adapter<>(), new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onData(Stream stream, DataFrame frame, Callback callback)
+            {
+                callback.succeeded();
+                if (frame.isEndStream())
+                    completeLatch.countDown();
+            }
+        });
+
+        Assert.assertTrue(completeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
     public void testCleanGoAwayDoesNotTriggerFailureNotification() throws Exception
     {
         start(new ServerSessionListener.Adapter()
@@ -459,4 +627,16 @@
         Assert.assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
         Assert.assertFalse(failureLatch.await(1, TimeUnit.SECONDS));
     }
+
+    private static void sleep(long time)
+    {
+        try
+        {
+            Thread.sleep(time);
+        }
+        catch (InterruptedException x)
+        {
+            throw new RuntimeException();
+        }
+    }
 }
diff --git a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/IdleTimeoutTest.java b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/IdleTimeoutTest.java
index ef39cde..4e59758 100644
--- a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/IdleTimeoutTest.java
+++ b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/IdleTimeoutTest.java
@@ -512,12 +512,20 @@
         session.newStream(requestFrame, promise, new Stream.Listener.Adapter());
         final Stream stream = promise.get(5, TimeUnit.SECONDS);
 
+        Callback.Completable completable1 = new Callback.Completable();
         sleep(idleTimeout / 2);
-        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1), false), Callback.NOOP);
-        sleep(idleTimeout / 2);
-        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1), false), Callback.NOOP);
-        sleep(idleTimeout / 2);
-        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1), true), Callback.NOOP);
+        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1), false), completable1);
+        completable1.thenCompose(nil ->
+        {
+            Callback.Completable completable2 = new Callback.Completable();
+            sleep(idleTimeout / 2);
+            stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1), false), completable2);
+            return completable2;
+        }).thenRun(() ->
+        {
+            sleep(idleTimeout / 2);
+            stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1), true), Callback.NOOP);
+        });
 
         Assert.assertFalse(resetLatch.await(0, TimeUnit.SECONDS));
     }
diff --git a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/ProxyProtocolTest.java b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/ProxyProtocolTest.java
index 1ec1566..5a9eb39 100644
--- a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/ProxyProtocolTest.java
+++ b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/ProxyProtocolTest.java
@@ -48,6 +48,7 @@
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.util.FuturePromise;
 import org.eclipse.jetty.util.Promise;
+import org.eclipse.jetty.util.TypeUtil;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Test;
@@ -80,13 +81,25 @@
     }
 
     @Test
-    public void test_PROXY_GET() throws Exception
+    public void test_PROXY_GET_v1() throws Exception
     {
         startServer(new AbstractHandler()
         {
             @Override
             public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
             {
+                try
+                {
+                    Assert.assertEquals("1.2.3.4",request.getRemoteAddr());
+                    Assert.assertEquals(1111,request.getRemotePort());
+                    Assert.assertEquals("5.6.7.8",request.getLocalAddr());
+                    Assert.assertEquals(2222,request.getLocalPort());
+                }
+                catch(Throwable th)
+                {
+                    th.printStackTrace();
+                    response.setStatus(500);
+                }
                 baseRequest.setHandled(true);
             }
         });
@@ -118,4 +131,56 @@
         });
         Assert.assertTrue(latch.await(5, TimeUnit.SECONDS));
     }
+    
+    @Test
+    public void test_PROXY_GET_v2() throws Exception
+    {
+        startServer(new AbstractHandler()
+        {
+            @Override
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                try
+                {
+                    Assert.assertEquals("10.0.0.4",request.getRemoteAddr());
+                    Assert.assertEquals(33824,request.getRemotePort());
+                    Assert.assertEquals("10.0.0.4",request.getLocalAddr());
+                    Assert.assertEquals(8888,request.getLocalPort());
+                }
+                catch(Throwable th)
+                {
+                    th.printStackTrace();
+                    response.setStatus(500);
+                }
+                baseRequest.setHandled(true);
+            }
+        });
+
+        String request1 = "0D0A0D0A000D0A515549540A211100140A0000040A000004842022B82000050000000000";
+        SocketChannel channel = SocketChannel.open();
+        channel.connect(new InetSocketAddress("localhost", connector.getLocalPort()));
+        channel.write(ByteBuffer.wrap(TypeUtil.fromHexString(request1)));
+
+        FuturePromise<Session> promise = new FuturePromise<>();
+        client.accept(null, channel, new Session.Listener.Adapter(), promise);
+        Session session = promise.get(5, TimeUnit.SECONDS);
+
+        HttpFields fields = new HttpFields();
+        String uri = "http://localhost:" + connector.getLocalPort() + "/";
+        MetaData.Request metaData = new MetaData.Request("GET", new HttpURI(uri), HttpVersion.HTTP_2, fields);
+        HeadersFrame frame = new HeadersFrame(metaData, null, true);
+        CountDownLatch latch = new CountDownLatch(1);
+        session.newStream(frame, new Promise.Adapter<>(), new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onHeaders(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = (MetaData.Response)frame.getMetaData();
+                Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+                if (frame.isEndStream())
+                    latch.countDown();
+            }
+        });
+        Assert.assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
 }
diff --git a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamCloseTest.java b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamCloseTest.java
index 72a6392..ce1b3ef 100644
--- a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamCloseTest.java
+++ b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamCloseTest.java
@@ -122,24 +122,26 @@
             {
                 MetaData.Response metaData = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
                 HeadersFrame response = new HeadersFrame(stream.getId(), metaData, null, false);
-                stream.headers(response, Callback.NOOP);
+                Callback.Completable completable = new Callback.Completable();
+                stream.headers(response, completable);
                 return new Stream.Listener.Adapter()
                 {
                     @Override
                     public void onData(final Stream stream, DataFrame frame, final Callback callback)
                     {
                         Assert.assertTrue(((HTTP2Stream)stream).isRemotelyClosed());
-                        stream.data(frame, new Callback()
-                        {
-                            @Override
-                            public void succeeded()
-                            {
-                                Assert.assertTrue(stream.isClosed());
-                                Assert.assertEquals(0, stream.getSession().getStreams().size());
-                                callback.succeeded();
-                                serverDataLatch.countDown();
-                            }
-                        });
+                        completable.thenRun(() ->
+                                stream.data(frame, new Callback()
+                                {
+                                    @Override
+                                    public void succeeded()
+                                    {
+                                        Assert.assertTrue(stream.isClosed());
+                                        Assert.assertEquals(0, stream.getSession().getStreams().size());
+                                        callback.succeeded();
+                                        serverDataLatch.countDown();
+                                    }
+                                }));
                     }
                 };
             }
diff --git a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamResetTest.java b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamResetTest.java
index ba19238..a6b7446 100644
--- a/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamResetTest.java
+++ b/jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamResetTest.java
@@ -37,6 +37,7 @@
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.http2.ErrorCode;
+import org.eclipse.jetty.http2.IStream;
 import org.eclipse.jetty.http2.api.Session;
 import org.eclipse.jetty.http2.api.Stream;
 import org.eclipse.jetty.http2.api.server.ServerSessionListener;
@@ -126,29 +127,38 @@
             {
                 MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, 200, new HttpFields());
                 HeadersFrame responseFrame = new HeadersFrame(stream.getId(), response, null, false);
-                stream.headers(responseFrame, Callback.NOOP);
+                Callback.Completable completable = new Callback.Completable();
+                stream.headers(responseFrame, completable);
                 return new Stream.Listener.Adapter()
                 {
                     @Override
                     public void onData(Stream stream, DataFrame frame, Callback callback)
                     {
                         callback.succeeded();
-                        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(16), true), Callback.NOOP);
-                        serverDataLatch.countDown();
+                        completable.thenRun(() ->
+                                stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(16), true), new Callback()
+                                {
+                                    @Override
+                                    public void succeeded()
+                                    {
+                                        serverDataLatch.countDown();
+                                    }
+                                }));
                     }
 
                     @Override
-                    public void onReset(Stream stream, ResetFrame frame)
+                    public void onReset(Stream s, ResetFrame frame)
                     {
                         // Simulate that there is pending data to send.
-                        stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(16), true), new Callback()
+                        IStream stream = (IStream)s;
+                        stream.getSession().frames(stream, new Callback()
                         {
                             @Override
                             public void failed(Throwable x)
                             {
                                 serverResetLatch.countDown();
                             }
-                        });
+                        }, new DataFrame(s.getId(), ByteBuffer.allocate(16), true));
                     }
                 };
             }
diff --git a/jetty-http2/http2-common/pom.xml b/jetty-http2/http2-common/pom.xml
index ff849ac..67f009d 100644
--- a/jetty-http2/http2-common/pom.xml
+++ b/jetty-http2/http2-common/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty.http2</groupId>
     <artifactId>http2-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java b/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java
index 54b087a..f33694f 100644
--- a/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java
+++ b/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java
@@ -1213,9 +1213,10 @@
             if (dataFrame.remaining() > 0)
             {
                 // We have written part of the frame, but there is more to write.
-                // We need to keep the correct ordering of frames, to avoid that other
-                // frames for the same stream are written before this one is finished.
-                flusher.prepend(this);
+                // The API will not allow to send two data frames for the same
+                // stream so we append the unfinished frame at the end to allow
+                // better interleaving with other streams.
+                flusher.append(this);
             }
             else
             {
diff --git a/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Stream.java b/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Stream.java
index 6a9b096..e6a9c4a 100644
--- a/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Stream.java
+++ b/jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Stream.java
@@ -20,6 +20,7 @@
 
 import java.io.EOFException;
 import java.io.IOException;
+import java.nio.channels.WritePendingException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeoutException;
@@ -39,12 +40,13 @@
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.Scheduler;
 
-public class HTTP2Stream extends IdleTimeout implements IStream
+public class HTTP2Stream extends IdleTimeout implements IStream, Callback
 {
     private static final Logger LOG = Log.getLogger(HTTP2Stream.class);
 
     private final AtomicReference<ConcurrentMap<String, Object>> attributes = new AtomicReference<>();
     private final AtomicReference<CloseState> closeState = new AtomicReference<>(CloseState.NOT_CLOSED);
+    private final AtomicReference<Callback> writing = new AtomicReference<>();
     private final AtomicInteger sendWindow = new AtomicInteger();
     private final AtomicInteger recvWindow = new AtomicInteger();
     private final ISession session;
@@ -83,8 +85,10 @@
     @Override
     public void headers(HeadersFrame frame, Callback callback)
     {
+        if (!checkWrite(callback))
+            return;
         notIdle();
-        session.frames(this, callback, frame, Frame.EMPTY_ARRAY);
+        session.frames(this, this, frame, Frame.EMPTY_ARRAY);
     }
 
     @Override
@@ -97,8 +101,10 @@
     @Override
     public void data(DataFrame frame, Callback callback)
     {
+        if (!checkWrite(callback))
+            return;
         notIdle();
-        session.data(this, callback, frame);
+        session.data(this, this, frame);
     }
 
     @Override
@@ -111,6 +117,14 @@
         session.frames(this, callback, frame, Frame.EMPTY_ARRAY);
     }
 
+    private boolean checkWrite(Callback callback)
+    {
+        if (writing.compareAndSet(null, callback))
+            return true;
+        callback.failed(new WritePendingException());
+        return false;
+    }
+
     @Override
     public Object getAttribute(String key)
     {
@@ -360,6 +374,20 @@
         onClose();
     }
 
+    @Override
+    public void succeeded()
+    {
+        Callback callback = writing.getAndSet(null);
+        callback.succeeded();
+    }
+
+    @Override
+    public void failed(Throwable x)
+    {
+        Callback callback = writing.getAndSet(null);
+        callback.failed(x);
+    }
+
     private void notifyData(Stream stream, DataFrame frame, Callback callback)
     {
         final Listener listener = this.listener;
diff --git a/jetty-http2/http2-hpack/pom.xml b/jetty-http2/http2-hpack/pom.xml
index 703caac..407e2db 100644
--- a/jetty-http2/http2-hpack/pom.xml
+++ b/jetty-http2/http2-hpack/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty.http2</groupId>
     <artifactId>http2-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http2/http2-http-client-transport/pom.xml b/jetty-http2/http2-http-client-transport/pom.xml
index d2c783b..e5aab8a 100644
--- a/jetty-http2/http2-http-client-transport/pom.xml
+++ b/jetty-http2/http2-http-client-transport/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.eclipse.jetty.http2</groupId>
         <artifactId>http2-parent</artifactId>
-        <version>9.3.8-SNAPSHOT</version>
+        <version>9.4.0-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpDestinationOverHTTP2.java b/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpDestinationOverHTTP2.java
index 8355369..5c5b6a8 100644
--- a/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpDestinationOverHTTP2.java
+++ b/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpDestinationOverHTTP2.java
@@ -23,8 +23,9 @@
 import org.eclipse.jetty.client.MultiplexHttpDestination;
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.SendFailure;
+import org.eclipse.jetty.client.api.Connection;
 
-public class HttpDestinationOverHTTP2 extends MultiplexHttpDestination<HttpConnectionOverHTTP2>
+public class HttpDestinationOverHTTP2 extends MultiplexHttpDestination
 {
     public HttpDestinationOverHTTP2(HttpClient client, Origin origin)
     {
@@ -32,8 +33,8 @@
     }
 
     @Override
-    protected SendFailure send(HttpConnectionOverHTTP2 connection, HttpExchange exchange)
+    protected SendFailure send(Connection connection, HttpExchange exchange)
     {
-        return connection.send(exchange);
+        return ((HttpConnectionOverHTTP2)connection).send(exchange);
     }
 }
diff --git a/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/MaxConcurrentStreamsTest.java b/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/MaxConcurrentStreamsTest.java
index c64ed4f..c456d02 100644
--- a/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/MaxConcurrentStreamsTest.java
+++ b/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/MaxConcurrentStreamsTest.java
@@ -63,6 +63,7 @@
                 sleep(sleep);
             }
         });
+        client.setMaxConnectionsPerDestination(1);
 
         // Prime the connection so that the maxConcurrentStream setting arrives to the client.
         client.newRequest("localhost", connector.getLocalPort()).path("/prime").send();
@@ -105,6 +106,7 @@
                 sleep(sleep);
             }
         });
+        client.setMaxConnectionsPerDestination(1);
 
         // Prime the connection so that the maxConcurrentStream setting arrives to the client.
         client.newRequest("localhost", connector.getLocalPort()).path("/prime").send();
@@ -149,6 +151,7 @@
                 sleep(sleep);
             }
         });
+        client.setMaxConnectionsPerDestination(1);
 
         // Prime the connection so that the maxConcurrentStream setting arrives to the client.
         client.newRequest("localhost", connector.getLocalPort()).path("/prime").send();
@@ -165,6 +168,35 @@
         Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
     }
 
+    @Test
+    public void testMultipleRequestsQueuedOnConnect() throws Exception
+    {
+        int maxConcurrent = 10;
+        long sleep = 500;
+        start(maxConcurrent, new AbstractHandler()
+        {
+            @Override
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                baseRequest.setHandled(true);
+                sleep(sleep);
+            }
+        });
+        client.setMaxConnectionsPerDestination(1);
+
+        // The first request will open the connection, the others will be queued.
+        CountDownLatch latch = new CountDownLatch(maxConcurrent);
+        for (int i = 0; i < maxConcurrent; ++i)
+        {
+            client.newRequest("localhost", connector.getLocalPort())
+                    .path("/" + i)
+                    .send(result -> latch.countDown());
+        }
+
+        // The requests should be processed in parallel, not sequentially.
+        Assert.assertTrue(latch.await(maxConcurrent * sleep / 2, TimeUnit.MILLISECONDS));
+    }
+
     private void sleep(long time)
     {
         try
diff --git a/jetty-http2/http2-server/pom.xml b/jetty-http2/http2-server/pom.xml
index 69e90ec..78e4dc3 100644
--- a/jetty-http2/http2-server/pom.xml
+++ b/jetty-http2/http2-server/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.eclipse.jetty.http2</groupId>
     <artifactId>http2-parent</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-http2/http2-server/src/main/config/modules/http2.mod b/jetty-http2/http2-server/src/main/config/modules/http2.mod
index 585c1fa..ece1e33 100644
--- a/jetty-http2/http2-server/src/main/config/modules/http2.mod
+++ b/jetty-http2/http2-server/src/main/config/modules/http2.mod
@@ -1,6 +1,6 @@
-#
-# HTTP2 Support Module
-#
+[description]
+Enables HTTP2 protocol support on the TLS(SSL) Connector,
+using the ALPN extension to select which protocol to use.
 
 [depend]
 ssl
diff --git a/jetty-http2/http2-server/src/main/config/modules/http2c.mod b/jetty-http2/http2-server/src/main/config/modules/http2c.mod
index 1c78016..dfca925 100644
--- a/jetty-http2/http2-server/src/main/config/modules/http2c.mod
+++ b/jetty-http2/http2-server/src/main/config/modules/http2c.mod
@@ -1,9 +1,6 @@
-#
-# HTTP2 Clear Text Support Module
-# This module adds support for HTTP/2 clear text to the
-# HTTP/1 clear text connector (defined in jetty-http.xml).
-# The resulting connector will accept both HTTP/1 and HTTP/2 connections.
-#
+[description]
+Enables the HTTP2C protocol on the HTTP Connector
+The connector will accept both HTTP/1 and HTTP/2 connections.
 
 [depend]
 http
diff --git a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java
index 8b6debe..6bb9112 100644
--- a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java
+++ b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java
@@ -123,9 +123,7 @@
         ServerSessionListener listener = newSessionListener(connector, endPoint);
 
         Generator generator = new Generator(connector.getByteBufferPool(), getMaxDynamicTableSize(), getMaxHeaderBlockFragment());
-        FlowControlStrategy flowControl = newFlowControlStrategy();
-        if (flowControl == null)
-            flowControl = getFlowControlStrategyFactory().newFlowControlStrategy();
+        FlowControlStrategy flowControl = getFlowControlStrategyFactory().newFlowControlStrategy();
         HTTP2ServerSession session = new HTTP2ServerSession(connector.getScheduler(), endPoint, generator, listener, flowControl);
         session.setMaxLocalStreams(getMaxConcurrentStreams());
         session.setMaxRemoteStreams(getMaxConcurrentStreams());
@@ -142,15 +140,6 @@
         return configure(connection, connector, endPoint);
     }
 
-    /**
-     * @deprecated use {@link #setFlowControlStrategyFactory(FlowControlStrategy.Factory)} instead
-     */
-    @Deprecated
-    protected FlowControlStrategy newFlowControlStrategy()
-    {
-        return null;
-    }
-
     protected abstract ServerSessionListener newSessionListener(Connector connector, EndPoint endPoint);
 
     protected ServerParser newServerParser(Connector connector, ServerParser.Listener listener)
diff --git a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java
index 102734b..d124252 100644
--- a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java
+++ b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java
@@ -44,7 +44,6 @@
     private static final Logger LOG = Log.getLogger(HttpTransportOverHTTP2.class);
 
     private final AtomicBoolean commit = new AtomicBoolean();
-    private final Callback commitCallback = new CommitCallback();
     private final Connector connector;
     private final HTTP2ServerConnection connection;
     private IStream stream;
@@ -62,7 +61,7 @@
         // copying we can defer to the endpoint
         return connection.getEndPoint().isOptimizedForDirectBuffers();
     }
-    
+
     public IStream getStream()
     {
         return stream;
@@ -101,8 +100,24 @@
             {
                 if (hasContent)
                 {
-                    commit(info, false, commitCallback);
-                    send(content, lastContent, callback);
+                    commit(info, false, new Callback()
+                    {
+                        @Override
+                        public void succeeded()
+                        {
+                            if (LOG.isDebugEnabled())
+                                LOG.debug("HTTP2 Response #{} committed", stream.getId());
+                            send(content, lastContent, callback);
+                        }
+
+                        @Override
+                        public void failed(Throwable x)
+                        {
+                            if (LOG.isDebugEnabled())
+                                LOG.debug("HTTP2 Response #" + stream.getId() + " failed to commit", x);
+                            callback.failed(x);
+                        }
+                    });
                 }
                 else
                 {
@@ -145,7 +160,7 @@
 
         if (LOG.isDebugEnabled())
             LOG.debug("HTTP/2 Push {}",request);
-        
+
         stream.push(new PushPromiseFrame(stream.getId(), 0, request), new Promise<Stream>()
         {
             @Override
@@ -211,21 +226,4 @@
         if (stream != null)
             stream.reset(new ResetFrame(stream.getId(), ErrorCode.INTERNAL_ERROR.code), Callback.NOOP);
     }
-
-    private class CommitCallback implements Callback.NonBlocking
-    {   
-        @Override
-        public void succeeded()
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("HTTP2 Response #{} committed", stream.getId());
-        }
-
-        @Override
-        public void failed(Throwable x)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("HTTP2 Response #" + stream.getId() + " failed to commit", x);
-        }
-    }
 }
diff --git a/jetty-http2/http2-server/src/test/java/org/eclipse/jetty/http2/server/HTTP2ServerTest.java b/jetty-http2/http2-server/src/test/java/org/eclipse/jetty/http2/server/HTTP2ServerTest.java
index 28a04a5..fdeb0b2 100644
--- a/jetty-http2/http2-server/src/test/java/org/eclipse/jetty/http2/server/HTTP2ServerTest.java
+++ b/jetty-http2/http2-server/src/test/java/org/eclipse/jetty/http2/server/HTTP2ServerTest.java
@@ -57,6 +57,7 @@
 import org.eclipse.jetty.http2.generator.Generator;
 import org.eclipse.jetty.http2.parser.Parser;
 import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.SelectChannelEndPoint;
 import org.eclipse.jetty.server.HttpChannel;
@@ -329,7 +330,7 @@
         ServerConnector connector2 = new ServerConnector(server, new HTTP2ServerConnectionFactory(new HttpConfiguration()))
         {
             @Override
-            protected SelectChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+            protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
             {
                 return new SelectChannelEndPoint(channel, selectSet, key, getScheduler(), getIdleTimeout())
                 {
diff --git a/jetty-http2/pom.xml b/jetty-http2/pom.xml
index 71c050c..f6ed2ff 100644
--- a/jetty-http2/pom.xml
+++ b/jetty-http2/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>jetty-project</artifactId>
     <groupId>org.eclipse.jetty</groupId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
diff --git a/jetty-infinispan/pom.xml b/jetty-infinispan/pom.xml
index 28d8249..37c7f33 100644
--- a/jetty-infinispan/pom.xml
+++ b/jetty-infinispan/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-infinispan</artifactId>
diff --git a/jetty-infinispan/src/main/config/modules/infinispan.mod b/jetty-infinispan/src/main/config/modules/infinispan.mod
index afa39fc..9a1e0b2 100644
--- a/jetty-infinispan/src/main/config/modules/infinispan.mod
+++ b/jetty-infinispan/src/main/config/modules/infinispan.mod
@@ -1,6 +1,6 @@
-#
-# Jetty Infinispan module
-#
+[description]
+Enables an Infinispan Session Manager for session 
+persistance and/or clustering
 
 [depend]
 annotations
diff --git a/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionDataStore.java b/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionDataStore.java
new file mode 100644
index 0000000..61e8279
--- /dev/null
+++ b/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionDataStore.java
@@ -0,0 +1,201 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2015 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+
+package org.eclipse.jetty.session.infinispan;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.server.SessionIdManager;
+import org.eclipse.jetty.server.session.AbstractSessionDataStore;
+import org.eclipse.jetty.server.session.SessionContext;
+import org.eclipse.jetty.server.session.SessionData;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+import org.infinispan.commons.api.BasicCache;
+
+/**
+ * InfinispanSessionDataStore
+ *
+ *
+ */
+public class InfinispanSessionDataStore extends AbstractSessionDataStore
+{
+    private  final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
+
+    public static final int DEFAULT_IDLE_EXPIRY_MULTIPLE = 2;
+
+
+    /**
+     * Clustered cache of sessions
+     */
+    private BasicCache<String, Object> _cache;
+
+    private SessionIdManager _idMgr = null;
+    
+    
+    private int _idleExpiryMultiple = DEFAULT_IDLE_EXPIRY_MULTIPLE;
+    
+
+    /**
+     * Get the clustered cache instance.
+     * 
+     * @return
+     */
+    public BasicCache<String, Object> getCache() 
+    {
+        return _cache;
+    }
+
+    
+    
+    /**
+     * Set the clustered cache instance.
+     * 
+     * @param cache
+     */
+    public void setCache (BasicCache<String, Object> cache) 
+    {
+        this._cache = cache;
+    }
+
+    public void setSessionIdManager (SessionIdManager idMgr)
+    {
+        _idMgr = idMgr;
+    }
+    
+    
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#load(org.eclipse.jetty.server.session.SessionKey)
+     */
+    @Override
+    public SessionData load(String id) throws Exception
+    {  
+        final AtomicReference<SessionData> reference = new AtomicReference<SessionData>();
+        final AtomicReference<Exception> exception = new AtomicReference<Exception>();
+        
+        Runnable load = new Runnable()
+        {
+            public void run ()
+            {
+                try
+                {
+
+                    SessionData sd = (SessionData)_cache.get(getCacheKey(id, _context));
+                    reference.set(sd);
+                }
+                catch (Exception e)
+                {
+                    exception.set(e);
+                }
+            }
+        };
+        
+        //ensure the load runs in the context classloader scope
+        _context.run(load);
+        
+        if (exception.get() != null)
+            throw exception.get();
+        
+        return reference.get();
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#delete(org.eclipse.jetty.server.session.SessionKey)
+     */
+    @Override
+    public boolean delete(String id) throws Exception
+    {
+        return (_cache.remove(getCacheKey(id, _context)) != null);
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#getExpired(java.util.Set)
+     */
+    @Override
+    public Set<String> getExpired(Set<String> candidates)
+    {
+       if (candidates == null  || candidates.isEmpty())
+           return candidates;
+       
+       long now = System.currentTimeMillis();
+       
+       Set<String> expired = new HashSet<String>();
+       if (LOG.isDebugEnabled())
+           LOG.debug("Getting expired sessions " + now);
+       
+       for (String candidate:candidates)
+       {
+           try
+           {
+               SessionData sd = load(candidate);
+               if (sd == null || sd.isExpiredAt(now))
+                   expired.add(candidate);
+                   
+           }
+           catch (Exception e)
+           {
+               LOG.warn("Error checking if session {} is expired", candidate, e);
+           }
+       }
+       
+       return expired;
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.AbstractSessionDataStore#doStore(org.eclipse.jetty.server.session.SessionKey, org.eclipse.jetty.server.session.SessionData, boolean)
+     */
+    @Override
+    public void doStore(String id, SessionData data, boolean isNew) throws Exception
+    {
+        //Put an idle timeout on the cache entry if the session is not immortal - 
+        //if no requests arrive at any node before this timeout occurs, or no node 
+        //scavenges the session before this timeout occurs, the session will be removed.
+        //NOTE: that no session listeners can be called for this.
+        if (data.getMaxInactiveMs() > 0)
+            _cache.put(getCacheKey(id, _context), data, -1, TimeUnit.MILLISECONDS, (data.getMaxInactiveMs() * _idleExpiryMultiple), TimeUnit.MILLISECONDS);
+        else
+            _cache.put(getCacheKey(id, _context), data);
+        
+        //tickle the session id manager to keep the sessionid entry for this session up-to-date
+        if (_idMgr != null && _idMgr instanceof InfinispanSessionIdManager)
+        {
+            ((InfinispanSessionIdManager)_idMgr).touch(id);
+        }
+    }
+    
+    
+    public static String getCacheKey (String id, SessionContext context)
+    {
+        return context.getCanonicalContextPath()+"_"+context.getVhost()+"_"+id;
+    }
+
+
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#isPassivating()
+     */
+    @Override
+    public boolean isPassivating()
+    {
+        return true;
+    }
+}
diff --git a/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionIdManager.java b/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionIdManager.java
index 4dfd6c0..2ef60da 100644
--- a/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionIdManager.java
+++ b/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionIdManager.java
@@ -28,8 +28,8 @@
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.SessionManager;
 import org.eclipse.jetty.server.handler.ContextHandler;
-import org.eclipse.jetty.server.session.AbstractSession;
 import org.eclipse.jetty.server.session.AbstractSessionIdManager;
+import org.eclipse.jetty.server.session.Session;
 import org.eclipse.jetty.server.session.SessionHandler;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
@@ -70,7 +70,6 @@
     public final static String ID_KEY = "__o.e.j.s.infinispanIdMgr__";
     public static final int DEFAULT_IDLE_EXPIRY_MULTIPLE = 2;
     protected BasicCache<String,Object> _cache;
-    private Server _server;
     private int _idleExpiryMultiple = DEFAULT_IDLE_EXPIRY_MULTIPLE;
 
     
@@ -79,14 +78,12 @@
     
     public InfinispanSessionIdManager(Server server)
     {
-        super();
-        _server = server;
+        super(server);
     }
 
     public InfinispanSessionIdManager(Server server, Random random)
     {
-       super(random);
-       _server = server;
+       super(server, random);
     }
 
     
@@ -123,15 +120,15 @@
      * 
      * This method will consult the cluster.
      * 
-     * @see org.eclipse.jetty.server.SessionIdManager#idInUse(java.lang.String)
+     * @see org.eclipse.jetty.server.SessionIdManager#isIdInUse(java.lang.String)
      */
     @Override
-    public boolean idInUse(String id)
+    public boolean isIdInUse(String id)
     {
         if (id == null)
             return false;
         
-        String clusterId = getClusterId(id);
+        String clusterId = getId(id);
         
         //ask the cluster - this should also tickle the idle expiration timer on the sessionid entry
         //keeping it valid
@@ -147,29 +144,6 @@
         
     }
 
-    /** 
-     * Remember a new in-use session id.
-     * 
-     * This will save the in-use session id to the cluster.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#addSession(javax.servlet.http.HttpSession)
-     */
-    @Override
-    public void addSession(HttpSession session)
-    {
-        if (session == null)
-            return;
-
-        //insert into the cache and set an idle expiry on the entry that
-        //is based off the max idle time configured for the session. If the
-        //session is immortal, then there is no idle expiry on the corresponding
-        //session id
-        if (session.getMaxInactiveInterval() == 0)
-            insert (((AbstractSession)session).getClusterId());
-        else
-            insert (((AbstractSession)session).getClusterId(), session.getMaxInactiveInterval() * getIdleExpiryMultiple());
-    }
-    
     
     public void setIdleExpiryMultiple (int multiplier)
     {
@@ -185,90 +159,8 @@
         return _idleExpiryMultiple;
     }
     
-    
-    /** 
-     * Remove a session id from the list of in-use ids.
-     * 
-     * This will remvove the corresponding session id from the cluster.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#removeSession(javax.servlet.http.HttpSession)
-     */
-    @Override
-    public void removeSession(HttpSession session)
-    {
-        if (session == null)
-            return;
+   
 
-        //delete from the cache
-        delete (((AbstractSession)session).getClusterId());
-    }
-
-    /** 
-     * Remove a session id. This compels all other contexts who have a session
-     * with the same id to also remove it.
-     * 
-     * @see org.eclipse.jetty.server.SessionIdManager#invalidateAll(java.lang.String)
-     */
-    @Override
-    public void invalidateAll(String id)
-    {
-        //delete the session id from list of in-use sessions
-        delete (id);
-
-
-        //tell all contexts that may have a session object with this id to
-        //get rid of them
-        Handler[] contexts = _server.getChildHandlersByClass(ContextHandler.class);
-        for (int i=0; contexts!=null && i<contexts.length; i++)
-        {
-            SessionHandler sessionHandler = ((ContextHandler)contexts[i]).getChildHandlerByClass(SessionHandler.class);
-            if (sessionHandler != null)
-            {
-                SessionManager manager = sessionHandler.getSessionManager();
-
-                if (manager != null && manager instanceof InfinispanSessionManager)
-                {
-                    ((InfinispanSessionManager)manager).invalidateSession(id);
-                }
-            }
-        }
-
-    }
-
-    /** 
-     * Change a session id. 
-     * 
-     * Typically this occurs when a previously existing session has passed through authentication.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionIdManager#renewSessionId(java.lang.String, java.lang.String, javax.servlet.http.HttpServletRequest)
-     */
-    @Override
-    public void renewSessionId(String oldClusterId, String oldNodeId, HttpServletRequest request)
-    {
-        //generate a new id
-        String newClusterId = newSessionId(request.hashCode());
-
-        delete(oldClusterId);
-        insert(newClusterId);
-
-
-        //tell all contexts to update the id 
-        Handler[] contexts = _server.getChildHandlersByClass(ContextHandler.class);
-        for (int i=0; contexts!=null && i<contexts.length; i++)
-        {
-            SessionHandler sessionHandler = ((ContextHandler)contexts[i]).getChildHandlerByClass(SessionHandler.class);
-            if (sessionHandler != null) 
-            {
-                SessionManager manager = sessionHandler.getSessionManager();
-
-                if (manager != null && manager instanceof InfinispanSessionManager)
-                {
-                    ((InfinispanSessionManager)manager).renewSessionId(oldClusterId, oldNodeId, newClusterId, getNodeId(newClusterId, request));
-                }
-            }
-        }
-
-    }
 
     /**
      * Get the cache.
@@ -351,12 +243,12 @@
      * 
      * @param id the session id
      */
-    protected void delete (String id)
+    protected boolean delete (String id)
     {
         if (_cache == null)
             throw new IllegalStateException ("No cache");
         
-        _cache.remove(makeKey(id));
+        return _cache.remove(makeKey(id)) != null;
     }
     
     
@@ -371,4 +263,28 @@
     {
         return ID_KEY+id;
     }
+
+    /** 
+     * @see org.eclipse.jetty.server.SessionIdManager#useId(java.lang.String)
+     */
+    @Override
+    public void useId(Session session)
+    {
+        if (session == null)
+            return;
+        
+      if (session.getMaxInactiveInterval() == 0)
+            insert (session.getId());
+        else
+            insert (session.getId(), session.getMaxInactiveInterval() * getIdleExpiryMultiple());
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.SessionIdManager#removeId(java.lang.String)
+     */
+    @Override
+    public boolean removeId(String id)
+    {
+       return delete (id);        
+    }
 }
diff --git a/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionManager.java b/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionManager.java
index ef635fe..4790096 100644
--- a/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionManager.java
+++ b/jetty-infinispan/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionManager.java
@@ -18,573 +18,35 @@
 
 package org.eclipse.jetty.session.infinispan;
 
-import java.io.IOException;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
-
-import javax.servlet.http.HttpServletRequest;
-
-import org.eclipse.jetty.server.handler.ContextHandler;
-import org.eclipse.jetty.server.handler.ContextHandler.Context;
-import org.eclipse.jetty.server.session.AbstractSession;
-import org.eclipse.jetty.server.session.AbstractSessionManager;
-import org.eclipse.jetty.server.session.MemSession;
+import org.eclipse.jetty.server.session.AbstractSessionStore;
+import org.eclipse.jetty.server.session.MemorySessionStore;
+import org.eclipse.jetty.server.session.SessionManager;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-import org.eclipse.jetty.util.thread.Scheduler;
-import org.infinispan.Cache;
-import org.infinispan.commons.api.BasicCache;
-import org.omg.CORBA._IDLTypeStub;
+
+
 
 /**
  * InfinispanSessionManager
  * 
- * The data for a session relevant to a particular context is stored in an Infinispan (clustered) cache:
- * <pre>
- * Key:   is the id of the session + the context path + the vhost for the context 
- * Value: is the data of the session
- * </pre>
- * 
- * The key is necessarily complex because the same session id can be in-use by more than one
- * context. In this case, the contents of the session will strictly be different for each
- * context, although the id will be the same.
- * 
- * Sessions are also kept in local memory when they are used by this session manager. This allows
- * multiple different request threads in the same context to call Request.getSession() and
- * obtain the same object.
- * 
- * This session manager support scavenging, which is only done over the set of sessions in its
- * local memory. This can result in some sessions being "stranded" in the cluster cache if no
- * session manager is currently managing it (eg the node managing the session crashed and it
- * was never requested on another node).
+ * Convenience class to create a MemorySessionStore and an InfinispanSessionDataStore.
  * 
  */
-public class InfinispanSessionManager extends AbstractSessionManager
+public class InfinispanSessionManager extends SessionManager
 {
     private  final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
     
-    /**
-     * Clustered cache of sessions
-     */
-    private BasicCache<String, Object> _cache;
-    
-    
-    /**
-     * Sessions known to this node held in memory
-     */
-    private ConcurrentHashMap<String, InfinispanSessionManager.Session> _sessions;
 
-    
-    /**
-     * The length of time a session can be in memory without being checked against
-     * the cluster. A value of 0 indicates that the session is never checked against
-     * the cluster - the current node is considered to be the master for the session.
-     *
-     */
-    private long _staleIntervalSec = 0;
-    
-    protected Scheduler.Task _task; //scavenge task
-    protected Scheduler _scheduler;
-    protected Scavenger _scavenger;
-    protected long _scavengeIntervalMs = 1000L * 60 * 10; //10mins
-    protected boolean _ownScheduler;
-    
-    
+    protected InfinispanSessionDataStore _sessionDataStore;
 
-    /**
-     * Scavenger
-     *
-     */
-    protected class Scavenger implements Runnable
+
+    public InfinispanSessionManager()
     {
-
-        @Override
-        public void run()
-        {
-           try
-           {
-               scavenge();
-           }
-           finally
-           {
-               if (_scheduler != null && _scheduler.isRunning())
-                   _task = _scheduler.schedule(this, _scavengeIntervalMs, TimeUnit.MILLISECONDS);
-           }
-        }
+        _sessionStore = new MemorySessionStore();
+        _sessionDataStore = new InfinispanSessionDataStore();
     }
     
     
-    /*
-     * Every time a Session is put into the cache one of these objects
-     * is created to copy the data out of the in-memory session, and 
-     * every time an object is read from the cache one of these objects
-     * a fresh Session object is created based on the data held by this
-     * object.
-     */
-    public class SerializableSessionData implements Serializable
-    {
-        /**
-         * 
-         */
-        private static final long serialVersionUID = -7779120106058533486L;
-        String clusterId;
-        String contextPath;
-        String vhost;
-        long accessed;
-        long lastAccessed;
-        long createTime;
-        long cookieSetTime;
-        String lastNode;
-        long expiry;
-        long maxInactive;
-        Map<String, Object> attributes;
-
-        public SerializableSessionData()
-        {
-
-        }
-
-       
-       public SerializableSessionData(Session s)
-       {
-           clusterId = s.getClusterId();
-           contextPath = s.getContextPath();
-           vhost = s.getVHost();
-           accessed = s.getAccessed();
-           lastAccessed = s.getLastAccessedTime();
-           createTime = s.getCreationTime();
-           cookieSetTime = s.getCookieSetTime();
-           lastNode = s.getLastNode();
-           expiry = s.getExpiry();
-           maxInactive = s.getMaxInactiveInterval();
-           attributes = s.getAttributeMap(); // TODO pointer, not a copy
-       }
-        
-        private void writeObject(java.io.ObjectOutputStream out) throws IOException
-        {  
-            out.writeUTF(clusterId); //session id
-            out.writeUTF(contextPath); //context path
-            out.writeUTF(vhost); //first vhost
-
-            out.writeLong(accessed);//accessTime
-            out.writeLong(lastAccessed); //lastAccessTime
-            out.writeLong(createTime); //time created
-            out.writeLong(cookieSetTime);//time cookie was set
-            out.writeUTF(lastNode); //name of last node managing
-      
-            out.writeLong(expiry); 
-            out.writeLong(maxInactive);
-            out.writeObject(attributes);
-        }
-        
-        private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException
-        {
-            clusterId = in.readUTF();
-            contextPath = in.readUTF();
-            vhost = in.readUTF();
-            
-            accessed = in.readLong();//accessTime
-            lastAccessed = in.readLong(); //lastAccessTime
-            createTime = in.readLong(); //time created
-            cookieSetTime = in.readLong();//time cookie was set
-            lastNode = in.readUTF(); //last managing node
-            expiry = in.readLong(); 
-            maxInactive = in.readLong();
-            attributes = (HashMap<String,Object>)in.readObject();
-        }
-        
-    }
-    
- 
-    
-    
-    /**
-     * Session
-     *
-     * Representation of a session in local memory.
-     */
-    public class Session extends MemSession
-    {
-        
-        private ReentrantLock _lock = new ReentrantLock();
-        
-        /**
-         * The (canonical) context path for with which this session is associated
-         */
-        private String _contextPath;
-        
-        
-        
-        /**
-         * The time in msec since the epoch at which this session should expire
-         */
-        private long _expiryTime; 
-        
-        
-        /**
-         * Time in msec since the epoch at which this session was last read from cluster
-         */
-        private long _lastSyncTime;
-        
-        
-        /**
-         * The workername of last node known to be managing the session
-         */
-        private String _lastNode;
-        
-        
-        /**
-         * If dirty, session needs to be (re)sent to cluster
-         */
-        protected boolean _dirty=false;
-        
-        
-     
-
-        /**
-         * Any virtual hosts for the context with which this session is associated
-         */
-        private String _vhost;
-
-        
-        /**
-         * Count of how many threads are active in this session
-         */
-        private AtomicInteger _activeThreads = new AtomicInteger(0);
-        
-        
-        
-        
-        /**
-         * A new session.
-         * 
-         * @param request the request
-         */
-        protected Session (HttpServletRequest request)
-        {
-            super(InfinispanSessionManager.this,request);
-            long maxInterval = getMaxInactiveInterval();
-            _expiryTime = (maxInterval <= 0 ? 0 : (System.currentTimeMillis() + maxInterval*1000L));
-            _lastNode = getSessionIdManager().getWorkerName();
-           setVHost(InfinispanSessionManager.getVirtualHost(_context));
-           setContextPath(InfinispanSessionManager.getContextPath(_context));
-           _activeThreads.incrementAndGet(); //access will not be called on a freshly created session so increment here
-        }
-        
-        
-        protected Session (SerializableSessionData sd)
-        {
-            super(InfinispanSessionManager.this, sd.createTime, sd.accessed, sd.clusterId);
-            _expiryTime = (sd.maxInactive <= 0 ? 0 : (System.currentTimeMillis() + sd.maxInactive*1000L));
-            setLastNode(sd.lastNode);
-            setContextPath(sd.contextPath);
-            setVHost(sd.vhost);
-            addAttributes(sd.attributes);
-        }
-        
-        
-        /**
-         * A restored session.
-         * 
-         * @param sessionId the session id
-         * @param created time created
-         * @param accessed time last accessed
-         * @param maxInterval max expiry interval
-         */
-        protected Session (String sessionId, long created, long accessed, long maxInterval)
-        {
-            super(InfinispanSessionManager.this, created, accessed, sessionId);
-            _expiryTime = (maxInterval <= 0 ? 0 : (System.currentTimeMillis() + maxInterval*1000L));
-        }
-        
-        /** 
-         * Called on entry to the session.
-         * 
-         * @see org.eclipse.jetty.server.session.AbstractSession#access(long)
-         */
-        @Override
-        protected boolean access(long time)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Access session({}) for context {} on worker {}", getId(), getContextPath(), getSessionIdManager().getWorkerName());
-            try
-            {
-
-                long now = System.currentTimeMillis();
-                //lock so that no other thread can call access or complete until the first one has refreshed the session object if necessary
-                _lock.lock();
-                //a request thread is entering
-                if (_activeThreads.incrementAndGet() == 1)
-                {
-                    //if the first thread, check that the session in memory is not stale, if we're checking for stale sessions
-                    if (getStaleIntervalSec() > 0  && (now - getLastSyncTime()) >= (getStaleIntervalSec() * 1000L))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Acess session({}) for context {} on worker {} stale session. Reloading.", getId(), getContextPath(), getSessionIdManager().getWorkerName());
-                        refresh();
-                    }
-                }
-            }
-            catch (Exception e)
-            {
-                LOG.warn(e);
-            }
-            finally
-            {            
-                _lock.unlock();
-            }
-
-            if (super.access(time))
-            {
-                int maxInterval=getMaxInactiveInterval();
-                _expiryTime = (maxInterval <= 0 ? 0 : (time + maxInterval*1000L));
-                return true;
-            }
-            return false;
-        }
-
-
-        /**
-         * Exit from session
-         * @see org.eclipse.jetty.server.session.AbstractSession#complete()
-         */
-        @Override
-        protected void complete()
-        {
-            super.complete();
-
-            //lock so that no other thread that might be calling access can proceed until this complete is done
-            _lock.lock();
-
-            try
-            {
-                //if this is the last request thread to be in the session
-                if (_activeThreads.decrementAndGet() == 0)
-                {
-                    try
-                    {
-                        //an invalid session will already have been removed from the
-                        //local session map and deleted from the cluster. If its valid save
-                        //it to the cluster.
-                        //TODO consider doing only periodic saves if only the last access
-                        //time to the session changes
-                        if (isValid())
-                        {
-                            //if session still valid && its dirty or stale or never been synced, write it to the cluster
-                            //otherwise, we just keep the updated last access time in memory
-                            if (_dirty || getLastSyncTime() == 0 || isStale(System.currentTimeMillis()))
-                            {
-                                willPassivate();
-                                save(this);
-                                didActivate();
-                            }
-                        }
-                    }
-                    catch (Exception e)
-                    {
-                        LOG.warn("Problem saving session({})",getId(), e);
-                    } 
-                    finally
-                    {
-                        _dirty = false;
-                    }
-                }
-            }
-            finally
-            {
-                _lock.unlock();
-            }
-        }
-        
-        /** Test if the session is stale
-         * @param atTime time when stale
-         * @return true if stale
-         */
-        protected boolean isStale (long atTime)
-        {
-            return (getStaleIntervalSec() > 0) && (atTime - getLastSyncTime() >= (getStaleIntervalSec()*1000L));
-        }
-        
-        
-        /** Test if the session is dirty
-         * @return true if dirty
-         */
-        protected boolean isDirty ()
-        {
-            return _dirty;
-        }
-
-        /** 
-         * Expire the session.
-         * 
-         * @see org.eclipse.jetty.server.session.AbstractSession#timeout()
-         */
-        @Override
-        protected void timeout()
-        {
-            super.timeout();
-        }
-        
-      
-        
-        /**
-         * Reload the session from the cluster. If the node that
-         * last managed the session from the cluster is ourself,
-         * then the session does not need refreshing.
-         * NOTE: this method MUST be called with sufficient locks
-         * in place to prevent 2 or more concurrent threads from
-         * simultaneously updating the session.
-         */
-        private void refresh ()
-        {
-            //get fresh copy from the cluster
-            Session fresh = load(makeKey(getClusterId(), _context));
-
-            //if the session no longer exists, invalidate
-            if (fresh == null)
-            {
-                invalidate();
-                return;
-            }
-
-            //cluster copy assumed to be the same as we were the last
-            //node to manage it
-            if (fresh.getLastNode().equals(getLastNode()))
-                return;
-
-            setLastNode(getSessionIdManager().getWorkerName());
-            
-            //prepare for refresh
-            willPassivate();
-
-            //if fresh has no attributes, remove them
-            if (fresh.getAttributes() == 0)
-                this.clearAttributes();
-            else
-            {
-                //reconcile attributes
-                for (String key:fresh.getAttributeMap().keySet())
-                {
-                    Object freshvalue = fresh.getAttribute(key);
-
-                    //session does not already contain this attribute, so bind it
-                    if (getAttribute(key) == null)
-                    { 
-                        doPutOrRemove(key,freshvalue);
-                        bindValue(key,freshvalue);
-                    }
-                    else //session already contains this attribute, update its value
-                    {
-                        doPutOrRemove(key,freshvalue);
-                    }
-
-                }
-                // cleanup, remove values from session, that don't exist in data anymore:
-                for (String key : getNames())
-                {
-                    if (fresh.getAttribute(key) == null)
-                    {
-                        Object oldvalue = getAttribute(key);
-                        doPutOrRemove(key,null);
-                        unbindValue(key,oldvalue);
-                    }
-                }
-            }
-            //finish refresh
-            didActivate();
-        }
-
-
-        public void setExpiry (long expiry)
-        {
-            _expiryTime = expiry;
-        }
-        
-
-        public long getExpiry ()
-        {
-            return _expiryTime;
-        }
-        
-        public void swapId (String newId, String newNodeId)
-        {
-            //TODO probably synchronize rather than use the access/complete lock?
-            _lock.lock();
-            setClusterId(newId);
-            setNodeId(newNodeId);
-            _lock.unlock();
-        }
-        
-        @Override
-        public void setAttribute (String name, Object value)
-        {
-            Object old = changeAttribute(name, value);
-            if (value == null && old == null)
-                return; //if same as remove attribute but attribute was already removed, no change
-            
-           _dirty = true;
-        }
-        
-        
-        public String getContextPath()
-        {
-            return _contextPath;
-        }
-
-
-        public void setContextPath(String contextPath)
-        {
-            this._contextPath = contextPath;
-        }
-
-
-        public String getVHost()
-        {
-            return _vhost;
-        }
-
-
-        public void setVHost(String vhost)
-        {
-            this._vhost = vhost;
-        }
-        
-        public String getLastNode()
-        {
-            return _lastNode;
-        }
-
-
-        public void setLastNode(String lastNode)
-        {
-            _lastNode = lastNode;
-        }
-
-
-        public long getLastSyncTime()
-        {
-            return _lastSyncTime;
-        }
-
-
-        public void setLastSyncTime(long lastSyncTime)
-        {
-            _lastSyncTime = lastSyncTime;
-        }
-
-    }
-
-
-
     
     /**
      * Start the session manager.
@@ -597,24 +59,9 @@
         if (_sessionIdManager == null)
             throw new IllegalStateException("No session id manager defined");
         
-        if (_cache == null)
-            throw new IllegalStateException("No session cache defined");
-        
-        _sessions = new ConcurrentHashMap<String, Session>();
-
-        //try and use a common scheduler, fallback to own
-        _scheduler = getSessionHandler().getServer().getBean(Scheduler.class);
-        if (_scheduler == null)
-        {
-            _scheduler = new ScheduledExecutorScheduler();
-            _ownScheduler = true;
-            _scheduler.start();
-        }
-        else if (!_scheduler.isStarted())
-            throw new IllegalStateException("Shared scheduler not started");
- 
-        setScavengeInterval(getScavengeInterval());
-        
+        ((AbstractSessionStore)_sessionStore).setSessionDataStore(_sessionDataStore);
+        _sessionDataStore.setSessionIdManager(_sessionIdManager);
+       
         super.doStart();
     }
 
@@ -629,542 +76,16 @@
     {
         super.doStop();
 
-        if (_task!=null)
-            _task.cancel();
-        _task=null;
-        if (_ownScheduler && _scheduler !=null)
-            _scheduler.stop();
-        _scheduler = null;
-        
-        _sessions.clear();
-        _sessions = null;
-    }
-    
-    
-    
-    /**
-     * Look for sessions in local memory that have expired.
-     */
-    /**
-     * 
-     */
-    public void scavenge ()
-    {
-        Set<String> candidateIds = new HashSet<String>();
-        long now = System.currentTimeMillis();
-        
-        LOG.info("SessionManager for context {} scavenging at {} ", getContextPath(getContext()), now);
-        for (Map.Entry<String, Session> entry:_sessions.entrySet())
-        {
-            long expiry = entry.getValue().getExpiry();
-            if (expiry > 0 && expiry < now)
-                candidateIds.add(entry.getKey());
-        }
-
-        for (String candidateId:candidateIds)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Session {} expired ", candidateId);
-            
-            Session candidateSession = _sessions.get(candidateId);
-            if (candidateSession != null)
-            {
-                //double check the state of the session in the cache, as the
-                //session may have migrated to another node. This leaves a window
-                //where the cached session may have been changed by another node
-                Session cachedSession = load(makeKey(candidateId, _context));
-                if (cachedSession == null)
-                {
-                   if (LOG.isDebugEnabled()) LOG.debug("Locally expired session({}) does not exist in cluster ",candidateId);
-                    //the session no longer exists, do a full invalidation
-                    candidateSession.timeout();
-                }
-                else if (getSessionIdManager().getWorkerName().equals(cachedSession.getLastNode()))
-                {
-                    if (LOG.isDebugEnabled()) LOG.debug("Expiring session({}) local to session manager",candidateId);
-                    //if I am the master of the session then it can be timed out
-                    candidateSession.timeout();
-                }
-                else
-                {
-                    //some other node is the master of the session, simply remove it from my memory
-                    if (LOG.isDebugEnabled()) LOG.debug("Session({}) not local to this session manager, removing from local memory", candidateId);
-                    candidateSession.willPassivate();
-                    _sessions.remove(candidateSession.getClusterId());
-                }
-
-            }
-        }
-    }
-    
-    
-
-    public long getScavengeInterval ()
-    {
-        return _scavengeIntervalMs/1000;
     }
 
-    
-    
-    /**
-     * Set the interval between runs of the scavenger. It should not be run too
-     * often.
-     * 
-     * 
-     * @param sec scavenge interval in seconds
-     */
-    public void setScavengeInterval (long sec)
-    {
-        if (sec<=0)
-            sec=60;
-
-        long old_period=_scavengeIntervalMs;
-        long period=sec*1000L;
-
-        _scavengeIntervalMs=period;
-
-        //add a bit of variability into the scavenge time so that not all
-        //nodes with the same scavenge time sync up
-        long tenPercent = _scavengeIntervalMs/10;
-        if ((System.currentTimeMillis()%2) == 0)
-            _scavengeIntervalMs += tenPercent;
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("Scavenging every "+_scavengeIntervalMs+" ms");
-        
-        synchronized (this)
-        {
-            if (_scheduler != null && (period!=old_period || _task==null))
-            {
-                if (_task!=null)
-                    _task.cancel();
-                if (_scavenger == null)
-                    _scavenger = new Scavenger();
-                
-                _task = _scheduler.schedule(_scavenger,_scavengeIntervalMs,TimeUnit.MILLISECONDS);
-            }
-        }
-    }
-    
-    
-    
-
-    /**
-     * Get the clustered cache instance.
-     * 
-     * @return the cache
-     */
-    public BasicCache<String, Object> getCache() 
-    {
-        return _cache;
-    }
-
-    
-    
-    /**
-     * Set the clustered cache instance.
-     * 
-     * @param cache the cache
-     */
-    public void setCache (BasicCache<String, Object> cache) 
-    {
-        this._cache = cache;
-    }
-
-
-    
-    
-    
-    public long getStaleIntervalSec()
-    {
-        return _staleIntervalSec;
-    }
-
-
-    public void setStaleIntervalSec(long staleIntervalSec)
-    {
-        _staleIntervalSec = staleIntervalSec;
-    }
-
-
-    /** 
-     * Add a new session for the context related to this session manager
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#addSession(org.eclipse.jetty.server.session.AbstractSession)
-     */
-    @Override
-    protected void addSession(AbstractSession session)
-    {
-        if (session==null)
-            return;
-        
-        if (LOG.isDebugEnabled()) LOG.debug("Adding session({}) to session manager for context {} on worker {}",session.getClusterId(), getContextPath(getContext()),getSessionIdManager().getWorkerName() + " with lastnode="+((Session)session).getLastNode());
-        _sessions.put(session.getClusterId(), (Session)session);
-        
-        try
-        {     
-                session.willPassivate();
-                save(((InfinispanSessionManager.Session)session));
-                session.didActivate();
-            
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Unable to store new session id="+session.getId() , e);
-        }
-    }
-
-    /** 
-     * Ask the cluster for the session.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#getSession(java.lang.String)
-     */
-    @Override
-    public AbstractSession getSession(String idInCluster)
-    {
-        Session session = null;
-
-        //try and find the session in this node's memory
-        Session memSession = (Session)_sessions.get(idInCluster);
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("getSession({}) {} in session map",idInCluster,(memSession==null?"not":""));
-
-        long now = System.currentTimeMillis();
-        try
-        {
-            //if the session is not in this node's memory, then load it from the cluster cache
-            if (memSession == null)
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("getSession({}): loading session data from cluster", idInCluster);
-
-                session = load(makeKey(idInCluster, _context));
-                if (session != null)
-                {
-                    //We retrieved a session with the same key from the database
-
-                    //Check that it wasn't expired
-                    if (session.getExpiry() > 0 && session.getExpiry() <= now)
-                    {
-                        if (LOG.isDebugEnabled()) LOG.debug("getSession ({}): Session expired", idInCluster);
-                        //ensure that the session id for the expired session is deleted so that a new session with the 
-                        //same id cannot be created (because the idInUse() test would succeed)
-                        ((InfinispanSessionIdManager)getSessionIdManager()).removeSession(session);
-                        return null;  
-                    }
-
-                    //Update the last worker node to me
-                    session.setLastNode(getSessionIdManager().getWorkerName());                            
-                    //TODO consider saving session here if lastNode was not this node
-
-                    //Check that another thread hasn't loaded the same session
-                    Session existingSession = _sessions.putIfAbsent(idInCluster, session);
-                    if (existingSession != null)
-                    {
-                        //use the one that the other thread inserted
-                        session = existingSession;
-                        LOG.debug("getSession({}): using session loaded by another request thread ", idInCluster);
-                    }
-                    else
-                    {
-                        //indicate that the session was reinflated
-                        session.didActivate();
-                        LOG.debug("getSession({}): loaded session from cluster", idInCluster);
-                    }
-                    return session;
-                }
-                else
-                {
-                    //The requested session does not exist anywhere in the cluster
-                    LOG.debug("getSession({}): No session in cluster matching",idInCluster);
-                    return null;
-                }
-            }
-            else
-            {
-               //The session exists in this node's memory
-               LOG.debug("getSession({}): returning session from local memory ", memSession.getClusterId());
-                return memSession;
-            }
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Unable to load session="+idInCluster, e);
-            return null;
-        }
-    }
-    
-    
-
-    /** 
-     * The session manager is stopping.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#shutdownSessions()
-     */
-    @Override
-    protected void shutdownSessions() throws Exception
-    {
-        Set<String> keys = new HashSet<String>(_sessions.keySet());
-        for (String key:keys)
-        {
-            Session session = _sessions.remove(key); //take the session out of the session list
-            //If the session is dirty, then write it to the cluster.
-            //If the session is simply stale do NOT write it to the cluster, as some other node
-            //may have started managing that session - this means that the last accessed/expiry time
-            //will not be updated, meaning it may look like it can expire sooner than it should.
-            try
-            {
-                if (session.isDirty())
-                {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Saving dirty session {} before exiting ", session.getId());
-                    save(session);
-                }
-            }
-            catch (Exception e)
-            {
-                LOG.warn(e);
-            }
-        }
-    }
-
-
-    @Override
-    protected AbstractSession newSession(HttpServletRequest request)
-    {
-        return new Session(request);
-    }
-
-    /** 
-     * Remove a session from local memory, and delete it from
-     * the cluster cache.
-     * 
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#removeSession(java.lang.String)
-     */
-    @Override
-    protected boolean removeSession(String idInCluster)
-    {
-        Session session = (Session)_sessions.remove(idInCluster);
-        try
-        {
-            if (session != null)
-                delete(session);
-        }
-        catch (Exception e)
-        {
-            LOG.warn("Problem deleting session id="+idInCluster, e);
-        }
-        return session!=null;
-    }
-    
-    
-    
-    
-    @Override
-    public void renewSessionId(String oldClusterId, String oldNodeId, String newClusterId, String newNodeId)
-    {
-        Session session = null;
-        try
-        {
-            //take the session with that id out of our managed list
-            session = (Session)_sessions.remove(oldClusterId);
-            if (session != null)
-            {
-                //TODO consider transactionality and ramifications if the session is live on another node
-                delete(session); //delete the old session from the cluster  
-                session.swapId(newClusterId, newNodeId); //update the session
-                _sessions.put(newClusterId, session); //put it into managed list under new key
-                save(session); //put the session under the new id into the cluster
-            }
-        }
-        catch (Exception e)
-        {
-            LOG.warn(e);
-        }
-
-        super.renewSessionId(oldClusterId, oldNodeId, newClusterId, newNodeId);
-    }
 
 
     /**
-     * Load a session from the clustered cache.
-     * 
-     * @param key the session key
-     * @return the session
-     */
-    protected Session load (String key)
-    {
-        if (_cache == null)
-            throw new IllegalStateException("No cache");
-        
-        if (LOG.isDebugEnabled()) LOG.debug("Loading session {} from cluster", key);
-
-        SerializableSessionData storableSession = (SerializableSessionData)_cache.get(key);
-        if (storableSession == null)
-        {
-            if (LOG.isDebugEnabled()) LOG.debug("No session {} in cluster ",key);
-            return null;
-        }
-        else
-        {
-            Session session = new Session (storableSession);
-            session.setLastSyncTime(System.currentTimeMillis());
-            return session;
-        }
-    }
-    
-    
-    
-    /**
-     * Save or update the session to the cluster cache
-     * 
-     * @param session the session
-     * @throws Exception if unable to save
-     */
-    protected void save (InfinispanSessionManager.Session session)
-    throws Exception
-    {
-        if (_cache == null)
-            throw new IllegalStateException("No cache");
-        
-        if (LOG.isDebugEnabled()) LOG.debug("Writing session {} to cluster", session.getId());
-    
-        SerializableSessionData storableSession = new SerializableSessionData(session);
-
-        //Put an idle timeout on the cache entry if the session is not immortal - 
-        //if no requests arrive at any node before this timeout occurs, or no node 
-        //scavenges the session before this timeout occurs, the session will be removed.
-        //NOTE: that no session listeners can be called for this.
-        InfinispanSessionIdManager sessionIdManager = (InfinispanSessionIdManager)getSessionIdManager();
-        if (storableSession.maxInactive > 0)
-            _cache.put(makeKey(session, _context), storableSession, -1, TimeUnit.SECONDS, storableSession.maxInactive*sessionIdManager.getIdleExpiryMultiple(), TimeUnit.SECONDS);
-        else
-            _cache.put(makeKey(session, _context), storableSession);
-        
-        //tickle the session id manager to keep the sessionid entry for this session up-to-date
-        sessionIdManager.touch(session.getClusterId());
-        
-        session.setLastSyncTime(System.currentTimeMillis());
-    }
-    
-    
-    
-    /**
-     * Remove the session from the cluster cache.
-     * 
-     * @param session the session
-     */
-    protected void delete (InfinispanSessionManager.Session session)
-    {  
-        if (_cache == null)
-            throw new IllegalStateException("No cache");
-        if (LOG.isDebugEnabled()) LOG.debug("Removing session {} from cluster", session.getId());
-        _cache.remove(makeKey(session, _context));
-    }
-
-    
-    /**
-     * Invalidate a session for this context with the given id
-     * 
-     * @param idInCluster session id in cluster
-     */
-    public void invalidateSession (String idInCluster)
-    {
-        Session session = (Session)_sessions.get(idInCluster);
-
-        if (session != null)
-        {
-            session.invalidate();
-        }
-    }
-
-    
-    /**
-     * Make a unique key for this session.
-     * As the same session id can be used across multiple contexts, to
-     * make it unique, the key must be composed of:
-     * <ol>
-     * <li>the id</li>
-     * <li>the context path</li>
-     * <li>the virtual hosts</li>
-     * </ol>
-     * 
-     *TODO consider the difference between getClusterId and getId
-     * @param session
      * @return
      */
-    private String makeKey (Session session, Context context)
+    public InfinispanSessionDataStore getSessionDataStore()
     {
-       return makeKey(session.getId(), context);
-    }
-    
-    /**
-     * Make a unique key for this session.
-     * As the same session id can be used across multiple contexts, to
-     * make it unique, the key must be composed of:
-     * <ol>
-     * <li>the id</li>
-     * <li>the context path</li>
-     * <li>the virtual hosts</li>
-     * </ol>
-     * 
-     *TODO consider the difference between getClusterId and getId
-     * @param session
-     * @return
-     */
-    private String makeKey (String id, Context context)
-    {
-        String key = getContextPath(context);
-        key = key + "_" + getVirtualHost(context);
-        key = key+"_"+id;
-        return key;
-    }
-    
-    /**
-     * Turn the context path into an acceptable string
-     * 
-     * @param context
-     * @return
-     */
-    private static String getContextPath (ContextHandler.Context context)
-    {
-        return canonicalize (context.getContextPath());
-    }
-
-    /**
-     * Get the first virtual host for the context.
-     *
-     * Used to help identify the exact session/contextPath.
-     *
-     * @return 0.0.0.0 if no virtual host is defined
-     */
-    private static String getVirtualHost (ContextHandler.Context context)
-    {
-        String vhost = "0.0.0.0";
-
-        if (context==null)
-            return vhost;
-
-        String [] vhosts = context.getContextHandler().getVirtualHosts();
-        if (vhosts==null || vhosts.length==0 || vhosts[0]==null)
-            return vhost;
-
-        return vhosts[0];
-    }
-
-    /**
-     * Make an acceptable name from a context path.
-     *
-     * @param path
-     * @return
-     */
-    private static String canonicalize (String path)
-    {
-        if (path==null)
-            return "";
-
-        return path.replace('/', '_').replace('.','_').replace('\\','_');
+        return _sessionDataStore;
     }
 
 }
diff --git a/jetty-io/pom.xml b/jetty-io/pom.xml
index a8c8634..3723e68 100644
--- a/jetty-io/pom.xml
+++ b/jetty-io/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <artifactId>jetty-project</artifactId>
     <groupId>org.eclipse.jetty</groupId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-io</artifactId>
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java
index 0d00a17..be1b1f2 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java
@@ -19,9 +19,9 @@
 package org.eclipse.jetty.io;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
@@ -31,10 +31,10 @@
 
 public abstract class AbstractEndPoint extends IdleTimeout implements EndPoint
 {
+    enum State {OPEN, ISHUTTING, ISHUT, OSHUTTING, OSHUT, CLOSED};
     private static final Logger LOG = Log.getLogger(AbstractEndPoint.class);
+    private final AtomicReference<State> _state = new AtomicReference<>(State.OPEN);
     private final long _created=System.currentTimeMillis();
-    private final InetSocketAddress _local;
-    private final InetSocketAddress _remote;
     private volatile Connection _connection;
 
     private final FillInterest _fillInterest = new FillInterest()
@@ -55,11 +55,231 @@
         }
     };
 
-    protected AbstractEndPoint(Scheduler scheduler,InetSocketAddress local,InetSocketAddress remote)
+    protected AbstractEndPoint(Scheduler scheduler)
     {
         super(scheduler);
-        _local=local;
-        _remote=remote;
+    }
+
+
+    protected final void shutdownInput()
+    {
+        while(true)
+        {
+            State s = _state.get();
+            switch(s)
+            {       
+                case OPEN:
+                    if (!_state.compareAndSet(s,State.ISHUTTING))
+                        continue;
+                    try
+                    {
+                        doShutdownInput();
+                    }
+                    finally
+                    {
+                        if(!_state.compareAndSet(State.ISHUTTING,State.ISHUT))
+                        {
+                            // If somebody else switched to CLOSED while we were ishutting,
+                            // then we do the close for them
+                            if (_state.get()==State.CLOSED)
+                                doOnClose();
+                            else
+                                throw new IllegalStateException();
+                        }
+                    }
+                    return;
+                
+                case ISHUTTING:  // Somebody else ishutting
+                case ISHUT: // Already ishut
+                    return;
+                    
+                case OSHUTTING:
+                    if (!_state.compareAndSet(s,State.CLOSED))
+                        continue;
+                    // The thread doing the OSHUT will close
+                    return;
+                    
+                case OSHUT:
+                    if (!_state.compareAndSet(s,State.CLOSED))
+                        continue;
+                    // Already OSHUT so we close
+                    doOnClose();
+                    return;
+
+                case CLOSED: // already closed
+                    return;
+            }
+        }
+    }
+
+    @Override
+    public final void shutdownOutput()
+    {
+        while(true)
+        {
+            State s = _state.get();
+            switch(s)
+            {
+                case OPEN:
+                    if (!_state.compareAndSet(s,State.OSHUTTING))
+                        continue;
+                    try
+                    {
+                        doShutdownOutput();
+                    }
+                    finally
+                    {
+                        if(!_state.compareAndSet(State.OSHUTTING,State.OSHUT))
+                        {
+                            // If somebody else switched to CLOSED while we were oshutting,
+                            // then we do the close for them
+                            if (_state.get()==State.CLOSED)
+                                doOnClose();
+                            else
+                                throw new IllegalStateException();
+                        }
+                    }
+                    return;
+                    
+                case ISHUTTING:
+                    if (!_state.compareAndSet(s,State.CLOSED))
+                        continue;
+                    // The thread doing the ISHUT will close
+                    return;
+                    
+                case ISHUT:
+                    if (!_state.compareAndSet(s,State.CLOSED))
+                        continue;
+                    // Already ISHUT so we close
+                    doOnClose();
+                    return;
+                
+                case OSHUTTING:  // Somebody else oshutting
+                case OSHUT: // Already oshut
+                    return;
+
+                case CLOSED: // already closed
+                    return;
+            }
+        }
+    }
+
+    @Override
+    public final void close()
+    {
+        while(true)
+        {
+            State s = _state.get();
+            switch(s)
+            {
+                case OPEN:
+                case ISHUT: // Already ishut
+                case OSHUT: // Already oshut
+                    if (!_state.compareAndSet(s,State.CLOSED))
+                        continue;
+                    doOnClose();
+                    return;
+
+                case ISHUTTING: // Somebody else ishutting
+                case OSHUTTING: // Somebody else oshutting
+                    if (!_state.compareAndSet(s,State.CLOSED))
+                        continue;
+                    // The thread doing the IO SHUT will call doOnClose
+                    return;                
+
+                case CLOSED: // already closed
+                    return;
+            }
+        }
+    }
+    
+    protected void doShutdownInput()
+    {}
+    
+    protected void doShutdownOutput()
+    {}
+    
+    protected void doClose()
+    {}
+    
+    private void doOnClose()
+    {
+        try
+        {
+            doClose();
+        }
+        finally
+        {
+            onClose();
+        }
+    }
+        
+
+    @Override
+    public boolean isOutputShutdown()
+    {
+        switch(_state.get())
+        {
+            case CLOSED:
+            case OSHUT:
+            case OSHUTTING:
+                return true;
+            default:
+                return false;            
+        }
+    }
+    @Override
+    public boolean isInputShutdown()
+    {
+        switch(_state.get())
+        {
+            case CLOSED:
+            case ISHUT:
+            case ISHUTTING:
+                return true;
+            default:
+                return false;            
+        }
+    }
+
+    @Override
+    public boolean isOpen()
+    {
+        switch(_state.get())
+        {
+            case CLOSED:
+                return false;
+            default:
+                return true;            
+        }
+    }
+    
+    public void checkFlush() throws IOException
+    {
+        State s=_state.get();
+        switch(s)
+        {
+            case OSHUT:
+            case OSHUTTING:
+            case CLOSED:
+                throw new IOException(s.toString());
+            default:
+                break;
+        }
+    }
+    
+    public void checkFill() throws IOException
+    {
+        State s=_state.get();
+        switch(s)
+        {
+            case ISHUT:
+            case ISHUTTING:
+            case CLOSED:
+                throw new IOException(s.toString());
+            default:
+                break;
+        }
     }
 
     @Override
@@ -69,18 +289,6 @@
     }
 
     @Override
-    public InetSocketAddress getLocalAddress()
-    {
-        return _local;
-    }
-
-    @Override
-    public InetSocketAddress getRemoteAddress()
-    {
-        return _remote;
-    }
-
-    @Override
     public Connection getConnection()
     {
         return _connection;
@@ -98,12 +306,22 @@
         return false;
     }
 
+
+    
+    protected void reset()
+    {
+        _state.set(State.OPEN);
+        _writeFlusher.onClose();
+        _fillInterest.onClose();
+    }
+    
     @Override
     public void onOpen()
     {
         if (LOG.isDebugEnabled())
             LOG.debug("onOpen {}",this);
-        super.onOpen();
+        if (_state.get()!=State.OPEN)
+            throw new IllegalStateException();
     }
 
     @Override
@@ -117,12 +335,6 @@
     }
 
     @Override
-    public void close()
-    {
-        onClose();
-    }
-
-    @Override
     public void fillInterested(Callback callback) throws IllegalStateException
     {
         notIdle();
@@ -211,15 +423,13 @@
             c=c.getSuperclass();
             name=c.getSimpleName();
         }
-
-        return String.format("%s@%x{%s<->%d,%s,%s,%s,%s,%s,%d/%d,%s}",
+        
+        return String.format("%s@%x{%s<->%s,%s,%s|%s,%d/%d,%s}",
                 name,
                 hashCode(),
                 getRemoteAddress(),
-                getLocalAddress().getPort(),
-                isOpen()?"Open":"CLOSED",
-                isInputShutdown()?"ISHUT":"in",
-                isOutputShutdown()?"OSHUT":"out",
+                getLocalAddress(),
+                _state.get(),
                 _fillInterest.toStateString(),
                 _writeFlusher.toStateString(),
                 getIdleFor(),
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/ByteArrayEndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/ByteArrayEndPoint.java
index 4b5a407..64179a6 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/ByteArrayEndPoint.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/ByteArrayEndPoint.java
@@ -20,7 +20,10 @@
 
 import java.io.EOFException;
 import java.io.IOException;
+import java.net.Inet4Address;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException;
 import java.nio.charset.Charset;
@@ -42,7 +45,28 @@
 public class ByteArrayEndPoint extends AbstractEndPoint
 {
     static final Logger LOG = Log.getLogger(ByteArrayEndPoint.class);
-    public final static InetSocketAddress NOIP=new InetSocketAddress(0);
+    static final InetAddress  NOIP;
+    static final InetSocketAddress NOIPPORT;
+    
+    static
+    {
+        InetAddress noip=null;
+        try
+        {
+            noip = Inet4Address.getByName("0.0.0.0");
+        }
+        catch (UnknownHostException e)
+        {
+            LOG.warn(e);
+        }
+        finally
+        {
+            NOIP=noip;
+            NOIPPORT=new InetSocketAddress(NOIP,0);
+        }
+    }
+    
+    
     private static final ByteBuffer EOF = BufferUtil.allocate(0);
 
     private final Runnable _runFillable = new Runnable()
@@ -57,9 +81,6 @@
     private final Locker _locker = new Locker();
     private final Queue<ByteBuffer> _inQ = new ArrayQueue<>();
     private ByteBuffer _out;
-    private boolean _ishut;
-    private boolean _oshut;
-    private boolean _closed;
     private boolean _growOutput;
 
     /* ------------------------------------------------------------ */
@@ -112,11 +133,26 @@
     /* ------------------------------------------------------------ */
     public ByteArrayEndPoint(Scheduler timer, long idleTimeoutMs, ByteBuffer input, ByteBuffer output)
     {
-        super(timer,NOIP,NOIP);
+        super(timer);
         if (BufferUtil.hasContent(input))
             addInput(input);
         _out=output==null?BufferUtil.allocate(1024):output;
         setIdleTimeout(idleTimeoutMs);
+        onOpen();
+    }
+    
+    /* ------------------------------------------------------------ */
+    @Override
+    public InetSocketAddress getLocalAddress()
+    {
+        return NOIPPORT;
+    }
+
+    /* ------------------------------------------------------------ */
+    @Override
+    public InetSocketAddress getRemoteAddress()
+    {
+        return NOIPPORT;
     }
 
     /* ------------------------------------------------------------ */
@@ -138,7 +174,7 @@
     {
         try(Locker.Lock lock = _locker.lock())
         {
-            if (_closed)
+            if (!isOpen())
                 throw new ClosedChannelException();
 
             ByteBuffer in = _inQ.peek();
@@ -288,92 +324,6 @@
     }
 
     /* ------------------------------------------------------------ */
-    /*
-     * @see org.eclipse.io.EndPoint#isOpen()
-     */
-    @Override
-    public boolean isOpen()
-    {
-        try(Locker.Lock lock = _locker.lock())
-        {
-            return !_closed;
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    /*
-     */
-    @Override
-    public boolean isInputShutdown()
-    {
-        try(Locker.Lock lock = _locker.lock())
-        {
-            return _ishut||_closed;
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    /*
-     */
-    @Override
-    public boolean isOutputShutdown()
-    {
-        try(Locker.Lock lock = _locker.lock())
-        {
-            return _oshut||_closed;
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    public void shutdownInput()
-    {
-        boolean close=false;
-        try(Locker.Lock lock = _locker.lock())
-        {
-            _ishut=true;
-            if (_oshut && !_closed)
-                close=_closed=true;
-        }
-        if (close)
-            super.close();
-    }
-
-    /* ------------------------------------------------------------ */
-    /*
-     * @see org.eclipse.io.EndPoint#shutdownOutput()
-     */
-    @Override
-    public void shutdownOutput()
-    {
-        boolean close=false;
-        try(Locker.Lock lock = _locker.lock())
-        {
-            _oshut=true;
-            if (_ishut && !_closed)
-                close=_closed=true;
-        }
-        if (close)
-            super.close();
-    }
-
-    /* ------------------------------------------------------------ */
-    /*
-     * @see org.eclipse.io.EndPoint#close()
-     */
-    @Override
-    public void close()
-    {
-        boolean close=false;
-        try(Locker.Lock lock = _locker.lock())
-        {
-            if (!_closed)
-                close=_closed=_ishut=_oshut=true;
-        }
-        if (close)
-            super.close();
-    }
-
-    /* ------------------------------------------------------------ */
     /**
      * @return <code>true</code> if there are bytes remaining to be read from the encoded input
      */
@@ -390,15 +340,14 @@
     public int fill(ByteBuffer buffer) throws IOException
     {
         int filled=0;
-        boolean close=false;
         try(Locker.Lock lock = _locker.lock())
         {
             while(true)
             {
-                if (_closed)
+                if (!isOpen())
                     throw new EofException("CLOSED");
 
-                if (_ishut)
+                if (isInputShutdown())
                     return -1;
 
                 if (_inQ.isEmpty())
@@ -407,9 +356,6 @@
                 ByteBuffer in= _inQ.peek();
                 if (in==EOF)
                 {
-                    _ishut=true;
-                    if (_oshut)
-                        close=_closed=true;
                     filled=-1;
                     break;
                 }
@@ -425,10 +371,10 @@
             }
         }
 
-        if (close)
-            super.close();
         if (filled>0)
             notIdle();
+        else if (filled<0)
+            shutdownInput();
         return filled;
     }
 
@@ -439,9 +385,9 @@
     @Override
     public boolean flush(ByteBuffer... buffers) throws IOException
     {
-        if (_closed)
+        if (!isOpen())
             throw new IOException("CLOSED");
-        if (_oshut)
+        if (isOutputShutdown())
             throw new IOException("OSHUT");
 
         boolean flushed=true;
@@ -483,13 +429,12 @@
      */
     public void reset()
     {
-        getFillInterest().onClose();
-        getWriteFlusher().onClose();
-        _ishut=false;
-        _oshut=false;
-        _closed=false;
-        _inQ.clear();
+        try(Locker.Lock lock = _locker.lock())
+        {
+            _inQ.clear();
+        }
         BufferUtil.clear(_out);
+        super.reset();
     }
 
     /* ------------------------------------------------------------ */
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/ChannelEndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/ChannelEndPoint.java
index 1952760..6e5688f 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/ChannelEndPoint.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/ChannelEndPoint.java
@@ -19,37 +19,126 @@
 package org.eclipse.jetty.io;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
 import java.nio.ByteBuffer;
 import java.nio.channels.ByteChannel;
-import java.nio.channels.SocketChannel;
+import java.nio.channels.CancelledKeyException;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.SelectionKey;
 
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.ExecutionStrategy.Rejectable;
+import org.eclipse.jetty.util.thread.Locker;
 import org.eclipse.jetty.util.thread.Scheduler;
 
 /**
  * Channel End Point.
  * <p>Holds the channel and socket for an NIO endpoint.
  */
-public class ChannelEndPoint extends AbstractEndPoint
+public abstract class ChannelEndPoint extends AbstractEndPoint implements ManagedSelector.Selectable
 {
     private static final Logger LOG = Log.getLogger(ChannelEndPoint.class);
 
-    private final SocketChannel _channel;
-    private final Socket _socket;
-    private volatile boolean _ishut;
-    private volatile boolean _oshut;
+    private final Locker _locker = new Locker();
+    private final ByteChannel _channel;
+    private final GatheringByteChannel _gather;
+    protected final ManagedSelector _selector;
+    protected final SelectionKey _key;
 
-    public ChannelEndPoint(Scheduler scheduler,SocketChannel channel)
+    private boolean _updatePending;
+
+    /**
+     * The current value for {@link SelectionKey#interestOps()}.
+     */
+    protected int _currentInterestOps;
+
+    /**
+     * The desired value for {@link SelectionKey#interestOps()}.
+     */
+    protected int _desiredInterestOps;
+
+    
+    private abstract class RunnableTask  implements Runnable
     {
-        super(scheduler,
-            (InetSocketAddress)channel.socket().getLocalSocketAddress(),
-            (InetSocketAddress)channel.socket().getRemoteSocketAddress());
+        final String _operation;
+        RunnableTask(String op)
+        {
+            _operation=op;
+        }
+        
+        @Override
+        public String toString()
+        {
+            return ChannelEndPoint.this.toString()+":"+_operation;
+        }
+    }
+    
+    private abstract class RejectableRunnable extends RunnableTask implements Rejectable
+    {
+        RejectableRunnable(String op)
+        {
+            super(op);
+        }
+
+        @Override 
+        public void reject()
+        {
+            try
+            {
+                close();
+            }
+            catch (Throwable x)
+            {
+                LOG.warn(x);
+            }
+        }
+    }
+    
+    private final Runnable _runUpdateKey = new RunnableTask("runUpdateKey")
+    {
+        @Override
+        public void run()
+        {
+            updateKey();
+        }
+    };
+
+    private final Runnable _runFillable = new RejectableRunnable("runFillable")
+    {
+        @Override
+        public void run()
+        {
+            getFillInterest().fillable();
+        }
+    };
+
+    private final Runnable _runCompleteWrite = new RejectableRunnable("runCompleteWrite")
+    {
+        @Override
+        public void run()
+        {
+            getWriteFlusher().completeWrite();
+        }
+    };
+
+    private final Runnable _runFillableCompleteWrite = new RejectableRunnable("runFillableCompleteWrite")
+    {
+        @Override
+        public void run()
+        {
+            getFillInterest().fillable();
+            getWriteFlusher().completeWrite();
+        }
+    };
+
+    public ChannelEndPoint(ByteChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
+    {
+        super(scheduler);
         _channel=channel;
-        _socket=channel.socket();
+        _selector=selector;
+        _key=key;
+        _gather=(channel instanceof GatheringByteChannel)?(GatheringByteChannel)channel:null;
     }
 
     @Override
@@ -64,60 +153,11 @@
         return _channel.isOpen();
     }
 
-    protected void shutdownInput()
+    @Override
+    public void doClose()
     {
         if (LOG.isDebugEnabled())
-            LOG.debug("ishut {}", this);
-        _ishut=true;
-        if (_oshut)
-            close();
-    }
-
-    @Override
-    public void shutdownOutput()
-    {
-        if (LOG.isDebugEnabled())
-            LOG.debug("oshut {}", this);
-        _oshut = true;
-        if (_channel.isOpen())
-        {
-            try
-            {
-                if (!_socket.isOutputShutdown())
-                    _socket.shutdownOutput();
-            }
-            catch (IOException e)
-            {
-                LOG.debug(e);
-            }
-            finally
-            {
-                if (_ishut)
-                {
-                    close();
-                }
-            }
-        }
-    }
-
-    @Override
-    public boolean isOutputShutdown()
-    {
-        return _oshut || !_channel.isOpen() || _socket.isOutputShutdown();
-    }
-
-    @Override
-    public boolean isInputShutdown()
-    {
-        return _ishut || !_channel.isOpen() || _socket.isInputShutdown();
-    }
-
-    @Override
-    public void close()
-    {
-        super.close();
-        if (LOG.isDebugEnabled())
-            LOG.debug("close {}", this);
+            LOG.debug("doClose {}", this);
         try
         {
             _channel.close();
@@ -128,15 +168,29 @@
         }
         finally
         {
-            _ishut=true;
-            _oshut=true;
+            super.doClose();
         }
     }
+    
+    @Override
+    public void onClose()
+    {
+        try
+        {
+            super.onClose();
+        }
+        finally
+        {
+            if (_selector!=null)
+                _selector.onClose(this);
+        }
+    }
+    
 
     @Override
     public int fill(ByteBuffer buffer) throws IOException
     {
-        if (_ishut)
+        if (isInputShutdown())
             return -1;
 
         int pos=BufferUtil.flipToFill(buffer);
@@ -173,8 +227,8 @@
         {
             if (buffers.length==1)
                 flushed=_channel.write(buffers[0]);
-            else if (buffers.length>1)
-                flushed=_channel.write(buffers,0,buffers.length);
+            else if (_gather!=null && buffers.length>1)
+                flushed=_gather.write(buffers,0,buffers.length);
             else
             {
                 for (ByteBuffer b : buffers)
@@ -218,20 +272,160 @@
         return _channel;
     }
 
-    public Socket getSocket()
+
+    @Override
+    protected void needsFillInterest()
     {
-        return _socket;
+        changeInterests(SelectionKey.OP_READ);
     }
 
     @Override
     protected void onIncompleteFlush()
     {
-        throw new UnsupportedOperationException();
+        changeInterests(SelectionKey.OP_WRITE);
     }
 
     @Override
-    protected void needsFillInterest() throws IOException
+    public Runnable onSelected()
     {
-        throw new UnsupportedOperationException();
+        /**
+         * This method may run concurrently with {@link #changeInterests(int)}.
+         */
+    
+        int readyOps = _key.readyOps();
+        int oldInterestOps;
+        int newInterestOps;
+        try (Locker.Lock lock = _locker.lock())
+        {
+            _updatePending = true;
+            // Remove the readyOps, that here can only be OP_READ or OP_WRITE (or both).
+            oldInterestOps = _desiredInterestOps;
+            newInterestOps = oldInterestOps & ~readyOps;
+            _desiredInterestOps = newInterestOps;
+        }
+    
+    
+        boolean readable = (readyOps & SelectionKey.OP_READ) != 0;
+        boolean writable = (readyOps & SelectionKey.OP_WRITE) != 0;
+    
+    
+        if (LOG.isDebugEnabled())
+            LOG.debug("onSelected {}->{} r={} w={} for {}", oldInterestOps, newInterestOps, readable, writable, this);
+        
+        // Run non-blocking code immediately.
+        // This producer knows that this non-blocking code is special
+        // and that it must be run in this thread and not fed to the
+        // ExecutionStrategy, which could not have any thread to run these
+        // tasks (or it may starve forever just after having run them).
+        if (readable && getFillInterest().isCallbackNonBlocking())
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Direct readable run {}",this);
+            _runFillable.run();
+            readable = false;
+        }
+        if (writable && getWriteFlusher().isCallbackNonBlocking())
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Direct writable run {}",this);
+            _runCompleteWrite.run();
+            writable = false;
+        }
+    
+        // return task to complete the job
+        Runnable task= readable ? (writable ? _runFillableCompleteWrite : _runFillable)
+                : (writable ? _runCompleteWrite : null);
+    
+        if (LOG.isDebugEnabled())
+            LOG.debug("task {}",task);
+        return task;
     }
+
+    @Override
+    public void updateKey()
+    {
+        /**
+         * This method may run concurrently with {@link #changeInterests(int)}.
+         */
+    
+        try
+        {
+            int oldInterestOps;
+            int newInterestOps;
+            try (Locker.Lock lock = _locker.lock())
+            {
+                _updatePending = false;
+                oldInterestOps = _currentInterestOps;
+                newInterestOps = _desiredInterestOps;
+                if (oldInterestOps != newInterestOps)
+                {
+                    _currentInterestOps = newInterestOps;
+                    _key.interestOps(newInterestOps);
+                }
+            }
+    
+            if (LOG.isDebugEnabled())
+                LOG.debug("Key interests updated {} -> {} on {}", oldInterestOps, newInterestOps, this);
+        }
+        catch (CancelledKeyException x)
+        {
+            LOG.debug("Ignoring key update for concurrently closed channel {}", this);
+            close();
+        }
+        catch (Throwable x)
+        {
+            LOG.warn("Ignoring key update for " + this, x);
+            close();
+        }
+    }
+
+    private void changeInterests(int operation)
+    {
+        /**
+         * This method may run concurrently with
+         * {@link #updateKey()} and {@link #onSelected()}.
+         */
+    
+        int oldInterestOps;
+        int newInterestOps;
+        boolean pending;
+        try (Locker.Lock lock = _locker.lock())
+        {
+            pending = _updatePending;
+            oldInterestOps = _desiredInterestOps;
+            newInterestOps = oldInterestOps | operation;
+            if (newInterestOps != oldInterestOps)
+                _desiredInterestOps = newInterestOps;
+        }
+    
+        if (LOG.isDebugEnabled())
+            LOG.debug("changeInterests p={} {}->{} for {}", pending, oldInterestOps, newInterestOps, this);
+    
+        if (!pending && _selector!=null)
+            _selector.submit(_runUpdateKey);
+    }
+    
+
+    @Override
+    public String toString()
+    {
+        // We do a best effort to print the right toString() and that's it.
+        try
+        {
+            boolean valid = _key != null && _key.isValid();
+            int keyInterests = valid ? _key.interestOps() : -1;
+            int keyReadiness = valid ? _key.readyOps() : -1;
+            return String.format("%s{io=%d/%d,kio=%d,kro=%d}",
+                    super.toString(),
+                    _currentInterestOps,
+                    _desiredInterestOps,
+                    keyInterests,
+                    keyReadiness);
+        }
+        catch (Throwable x)
+        {
+            return String.format("%s{io=%s,kio=-2,kro=-2}", super.toString(), _desiredInterestOps);
+        }
+    }
+    
 }
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/Connection.java b/jetty-io/src/main/java/org/eclipse/jetty/io/Connection.java
index 564493c..cf65024 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/Connection.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/Connection.java
@@ -63,7 +63,7 @@
      * @return the {@link EndPoint} associated with this {@link Connection}
      */
     public EndPoint getEndPoint();
-
+    
     /**
      * <p>Performs a logical close of this connection.</p>
      * <p>For simple connections, this may just mean to delegate the close to the associated
@@ -91,8 +91,8 @@
     public long getBytesIn();
     public long getBytesOut();
     public long getCreatedTimeStamp();
-
-    public interface UpgradeFrom extends Connection
+    
+    public interface UpgradeFrom
     {
         /**
          * <p>Takes the input buffer from the connection on upgrade.</p>
@@ -104,8 +104,8 @@
          */
         ByteBuffer onUpgradeFrom();
     }
-
-    public interface UpgradeTo extends Connection
+    
+    public interface UpgradeTo
     {
         /**
          * <p>Callback method invoked when this connection is upgraded.</p>
@@ -117,8 +117,8 @@
          */
         void onUpgradeTo(ByteBuffer prefilled);
     }
-
-    /**
+    
+    /** 
      * <p>A Listener for connection events.</p>
      * <p>Listeners can be added to a {@link Connection} to get open and close events.
      * The AbstractConnectionFactory implements a pattern where objects implement
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/EndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/EndPoint.java
index 8f285c3..68f795c 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/EndPoint.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/EndPoint.java
@@ -94,7 +94,7 @@
  * </pre></blockquote>
  */
 public interface EndPoint extends Closeable
-{
+{    
     /* ------------------------------------------------------------ */
     /**
      * @return The local Inet address to which this <code>EndPoint</code> is bound, or <code>null</code>
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/FillInterest.java b/jetty-io/src/main/java/org/eclipse/jetty/io/FillInterest.java
index a00a893..2ccb741 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/FillInterest.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/FillInterest.java
@@ -131,6 +131,8 @@
     public void onClose()
     {
         Callback callback = _interested.get();
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} onClose {}",this,callback);
         if (callback != null && _interested.compareAndSet(callback, null))
             callback.failed(new ClosedChannelException());
     }
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java b/jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java
index 0af6827..40d5a01 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java
@@ -23,10 +23,9 @@
 import java.net.ConnectException;
 import java.net.SocketTimeoutException;
 import java.nio.channels.CancelledKeyException;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.Selector;
-import java.nio.channels.ServerSocketChannel;
-import java.nio.channels.SocketChannel;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -78,12 +77,7 @@
     protected void doStart() throws Exception
     {
         super.doStart();
-        _selector = newSelector();
-    }
-
-    protected Selector newSelector() throws IOException
-    {
-        return Selector.open();
+        _selector = _selectorManager.newSelector();
     }
 
     public int size()
@@ -138,10 +132,10 @@
     }
 
     /**
-     * A {@link SelectableEndPoint} is an {@link EndPoint} that wish to be
+     * A {@link Selectable} is an {@link EndPoint} that wish to be
      * notified of non-blocking events by the {@link ManagedSelector}.
      */
-    public interface SelectableEndPoint extends EndPoint
+    public interface Selectable 
     {
         /**
          * Callback method invoked when a read or write events has been
@@ -265,12 +259,14 @@
                 if (key.isValid())
                 {
                     Object attachment = key.attachment();
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("selected {} {} ",key,attachment);
                     try
                     {
-                        if (attachment instanceof SelectableEndPoint)
+                        if (attachment instanceof Selectable)
                         {
                             // Try to produce a task
-                            Runnable task = ((SelectableEndPoint)attachment).onSelected();
+                            Runnable task = ((Selectable)attachment).onSelected();
                             if (task != null)
                                 return task;
                         }
@@ -324,8 +320,8 @@
         private void updateKey(SelectionKey key)
         {
             Object attachment = key.attachment();
-            if (attachment instanceof SelectableEndPoint)
-                ((SelectableEndPoint)attachment).updateKey();
+            if (attachment instanceof Selectable)
+                ((Selectable)attachment).updateKey();
         }
     }
 
@@ -335,11 +331,11 @@
 
     private Runnable processConnect(SelectionKey key, final Connect connect)
     {
-        SocketChannel channel = (SocketChannel)key.channel();
+        SelectableChannel channel = (SelectableChannel)key.channel();
         try
         {
             key.attach(connect.attachment);
-            boolean connected = _selectorManager.finishConnect(channel);
+            boolean connected = _selectorManager.doFinishConnect(channel);
             if (LOG.isDebugEnabled())
                 LOG.debug("Connected {} {}", connected, channel);
             if (connected)
@@ -376,14 +372,13 @@
 
     private void processAccept(SelectionKey key)
     {
-        ServerSocketChannel server = (ServerSocketChannel)key.channel();
-        SocketChannel channel = null;
+        SelectableChannel server = key.channel();
+        SelectableChannel channel = null;
         try
         {
-            while ((channel = server.accept()) != null)
-            {
+            channel = _selectorManager.doAccept(server);
+            if (channel!=null)
                 _selectorManager.accepted(channel);
-            }
         }
         catch (Throwable x)
         {
@@ -405,7 +400,7 @@
         }
     }
 
-    private EndPoint createEndPoint(SocketChannel channel, SelectionKey selectionKey) throws IOException
+    private EndPoint createEndPoint(SelectableChannel channel, SelectionKey selectionKey) throws IOException
     {
         EndPoint endPoint = _selectorManager.newEndPoint(channel, this, selectionKey);
         _selectorManager.endPointOpened(endPoint);
@@ -418,7 +413,7 @@
         return endPoint;
     }
 
-    public void destroyEndPoint(final EndPoint endPoint)
+    public void onClose(final EndPoint endPoint)
     {
         final Connection connection = endPoint.getConnection();
         submit(new Product()
@@ -518,9 +513,9 @@
 
     class Acceptor implements Runnable
     {
-        private final ServerSocketChannel _channel;
+        private final SelectableChannel _channel;
 
-        public Acceptor(ServerSocketChannel channel)
+        public Acceptor(SelectableChannel channel)
         {
             this._channel = channel;
         }
@@ -544,10 +539,10 @@
 
     class Accept implements Runnable, Rejectable
     {
-        private final SocketChannel channel;
+        private final SelectableChannel channel;
         private final Object attachment;
 
-        Accept(SocketChannel channel, Object attachment)
+        Accept(SelectableChannel channel, Object attachment)
         {
             this.channel = channel;
             this.attachment = attachment;
@@ -578,10 +573,10 @@
 
     private class CreateEndPoint implements Product, Rejectable
     {
-        private final SocketChannel channel;
+        private final SelectableChannel channel;
         private final SelectionKey key;
 
-        public CreateEndPoint(SocketChannel channel, SelectionKey key)
+        public CreateEndPoint(SelectableChannel channel, SelectionKey key)
         {
             this.channel = channel;
             this.key = key;
@@ -618,11 +613,11 @@
     class Connect implements Runnable
     {
         private final AtomicBoolean failed = new AtomicBoolean();
-        private final SocketChannel channel;
+        private final SelectableChannel channel;
         private final Object attachment;
         private final Scheduler.Task timeout;
 
-        Connect(SocketChannel channel, Object attachment)
+        Connect(SelectableChannel channel, Object attachment)
         {
             this.channel = channel;
             this.attachment = attachment;
@@ -665,8 +660,8 @@
         @Override
         public void run()
         {
-            SocketChannel channel = connect.channel;
-            if (channel.isConnectionPending())
+            SelectableChannel channel = connect.channel;
+            if (_selectorManager.isConnectionPending(channel))
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("Channel {} timed out while connecting, closing it", channel);
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/SelectChannelEndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/SelectChannelEndPoint.java
index 435c816..11ca41f 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/SelectChannelEndPoint.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/SelectChannelEndPoint.java
@@ -18,303 +18,24 @@
 
 package org.eclipse.jetty.io;
 
-import java.nio.channels.CancelledKeyException;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
-import java.nio.channels.SocketChannel;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.ExecutionStrategy.Rejectable;
-import org.eclipse.jetty.util.thread.Locker;
 import org.eclipse.jetty.util.thread.Scheduler;
 
 /**
  * An ChannelEndpoint that can be scheduled by {@link SelectorManager}.
  */
-public class SelectChannelEndPoint extends ChannelEndPoint implements ManagedSelector.SelectableEndPoint
+@Deprecated
+public class SelectChannelEndPoint extends SocketChannelEndPoint implements ManagedSelector.Selectable
 {
     public static final Logger LOG = Log.getLogger(SelectChannelEndPoint.class);
 
-    private final Locker _locker = new Locker();
-    private boolean _updatePending;
-
-    /**
-     * true if {@link ManagedSelector#destroyEndPoint(EndPoint)} has not been called
-     */
-    private final AtomicBoolean _open = new AtomicBoolean();
-    private final ManagedSelector _selector;
-    private final SelectionKey _key;
-    /**
-     * The current value for {@link SelectionKey#interestOps()}.
-     */
-    private int _currentInterestOps;
-    /**
-     * The desired value for {@link SelectionKey#interestOps()}.
-     */
-    private int _desiredInterestOps;
-
-    private final Runnable _runUpdateKey = new Runnable()
+    public SelectChannelEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler, long idleTimeout)
     {
-        @Override
-        public void run()
-        {
-            updateKey();
-        }
-
-        @Override
-        public String toString()
-        {
-            return SelectChannelEndPoint.this.toString()+":runUpdateKey";
-        }
-    };
-    
-    private abstract class RejectableRunnable implements Runnable,Rejectable
-    {
-        @Override 
-        public void reject()
-        {
-            try
-            {
-                close();
-            }
-            catch (Throwable x)
-            {
-                LOG.warn(x);
-            }
-        }
-    }
-    
-    private final Runnable _runFillable = new RejectableRunnable()
-    {
-        @Override
-        public void run()
-        {
-            getFillInterest().fillable();
-        }
-
-        @Override
-        public String toString()
-        {
-            return SelectChannelEndPoint.this.toString()+":runFillable";
-        }
-    };
-    private final Runnable _runCompleteWrite = new RejectableRunnable()
-    {
-        @Override
-        public void run()
-        {
-            getWriteFlusher().completeWrite();
-        }
-
-        @Override
-        public String toString()
-        {
-            return SelectChannelEndPoint.this.toString()+":runCompleteWrite";
-        }
-    };
-    private final Runnable _runFillableCompleteWrite = new RejectableRunnable()
-    {
-        @Override
-        public void run()
-        {
-            getFillInterest().fillable();
-            getWriteFlusher().completeWrite();
-        }
-
-        @Override
-        public String toString()
-        {
-            return SelectChannelEndPoint.this.toString()+":runFillableCompleteWrite";
-        }
-    };
-
-    public SelectChannelEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler, long idleTimeout)
-    {
-        super(scheduler, channel);
-        _selector = selector;
-        _key = key;
+        super(channel,selector,key,scheduler);
         setIdleTimeout(idleTimeout);
     }
-
-    @Override
-    protected void needsFillInterest()
-    {
-        changeInterests(SelectionKey.OP_READ);
-    }
-
-    @Override
-    protected void onIncompleteFlush()
-    {
-        changeInterests(SelectionKey.OP_WRITE);
-    }
-
-    @Override
-    public Runnable onSelected()
-    {
-        /**
-         * This method may run concurrently with {@link #changeInterests(int)}.
-         */
-
-        int readyOps = _key.readyOps();
-        int oldInterestOps;
-        int newInterestOps;
-        try (Locker.Lock lock = _locker.lock())
-        {
-            _updatePending = true;
-            // Remove the readyOps, that here can only be OP_READ or OP_WRITE (or both).
-            oldInterestOps = _desiredInterestOps;
-            newInterestOps = oldInterestOps & ~readyOps;
-            _desiredInterestOps = newInterestOps;
-        }
-
-
-        boolean readable = (readyOps & SelectionKey.OP_READ) != 0;
-        boolean writable = (readyOps & SelectionKey.OP_WRITE) != 0;
-
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("onSelected {}->{} r={} w={} for {}", oldInterestOps, newInterestOps, readable, writable, this);
-        
-        // Run non-blocking code immediately.
-        // This producer knows that this non-blocking code is special
-        // and that it must be run in this thread and not fed to the
-        // ExecutionStrategy, which could not have any thread to run these
-        // tasks (or it may starve forever just after having run them).
-        if (readable && getFillInterest().isCallbackNonBlocking())
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Direct readable run {}",this);
-            _runFillable.run();
-            readable = false;
-        }
-        if (writable && getWriteFlusher().isCallbackNonBlocking())
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Direct writable run {}",this);
-            _runCompleteWrite.run();
-            writable = false;
-        }
-
-        // return task to complete the job
-        Runnable task= readable ? (writable ? _runFillableCompleteWrite : _runFillable)
-                : (writable ? _runCompleteWrite : null);
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("task {}",task);
-        return task;
-    }
-
-    @Override
-    public void updateKey()
-    {
-        /**
-         * This method may run concurrently with {@link #changeInterests(int)}.
-         */
-
-        try
-        {
-            int oldInterestOps;
-            int newInterestOps;
-            try (Locker.Lock lock = _locker.lock())
-            {
-                _updatePending = false;
-                oldInterestOps = _currentInterestOps;
-                newInterestOps = _desiredInterestOps;
-                if (oldInterestOps != newInterestOps)
-                {
-                    _currentInterestOps = newInterestOps;
-                    _key.interestOps(newInterestOps);
-                }
-            }
-
-            if (LOG.isDebugEnabled())
-                LOG.debug("Key interests updated {} -> {} on {}", oldInterestOps, newInterestOps, this);
-        }
-        catch (CancelledKeyException x)
-        {
-            LOG.debug("Ignoring key update for concurrently closed channel {}", this);
-            close();
-        }
-        catch (Throwable x)
-        {
-            LOG.warn("Ignoring key update for " + this, x);
-            close();
-        }
-    }
-
-    private void changeInterests(int operation)
-    {
-        /**
-         * This method may run concurrently with
-         * {@link #updateKey()} and {@link #onSelected()}.
-         */
-
-        int oldInterestOps;
-        int newInterestOps;
-        boolean pending;
-        try (Locker.Lock lock = _locker.lock())
-        {
-            pending = _updatePending;
-            oldInterestOps = _desiredInterestOps;
-            newInterestOps = oldInterestOps | operation;
-            if (newInterestOps != oldInterestOps)
-                _desiredInterestOps = newInterestOps;
-        }
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("changeInterests p={} {}->{} for {}", pending, oldInterestOps, newInterestOps, this);
-
-        if (!pending)
-            _selector.submit(_runUpdateKey);
-    }
-
-
-    @Override
-    public void close()
-    {
-        if (_open.compareAndSet(true, false))
-        {
-            super.close();
-            _selector.destroyEndPoint(this);
-        }
-    }
-
-    @Override
-    public boolean isOpen()
-    {
-        // We cannot rely on super.isOpen(), because there is a race between calls to close() and isOpen():
-        // a thread may call close(), which flips the boolean but has not yet called super.close(), and
-        // another thread calls isOpen() which would return true - wrong - if based on super.isOpen().
-        return _open.get();
-    }
-
-    @Override
-    public void onOpen()
-    {
-        if (_open.compareAndSet(false, true))
-            super.onOpen();
-    }
-
-    @Override
-    public String toString()
-    {
-        // We do a best effort to print the right toString() and that's it.
-        try
-        {
-            boolean valid = _key != null && _key.isValid();
-            int keyInterests = valid ? _key.interestOps() : -1;
-            int keyReadiness = valid ? _key.readyOps() : -1;
-            return String.format("%s{io=%d/%d,kio=%d,kro=%d}",
-                    super.toString(),
-                    _currentInterestOps,
-                    _desiredInterestOps,
-                    keyInterests,
-                    keyReadiness);
-        }
-        catch (Throwable x)
-        {
-            return String.format("%s{io=%s,kio=-2,kro=-2}", super.toString(), _desiredInterestOps);
-        }
-    }
 }
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java b/jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java
index d13b8ac..53631ee 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java
@@ -22,7 +22,9 @@
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketAddress;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
 import java.util.concurrent.Executor;
@@ -133,7 +135,7 @@
         return _selectors.length;
     }
 
-    private ManagedSelector chooseSelector(SocketChannel channel)
+    private ManagedSelector chooseSelector(SelectableChannel channel)
     {
         // Ideally we would like to have all connections from the same client end
         // up on the same selector (to try to avoid smearing the data from a single 
@@ -145,14 +147,17 @@
         {
             try
             {
-                SocketAddress remote = channel.getRemoteAddress();
-                if (remote instanceof InetSocketAddress)
+                if (channel instanceof SocketChannel)
                 {
-                    byte[] addr = ((InetSocketAddress)remote).getAddress().getAddress();
-                    if (addr != null)
+                    SocketAddress remote = ((SocketChannel)channel).getRemoteAddress();
+                    if (remote instanceof InetSocketAddress)
                     {
-                        int s = addr[addr.length - 1] & 0xFF;
-                        candidate1 = _selectors[s % getSelectorCount()];
+                        byte[] addr = ((InetSocketAddress)remote).getAddress().getAddress();
+                        if (addr != null)
+                        {
+                            int s = addr[addr.length - 1] & 0xFF;
+                            candidate1 = _selectors[s % getSelectorCount()];
+                        }
                     }
                 }
             }
@@ -182,9 +187,9 @@
      *
      * @param channel    the channel to register
      * @param attachment the attachment object
-     * @see #accept(SocketChannel, Object)
+     * @see #accept(SelectableChannel, Object)
      */
-    public void connect(SocketChannel channel, Object attachment)
+    public void connect(SelectableChannel channel, Object attachment)
     {
         ManagedSelector set = chooseSelector(channel);
         set.submit(set.new Connect(channel, attachment));
@@ -192,9 +197,9 @@
 
     /**
      * @param channel the channel to accept
-     * @see #accept(SocketChannel, Object)
+     * @see #accept(SelectableChannel, Object)
      */
-    public void accept(SocketChannel channel)
+    public void accept(SelectableChannel channel)
     {
         accept(channel, null);
     }
@@ -209,7 +214,7 @@
      * @param channel    the channel to register
      * @param attachment the attachment object
      */
-    public void accept(SocketChannel channel, Object attachment)
+    public void accept(SelectableChannel channel, Object attachment)
     {
         final ManagedSelector selector = chooseSelector(channel);
         selector.submit(selector.new Accept(channel, attachment));
@@ -218,12 +223,12 @@
     /**
      * <p>Registers a server channel for accept operations.
      * When a {@link SocketChannel} is accepted from the given {@link ServerSocketChannel}
-     * then the {@link #accepted(SocketChannel)} method is called, which must be
+     * then the {@link #accepted(SelectableChannel)} method is called, which must be
      * overridden by a derivation of this class to handle the accepted channel
      *
      * @param server the server channel to register
      */
-    public void acceptor(ServerSocketChannel server)
+    public void acceptor(SelectableChannel server)
     {
         final ManagedSelector selector = chooseSelector(null);
         selector.submit(selector.new Acceptor(server));
@@ -231,14 +236,14 @@
 
     /**
      * Callback method when a channel is accepted from the {@link ServerSocketChannel}
-     * passed to {@link #acceptor(ServerSocketChannel)}.
+     * passed to {@link #acceptor(SelectableChannel)}.
      * The default impl throws an {@link UnsupportedOperationException}, so it must
      * be overridden by subclasses if a server channel is provided.
      *
      * @param channel the
      * @throws IOException if unable to accept channel
      */
-    protected void accepted(SocketChannel channel) throws IOException
+    protected void accepted(SelectableChannel channel) throws IOException
     {
         throw new UnsupportedOperationException();
     }
@@ -292,7 +297,6 @@
      */
     protected void endPointClosed(EndPoint endpoint)
     {
-        endpoint.onClose();
     }
 
     /**
@@ -332,10 +336,21 @@
         }
     }
 
-    protected boolean finishConnect(SocketChannel channel) throws IOException
+    protected boolean doFinishConnect(SelectableChannel channel) throws IOException
     {
-        return channel.finishConnect();
+        return ((SocketChannel)channel).finishConnect();
     }
+    
+    protected boolean isConnectionPending(SelectableChannel channel)
+    {
+        return ((SocketChannel)channel).isConnectionPending();
+    }
+    
+    protected SelectableChannel doAccept(SelectableChannel server) throws IOException
+    {
+        return ((ServerSocketChannel)server).accept();
+    }
+
 
     /**
      * <p>Callback method invoked when a non-blocking connect cannot be completed.</p>
@@ -345,24 +360,29 @@
      * @param ex         the exception that caused the connect to fail
      * @param attachment the attachment object associated at registration
      */
-    protected void connectionFailed(SocketChannel channel, Throwable ex, Object attachment)
+    protected void connectionFailed(SelectableChannel channel, Throwable ex, Object attachment)
     {
         LOG.warn(String.format("%s - %s", channel, attachment), ex);
     }
 
+    protected Selector newSelector() throws IOException
+    {
+        return Selector.open();
+    }
+    
     /**
      * <p>Factory method to create {@link EndPoint}.</p>
-     * <p>This method is invoked as a result of the registration of a channel via {@link #connect(SocketChannel, Object)}
-     * or {@link #accept(SocketChannel)}.</p>
+     * <p>This method is invoked as a result of the registration of a channel via {@link #connect(SelectableChannel, Object)}
+     * or {@link #accept(SelectableChannel)}.</p>
      *
      * @param channel      the channel associated to the endpoint
      * @param selector     the selector the channel is registered to
      * @param selectionKey the selection key
      * @return a new endpoint
      * @throws IOException if the endPoint cannot be created
-     * @see #newConnection(SocketChannel, EndPoint, Object)
+     * @see #newConnection(SelectableChannel, EndPoint, Object)
      */
-    protected abstract EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException;
+    protected abstract EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException;
 
     /**
      * <p>Factory method to create {@link Connection}.</p>
@@ -372,9 +392,8 @@
      * @param attachment the attachment
      * @return a new connection
      * @throws IOException if unable to create new connection
-     * @see #newEndPoint(SocketChannel, ManagedSelector, SelectionKey)
      */
-    public abstract Connection newConnection(SocketChannel channel, EndPoint endpoint, Object attachment) throws IOException;
+    public abstract Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment) throws IOException;
 
     @Override
     public String dump()
@@ -388,4 +407,5 @@
         ContainerLifeCycle.dumpObject(out, this);
         ContainerLifeCycle.dump(out, indent, TypeUtil.asList(_selectors));
     }
+    
 }
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java
new file mode 100644
index 0000000..2c92498
--- /dev/null
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java
@@ -0,0 +1,81 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.io;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.Scheduler;
+
+public class SocketChannelEndPoint extends ChannelEndPoint
+{
+    private static final Logger LOG = Log.getLogger(SocketChannelEndPoint.class);
+    private final Socket _socket;
+    private final InetSocketAddress _local;
+    private final InetSocketAddress _remote;
+
+    public SocketChannelEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
+    {
+        this((SocketChannel)channel,selector,key,scheduler);
+    }
+    
+    public SocketChannelEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
+    {
+        super(channel,selector,key,scheduler);
+        
+        _socket=channel.socket();
+        _local=(InetSocketAddress)_socket.getLocalSocketAddress();
+        _remote=(InetSocketAddress)_socket.getRemoteSocketAddress();
+    }
+
+    public Socket getSocket()
+    {
+        return _socket;
+    }
+
+    public InetSocketAddress getLocalAddress()
+    {
+        return _local;
+    }
+
+    public InetSocketAddress getRemoteAddress()
+    {
+        return _remote;
+    }
+    
+    @Override
+    protected void doShutdownOutput()
+    {
+        try
+        {
+            if (!_socket.isOutputShutdown())
+                _socket.shutdownOutput();
+        }
+        catch (IOException e)
+        {
+            LOG.debug(e);
+        }
+    }
+}
diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java b/jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java
index 50cfa92..9916ddc 100644
--- a/jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java
+++ b/jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java
@@ -19,6 +19,7 @@
 package org.eclipse.jetty.io.ssl;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException;
 import java.util.concurrent.Executor;
@@ -334,7 +335,7 @@
         public DecryptedEndPoint()
         {
             // Disable idle timeout checking: no scheduler and -1 timeout for this instance.
-            super(null, getEndPoint().getLocalAddress(), getEndPoint().getRemoteAddress());
+            super(null);
             super.setIdleTimeout(-1);
         }
 
@@ -357,6 +358,18 @@
         }
 
         @Override
+        public InetSocketAddress getLocalAddress()
+        {
+            return getEndPoint().getLocalAddress();
+        }
+
+        @Override
+        public InetSocketAddress getRemoteAddress()
+        {
+            return getEndPoint().getRemoteAddress();
+        }
+
+        @Override
         protected WriteFlusher getWriteFlusher()
         {
             return super.getWriteFlusher();
@@ -885,12 +898,11 @@
         }
 
         @Override
-        public void shutdownOutput()
+        public void doShutdownOutput()
         {
             boolean ishut = isInputShutdown();
-            boolean oshut = isOutputShutdown();
             if (LOG.isDebugEnabled())
-                LOG.debug("{} shutdownOutput: oshut={}, ishut={}", SslConnection.this, oshut, ishut);
+                LOG.debug("{} shutdownOutput: ishut={}", SslConnection.this, ishut);
             if (ishut)
             {
                 // Aggressively close, since inbound close alert has already been processed
@@ -899,7 +911,7 @@
                 // reply. If a TLS close reply is sent, most implementations send a RST.
                 getEndPoint().close();
             }
-            else if (!oshut)
+            else
             {
                 try
                 {
@@ -931,12 +943,27 @@
         }
 
         @Override
-        public void close()
+        public void doClose()
         {
             // First send the TLS Close Alert, then the FIN
-            shutdownOutput();
+            if (!_sslEngine.isOutboundDone())
+            {
+                try
+                {
+                    synchronized (this) // TODO review synchronized boundary
+                    {
+                        _sslEngine.closeOutbound();
+                        flush(BufferUtil.EMPTY_BUFFER); // Send close handshake
+                        ensureFillInterested();
+                    }
+                }
+                catch (Exception e)
+                {
+                    LOG.ignore(e);
+                }
+            }
             getEndPoint().close();
-            super.close();
+            super.doClose();
         }
 
         @Override
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/IOTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/IOTest.java
index 7aec534..3a65a07 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/IOTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/IOTest.java
@@ -18,6 +18,11 @@
 
 package org.eclipse.jetty.io;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -45,11 +50,6 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 public class IOTest
 {
     @Test
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointInterestsTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointInterestsTest.java
index 495069c..9bf07ca 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointInterestsTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointInterestsTest.java
@@ -24,6 +24,7 @@
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
@@ -62,10 +63,11 @@
 
         selectorManager = new SelectorManager(threadPool, scheduler)
         {
+
             @Override
-            protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
+            protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key) throws IOException
             {
-                return new SelectChannelEndPoint(channel, selector, selectionKey, getScheduler(), 60000)
+                SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler())
                 {
                     @Override
                     protected void onIncompleteFlush()
@@ -74,10 +76,13 @@
                         interested.onIncompleteFlush();
                     }
                 };
+                        
+                endp.setIdleTimeout(60000);
+                return endp;
             }
 
             @Override
-            public Connection newConnection(SocketChannel channel, final EndPoint endPoint, Object attachment)
+            public Connection newConnection(SelectableChannel channel, final EndPoint endPoint, Object attachment)
             {
                 return new AbstractConnection(endPoint, getExecutor())
                 {
@@ -136,7 +141,7 @@
                         connection.fillInterested();
 
                         ByteBuffer output = ByteBuffer.allocate(size.get());
-                        endPoint.write(new Callback.Adapter(), output);
+                        endPoint.write(new Callback(){}, output);
 
                         latch1.countDown();
                     }
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointSslTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointSslTest.java
index a82fbcd..06bdd7a 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointSslTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointSslTest.java
@@ -26,6 +26,7 @@
 import java.io.IOException;
 import java.net.Socket;
 import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SocketChannel;
 import java.nio.charset.StandardCharsets;
 
@@ -71,7 +72,7 @@
     }
 
     @Override
-    protected Connection newConnection(SocketChannel channel, EndPoint endpoint)
+    protected Connection newConnection(SelectableChannel channel, EndPoint endpoint)
     {
         SSLEngine engine = __sslCtxFactory.newSSLEngine();
         engine.setUseClientMode(false);
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointTest.java
index efcf502..99632fe 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointTest.java
@@ -32,6 +32,7 @@
 import java.net.Socket;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
@@ -69,19 +70,21 @@
     protected SelectorManager _manager = new SelectorManager(_threadPool, _scheduler)
     {
         @Override
-        public Connection newConnection(SocketChannel channel, EndPoint endpoint, Object attachment)
+        public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment)
         {
             return SelectChannelEndPointTest.this.newConnection(channel, endpoint);
         }
 
         @Override
-        protected SelectChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey selectionKey) throws IOException
+        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key) throws IOException
         {
-            SelectChannelEndPoint endp = new SelectChannelEndPoint(channel, selectSet, selectionKey, getScheduler(), 60000);
+            SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler());
+            endp.setIdleTimeout(60000);
             _lastEndPoint = endp;
             _lastEndPointLatch.countDown();
             return endp;
         }
+        
     };
 
     // Must be volatile or the test may fail spuriously
@@ -115,7 +118,7 @@
         return new Socket(_connector.socket().getInetAddress(), _connector.socket().getLocalPort());
     }
 
-    protected Connection newConnection(SocketChannel channel, EndPoint endpoint)
+    protected Connection newConnection(SelectableChannel channel, EndPoint endpoint)
     {
         return new TestConnection(endpoint);
     }
@@ -253,11 +256,11 @@
             }
             catch (InterruptedException | EofException e)
             {
-                SelectChannelEndPoint.LOG.ignore(e);
+                Log.getRootLogger().ignore(e);
             }
             catch (Exception e)
             {
-                SelectChannelEndPoint.LOG.warn(e);
+                Log.getRootLogger().warn(e);
             }
             finally
             {
@@ -709,20 +712,21 @@
         _threadPool = new QueuedThreadPool(4,4,60000,q);
         _manager = new SelectorManager(_threadPool, _scheduler, 1)
         {
-            @Override
-            public Connection newConnection(SocketChannel channel, EndPoint endpoint, Object attachment)
-            {
-                return new TestConnection(endpoint,latch);
-            }
 
             @Override
-            protected SelectChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey selectionKey) throws IOException
+            protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
             {
-                SelectChannelEndPoint endp = new SelectChannelEndPoint(channel, selectSet, selectionKey, getScheduler(), 60000);
+                SocketChannelEndPoint endp = new SocketChannelEndPoint(channel,selector,selectionKey,getScheduler());
                 _lastEndPoint = endp;
                 _lastEndPointLatch.countDown();
                 return endp;
             }
+
+            @Override
+            public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment) throws IOException
+            {
+                return new TestConnection(endpoint,latch);
+            }
         };
         
         _threadPool.start();
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java
index f2c8d3c..6aaff5a 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
@@ -69,20 +70,22 @@
         SelectorManager selectorManager = new SelectorManager(executor, scheduler)
         {
             @Override
-            protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
+            protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key) throws IOException
             {
-                return new SelectChannelEndPoint(channel, selector, selectionKey, getScheduler(), connectTimeout / 2);
+                SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler());
+                endp.setIdleTimeout(connectTimeout/2);
+                return endp;
             }
-
+            
             @Override
-            protected boolean finishConnect(SocketChannel channel) throws IOException
+            protected boolean doFinishConnect(SelectableChannel channel) throws IOException
             {
                 try
                 {
                     long timeout = timeoutConnection.get();
                     if (timeout > 0)
                         TimeUnit.MILLISECONDS.sleep(timeout);
-                    return super.finishConnect(channel);
+                    return super.doFinishConnect(channel);
                 }
                 catch (InterruptedException e)
                 {
@@ -91,7 +94,7 @@
             }
 
             @Override
-            public Connection newConnection(SocketChannel channel, EndPoint endpoint, Object attachment) throws IOException
+            public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment) throws IOException
             {
                 ((Callback)attachment).succeeded();
                 return new AbstractConnection(endpoint, executor)
@@ -104,7 +107,7 @@
             }
 
             @Override
-            protected void connectionFailed(SocketChannel channel, Throwable ex, Object attachment)
+            protected void connectionFailed(SelectableChannel channel, Throwable ex, Object attachment)
             {
                 ((Callback)attachment).failed(ex);
             }
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/ChannelEndPointTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SocketChannelEndPointTest.java
similarity index 73%
rename from jetty-io/src/test/java/org/eclipse/jetty/io/ChannelEndPointTest.java
rename to jetty-io/src/test/java/org/eclipse/jetty/io/SocketChannelEndPointTest.java
index 6a1a397..e6e6af1 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/ChannelEndPointTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SocketChannelEndPointTest.java
@@ -24,7 +24,7 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
-public class ChannelEndPointTest extends EndPointTest<ChannelEndPoint>
+public class SocketChannelEndPointTest extends EndPointTest<SocketChannelEndPoint>
 {
     static ServerSocketChannel connector;
 
@@ -43,16 +43,22 @@
     }
 
     @Override
-    protected EndPointPair<ChannelEndPoint> newConnection() throws Exception
+    protected EndPointPair<SocketChannelEndPoint> newConnection() throws Exception
     {
-        EndPointPair<ChannelEndPoint> c = new EndPointPair<>();
+        EndPointPair<SocketChannelEndPoint> c = new EndPointPair<>();
 
-        c.client=new ChannelEndPoint(null,SocketChannel.open(connector.socket().getLocalSocketAddress()));
-        c.server=new ChannelEndPoint(null,connector.accept());
+        c.client=new SocketChannelEndPoint(SocketChannel.open(connector.socket().getLocalSocketAddress()),null,null,null);
+        c.server=new SocketChannelEndPoint(connector.accept(),null,null,null);
         return c;
     }
 
     @Override
+    public void testClientClose() throws Exception
+    {
+        super.testClientClose();
+    }
+
+    @Override
     public void testClientServerExchange() throws Exception
     {
         super.testClientServerExchange();
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SslConnectionTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SslConnectionTest.java
index 890bbe8..794979e 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/SslConnectionTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SslConnectionTest.java
@@ -24,6 +24,7 @@
 import java.io.InputStreamReader;
 import java.net.Socket;
 import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
@@ -39,6 +40,7 @@
 import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.FutureCallback;
+import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.Scheduler;
@@ -74,7 +76,7 @@
     protected SelectorManager _manager = new SelectorManager(_threadPool, _scheduler)
     {
         @Override
-        public Connection newConnection(SocketChannel channel, EndPoint endpoint, Object attachment)
+        public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment)
         {
             SSLEngine engine = __sslCtxFactory.newSSLEngine();
             engine.setUseClientMode(false);
@@ -85,10 +87,12 @@
             return sslConnection;
         }
 
+
         @Override
-        protected SelectChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey selectionKey) throws IOException
+        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
         {
-            SelectChannelEndPoint endp = new TestEP(channel,selectSet, selectionKey, getScheduler(), 60000);
+            SocketChannelEndPoint endp = new TestEP(channel, selector, selectionKey, getScheduler());
+            endp.setIdleTimeout(60000);
             _lastEndp=endp;
             return endp;
         }
@@ -96,12 +100,11 @@
 
     static final AtomicInteger __startBlocking = new AtomicInteger();
     static final AtomicInteger __blockFor = new AtomicInteger();
-    private static class TestEP extends SelectChannelEndPoint
+    private static class TestEP extends SocketChannelEndPoint
     {
-       
-        public TestEP(SocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler, long idleTimeout)
+        public TestEP(SelectableChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
         {
-            super(channel,selector,key,scheduler,idleTimeout);
+            super((SocketChannel)channel,selector,key,scheduler);
         }
 
         @Override
@@ -121,7 +124,6 @@
                     return false;
                 }
             }
-            String s=BufferUtil.toDetailString(buffers[0]);
             boolean flushed=super.flush(buffers);
             return flushed;
         }
@@ -235,11 +237,11 @@
             }
             catch(InterruptedException|EofException e)
             {
-                SelectChannelEndPoint.LOG.ignore(e);
+                Log.getRootLogger().ignore(e);
             }
             catch(Exception e)
             {
-                SelectChannelEndPoint.LOG.warn(e);
+                Log.getRootLogger().warn(e);
             }
             finally
             {
diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/WriteFlusherTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/WriteFlusherTest.java
index 342a49c..a539ab5 100644
--- a/jetty-io/src/test/java/org/eclipse/jetty/io/WriteFlusherTest.java
+++ b/jetty-io/src/test/java/org/eclipse/jetty/io/WriteFlusherTest.java
@@ -18,15 +18,6 @@
 
 package org.eclipse.jetty.io;
 
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.when;
-
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.WritePendingException;
@@ -59,6 +50,15 @@
 import org.mockito.runners.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
 @RunWith(MockitoJUnitRunner.class)
 public class WriteFlusherTest
 {
@@ -414,7 +414,7 @@
         Arrays.fill(chunk1, (byte)2);
         ByteBuffer buffer2 = ByteBuffer.wrap(chunk2);
 
-        _flusher.write(new Callback.Adapter(), buffer1, buffer2);
+        _flusher.write(Callback.NOOP, buffer1, buffer2);
         assertTrue(_flushIncomplete.get());
         assertFalse(buffer1.hasRemaining());
 
@@ -585,7 +585,7 @@
                     stalled.set(true);
                     return false;
                 }
-                
+
                 // make sure failed is called before we go on
                 try
                 {
@@ -624,15 +624,15 @@
             @Override
             protected void onIncompleteFlush()
             {
-                executor.submit(new Runnable() 
-                { 
-                    public void run() 
+                executor.submit(new Runnable()
+                {
+                    public void run()
                     {
                         try
                         {
                             while(window.get()==0)
                                 window.addAndGet(exchange.exchange(0));
-                            completeWrite(); 
+                            completeWrite();
                         }
                         catch(Throwable th)
                         {
@@ -647,25 +647,25 @@
         BlockingCallback callback = new BlockingCallback();
         writeFlusher.write(callback,BufferUtil.toBuffer("How "),BufferUtil.toBuffer("now "),BufferUtil.toBuffer("brown "),BufferUtil.toBuffer("cow."));
         exchange.exchange(0);
-        
+
         Assert.assertThat(endp.takeOutputString(StandardCharsets.US_ASCII),Matchers.equalTo("How now br"));
-        
+
         exchange.exchange(1);
         exchange.exchange(0);
-        
+
         Assert.assertThat(endp.takeOutputString(StandardCharsets.US_ASCII),Matchers.equalTo("o"));
-        
+
         exchange.exchange(8);
         callback.block();
-        
+
         Assert.assertThat(endp.takeOutputString(StandardCharsets.US_ASCII),Matchers.equalTo("wn cow."));
-        
+
     }
 
     private static class EndPointIterationOnNonBlockedStallMock extends ByteArrayEndPoint
     {
         final AtomicInteger _window;
-        
+
         public EndPointIterationOnNonBlockedStallMock(AtomicInteger window)
         {
             _window=window;
@@ -675,7 +675,7 @@
         public boolean flush(ByteBuffer... buffers) throws IOException
         {
             ByteBuffer byteBuffer = buffers[0];
-            
+
             if (_window.get()>0 && byteBuffer.hasRemaining())
             {
                 // consume 1 byte
@@ -692,7 +692,7 @@
             return true;
         }
     }
-    
+
 
     private static class FailedCaller implements Callable<FutureCallback>
     {
diff --git a/jetty-jaas/pom.xml b/jetty-jaas/pom.xml
index 5b579a0..70fe50f 100644
--- a/jetty-jaas/pom.xml
+++ b/jetty-jaas/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-jaas</artifactId>
diff --git a/jetty-jaas/src/main/config/modules/jaas.mod b/jetty-jaas/src/main/config/modules/jaas.mod
index fee3f59..26c68ff 100644
--- a/jetty-jaas/src/main/config/modules/jaas.mod
+++ b/jetty-jaas/src/main/config/modules/jaas.mod
@@ -1,6 +1,5 @@
-#
-# JAAS Module
-#
+[description]
+Enable JAAS for deployed webapplications.
 
 [depend]
 server
diff --git a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/JAASLoginService.java b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/JAASLoginService.java
index cda28e6..12e2f1e 100644
--- a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/JAASLoginService.java
+++ b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/JAASLoginService.java
@@ -222,7 +222,7 @@
             }
             else
             {
-                Class<?> clazz = Loader.loadClass(getClass(), _callbackHandlerClass);
+                Class<?> clazz = Loader.loadClass(_callbackHandlerClass);
                 callbackHandler = (CallbackHandler)clazz.newInstance();
             }
             //set up the login context
diff --git a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/AbstractLoginModule.java b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/AbstractLoginModule.java
index 5bd9123..001b993 100644
--- a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/AbstractLoginModule.java
+++ b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/AbstractLoginModule.java
@@ -289,6 +289,7 @@
     public boolean logout() throws LoginException
     {
         this.currentUser.unsetJAASInfo(this.subject);
+        this.currentUser = null;
         return true;
     }
 
diff --git a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/JDBCLoginModule.java b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/JDBCLoginModule.java
index 9ee7a01..aa96ad0 100644
--- a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/JDBCLoginModule.java
+++ b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/JDBCLoginModule.java
@@ -103,7 +103,7 @@
                 dbPassword = "";
 
             if (dbDriver != null)
-                Loader.loadClass(this.getClass(), dbDriver).newInstance();
+                Loader.loadClass(dbDriver).newInstance();
         }
         catch (ClassNotFoundException e)
         {
diff --git a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/PropertyFileLoginModule.java b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/PropertyFileLoginModule.java
index 14de803..9fc6304 100644
--- a/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/PropertyFileLoginModule.java
+++ b/jetty-jaas/src/main/java/org/eclipse/jetty/jaas/spi/PropertyFileLoginModule.java
@@ -48,6 +48,8 @@
     private int _refreshInterval = 0;
     private String _filename = DEFAULT_FILENAME;
 
+    
+   
     /**
      * Read contents of the configured property file.
      *
@@ -73,7 +75,6 @@
         {
             PropertyUserStore propertyUserStore = new PropertyUserStore();
             propertyUserStore.setConfig(_filename);
-            propertyUserStore.setRefreshInterval(_refreshInterval);
 
             PropertyUserStore prev = _propertyUserStores.putIfAbsent(_filename, propertyUserStore);
             if (prev == null)
diff --git a/jetty-jaspi/pom.xml b/jetty-jaspi/pom.xml
index 7a2bb5e..db67f58 100644
--- a/jetty-jaspi/pom.xml
+++ b/jetty-jaspi/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-jaspi</artifactId>
diff --git a/jetty-jaspi/src/main/config/modules/jaspi.mod b/jetty-jaspi/src/main/config/modules/jaspi.mod
index e7019ae..0d55273 100644
--- a/jetty-jaspi/src/main/config/modules/jaspi.mod
+++ b/jetty-jaspi/src/main/config/modules/jaspi.mod
@@ -1,6 +1,5 @@
-#
-# Jetty JASPI Module
-#
+[description]
+Enable JASPI authentication for deployed webapplications.
 
 [depend]
 security
diff --git a/jetty-jaspi/src/test/java/org/eclipse/jetty/security/jaspi/JaspiTest.java b/jetty-jaspi/src/test/java/org/eclipse/jetty/security/jaspi/JaspiTest.java
index 892d3ff..006d202 100644
--- a/jetty-jaspi/src/test/java/org/eclipse/jetty/security/jaspi/JaspiTest.java
+++ b/jetty-jaspi/src/test/java/org/eclipse/jetty/security/jaspi/JaspiTest.java
@@ -22,14 +22,16 @@
 import static org.junit.Assert.assertThat;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.eclipse.jetty.security.AbstractLoginService;
 import org.eclipse.jetty.security.ConstraintMapping;
 import org.eclipse.jetty.security.ConstraintSecurityHandler;
-import org.eclipse.jetty.security.HashLoginService;
 import org.eclipse.jetty.server.LocalConnector;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
@@ -38,6 +40,7 @@
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.util.B64Code;
 import org.eclipse.jetty.util.security.Constraint;
+import org.eclipse.jetty.util.security.Credential;
 import org.eclipse.jetty.util.security.Password;
 import org.hamcrest.Matchers;
 import org.junit.After;
@@ -48,6 +51,43 @@
 {
     Server _server;
     LocalConnector _connector;
+    public class TestLoginService extends AbstractLoginService
+    {
+        protected Map<String, UserPrincipal> _users = new HashMap<>();
+        protected Map<String, String[]> _roles = new HashMap();
+     
+      
+
+        public TestLoginService(String name)
+        {
+            setName(name);
+        }
+
+        public void putUser (String username, Credential credential, String[] roles)
+        {
+            UserPrincipal userPrincipal = new UserPrincipal(username,credential);
+            _users.put(username, userPrincipal);
+            _roles.put(username, roles);
+        }
+        
+        /** 
+         * @see org.eclipse.jetty.security.AbstractLoginService#loadRoleInfo(org.eclipse.jetty.security.AbstractLoginService.UserPrincipal)
+         */
+        @Override
+        protected String[] loadRoleInfo(UserPrincipal user)
+        {
+           return _roles.get(user.getName());
+        }
+
+        /** 
+         * @see org.eclipse.jetty.security.AbstractLoginService#loadUserInfo(java.lang.String)
+         */
+        @Override
+        protected UserPrincipal loadUserInfo(String username)
+        {
+            return _users.get(username);
+        }
+    }
     
     @Before
     public void before() throws Exception
@@ -60,7 +100,7 @@
         ContextHandlerCollection contexts = new ContextHandlerCollection();
         _server.setHandler(contexts);
         
-        HashLoginService loginService = new HashLoginService("TestRealm");
+        TestLoginService loginService = new TestLoginService("TestRealm");
         loginService.putUser("user",new Password("password"),new String[]{"users"});
         loginService.putUser("admin",new Password("secret"),new String[]{"users","admins"});
         _server.addBean(loginService);
diff --git a/jetty-jmx/pom.xml b/jetty-jmx/pom.xml
index 5b736be..8c15a72 100644
--- a/jetty-jmx/pom.xml
+++ b/jetty-jmx/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-jmx</artifactId>
diff --git a/jetty-jmx/src/main/config/modules/jmx-remote.mod b/jetty-jmx/src/main/config/modules/jmx-remote.mod
index f8a5111..7a10a01 100644
--- a/jetty-jmx/src/main/config/modules/jmx-remote.mod
+++ b/jetty-jmx/src/main/config/modules/jmx-remote.mod
@@ -1,6 +1,5 @@
-#
-# JMX Remote Module
-#
+[description]
+Enables remote RMI access to JMX
 
 [depend]
 jmx
diff --git a/jetty-jmx/src/main/config/modules/jmx.mod b/jetty-jmx/src/main/config/modules/jmx.mod
index ee091c7..a59c6dd 100644
--- a/jetty-jmx/src/main/config/modules/jmx.mod
+++ b/jetty-jmx/src/main/config/modules/jmx.mod
@@ -1,6 +1,6 @@
-#
-# JMX Module
-#
+[description]
+Enables JMX instrumentation for server beans and 
+enables JMX agent.
 
 [depend]
 server
diff --git a/jetty-jmx/src/main/java/org/eclipse/jetty/jmx/ObjectMBean.java b/jetty-jmx/src/main/java/org/eclipse/jetty/jmx/ObjectMBean.java
index 38cf6f8..a705f5d 100644
--- a/jetty-jmx/src/main/java/org/eclipse/jetty/jmx/ObjectMBean.java
+++ b/jetty-jmx/src/main/java/org/eclipse/jetty/jmx/ObjectMBean.java
@@ -129,8 +129,21 @@
                 String mName = pName + ".jmx." + cName + "MBean";
 
                 try
-                {
-                    Class<?> mClass = (Object.class.equals(oClass))?oClass=ObjectMBean.class:Loader.loadClass(oClass,mName);
+                { 
+                    Class<?> mClass;
+                    try
+                    {
+                        // Look for an MBean class from the same loader that loaded the original class
+                        mClass = (Object.class.equals(oClass))?oClass=ObjectMBean.class:Loader.loadClass(oClass,mName);
+                    }
+                    catch (ClassNotFoundException e)
+                    {
+                        // Not found, so if not the same as the thread context loader, try that.
+                        if (Thread.currentThread().getContextClassLoader()==oClass.getClassLoader())
+                            throw e;
+                        LOG.ignore(e);
+                        mClass=Loader.loadClass(oClass,mName);
+                    }
 
                     if (LOG.isDebugEnabled())
                         LOG.debug("ObjectMbean: mbeanFor {} mClass={}", o, mClass);
diff --git a/jetty-jndi/pom.xml b/jetty-jndi/pom.xml
index 2d4855d..9df46b8 100644
--- a/jetty-jndi/pom.xml
+++ b/jetty-jndi/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-jndi</artifactId>
diff --git a/jetty-jndi/src/main/config/modules/jndi.mod b/jetty-jndi/src/main/config/modules/jndi.mod
index 33c077c..b0d3fc4 100644
--- a/jetty-jndi/src/main/config/modules/jndi.mod
+++ b/jetty-jndi/src/main/config/modules/jndi.mod
@@ -1,6 +1,5 @@
-#
-# JNDI Support
-#
+[description]
+Adds the Jetty JNDI implementation to the classpath.
 
 [depend]
 server
diff --git a/jetty-jspc-maven-plugin/pom.xml b/jetty-jspc-maven-plugin/pom.xml
index ee7a6ac..9923d8d 100644
--- a/jetty-jspc-maven-plugin/pom.xml
+++ b/jetty-jspc-maven-plugin/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-jspc-maven-plugin</artifactId>
diff --git a/jetty-maven-plugin/pom.xml b/jetty-maven-plugin/pom.xml
index 5bf2bfc..9d6d2a3 100644
--- a/jetty-maven-plugin/pom.xml
+++ b/jetty-maven-plugin/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-maven-plugin</artifactId>
diff --git a/jetty-monitor/pom.xml b/jetty-monitor/pom.xml
index f45c120..3374ec2 100644
--- a/jetty-monitor/pom.xml
+++ b/jetty-monitor/pom.xml
@@ -1,25 +1,9 @@
-<!-- 
-// ========================================================================
-// Copyright (c) Webtide LLC
-// 
-// All rights reserved. This program and the accompanying materials
-// are made available under the terms of the Eclipse Public License v1.0
-// and Apache License v2.0 which accompanies this distribution.
-//
-// The Eclipse Public License is available at 
-// http://www.eclipse.org/legal/epl-v10.html
-//
-// The Apache License v2.0 is available at
-// http://www.apache.org/licenses/LICENSE-2.0.txt
-//
-// You may elect to redistribute this code under either of these licenses. 
-// ========================================================================
--->
+<?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-monitor</artifactId>
diff --git a/jetty-monitor/src/main/config/modules/monitor.mod b/jetty-monitor/src/main/config/modules/monitor.mod
index 09132c7..f1fa81f 100644
--- a/jetty-monitor/src/main/config/modules/monitor.mod
+++ b/jetty-monitor/src/main/config/modules/monitor.mod
@@ -1,6 +1,6 @@
-#
-# Jetty Monitor module
-#
+[description]
+Enables the Jetty Monitor Module to periodically
+check/publish JMX parameters of the server.
 
 [depend]
 server
diff --git a/jetty-nosql/pom.xml b/jetty-nosql/pom.xml
index 3375415..55be194 100644
--- a/jetty-nosql/pom.xml
+++ b/jetty-nosql/pom.xml
@@ -2,7 +2,7 @@
   <parent>
     <groupId>org.eclipse.jetty</groupId>
     <artifactId>jetty-project</artifactId>
-    <version>9.3.8-SNAPSHOT</version>
+    <version>9.4.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>jetty-nosql</artifactId>
diff --git a/jetty-nosql/src/main/config/modules/nosql.mod b/jetty-nosql/src/main/config/modules/nosql.mod
index a5b5a9e..fb4ed66 100644
--- a/jetty-nosql/src/main/config/modules/nosql.mod
+++ b/jetty-nosql/src/main/config/modules/nosql.mod
@@ -1,6 +1,5 @@
-#
-# Jetty NoSql module
-#
+[description]
+Enables NoSql session management with a MongoDB driver.
 
 [depend]
 webapp
diff --git a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSession.java b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSession.java
deleted file mode 100644
index afdd9ed..0000000
--- a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSession.java
+++ /dev/null
@@ -1,224 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
-//  ------------------------------------------------------------------------
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-package org.eclipse.jetty.nosql;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.servlet.http.HttpServletRequest;
-
-import org.eclipse.jetty.server.session.MemSession;
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
-
-
-/* ------------------------------------------------------------ */
-public class NoSqlSession extends MemSession
-{
-    private final static Logger __log = Log.getLogger("org.eclipse.jetty.server.session");
-
-    private final NoSqlSessionManager _manager;
-    private Set<String> _dirty;
-    private final AtomicInteger _active = new AtomicInteger();
-    private Object _version;
-    private long _lastSync;
-
-    /* ------------------------------------------------------------ */
-    public NoSqlSession(NoSqlSessionManager manager, HttpServletRequest request)
-    {
-        super(manager, request);
-        _manager=manager;
-        _active.incrementAndGet();
-    }
-    
-    /* ------------------------------------------------------------ */
-    public NoSqlSession(NoSqlSessionManager manager, long created, long accessed, String clusterId, Object version)
-    {
-        super(manager, created,accessed,clusterId);
-        _manager=manager;
-        _version=version;
-    }
-    
-    /* ------------------------------------------------------------ */
-    @Override
-    public Object doPutOrRemove(String name, Object value)
-    {
-        synchronized (this)
-        {
-            Object old = super.doPutOrRemove(name,value);
-            
-            if (_manager.getSavePeriod()==-2)
-            {
-                save(true);
-            }
-            return old;
-        }
-    }
-    
-    
-
-    @Override
-    public void setAttribute(String name, Object value)
-    {
-        Object old = changeAttribute(name,value);
-        if (value == null && old == null)
-            return; //not dirty, no change
-        
-        if (value==null || !value.equals(old))
-        {
-            if (_dirty==null)
-            {
-                _dirty=new HashSet<String>();
-            }
-            
-            _dirty.add(name);
-        }
-    }
-    
-    
-
-    @Override
-    protected void timeout() throws IllegalStateException
-    {
-        super.timeout();
-    }
-
-
-    
-    /* ------------------------------------------------------------ */
-    @Override
-    protected void checkValid() throws IllegalStateException
-    {
-        super.checkValid();
-    }
-
-    /* ------------------------------------------------------------ */
-    @Override
-    protected boolean access(long time)
-    {
-        __log.debug("NoSqlSession:access:active {} time {}", _active, time);
-        if (_active.incrementAndGet()==1)
-        {
-            long period=_manager.getStalePeriod()*1000L;
-            if (period==0)
-                refresh();
-            else if (period>0)
-            {
-                long stale=time-_lastSync;
-                __log.debug("NoSqlSession:access:stale "+stale);
-                if (stale>period)
-                    refresh();
-            }
-        }
-
-        return super.access(time);
-    }
-
-    /* ------------------------------------------------------------ */
-    @Override
-    protected void complete()
-    {
-        super.complete();
-        if(_active.decrementAndGet()==0)
-        {
-            switch(_manager.getSavePeriod())
-            {
-                case 0: 
-                    save(isValid());
-                    break;
-                case 1:
-                    if (isDirty())
-                        save(isValid());
-                    break;
-
-            }
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    @Override
-    protected void doInvalidate() throws IllegalStateException
-    {
-        super.doInvalidate();
-        //jb why save here? if the session is invalidated it should be removed
-        save(false);
-    }
-    
-    /* ------------------------------------------------------------ */
-    protected void save(boolean activateAfterSave)
-    {
-        synchronized (this)
-        {
-            _version=_manager.save(this,_version,activateAfterSave);
-            _lastSync=getAccessed();
-        }
-    }
-
-
-    /* ------------------------------------------------------------ */
-    protected void refresh()
-    {
-        synchronized (this)
-        {
-            _version=_manager.refresh(this,_version);
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    public boolean isDirty()
-    {
-        synchronized (this)
-        {
-            return _dirty!=null && !_dirty.isEmpty();
-        }
-    }
-    
-    /* ------------------------------------------------------------ */
-    public Set<String> takeDirty()
-    {
-        synchronized (this)
-        {
-            Set<String> dirty=_dirty;
-            if (dirty==null)
-                dirty= new HashSet<String>();
-            else
-                _dirty=null;
-            return dirty;
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    public Object getVersion()
-    {
-        return _version;
-    }
-
-    @Override
-    public void setClusterId(String clusterId)
-    {
-        super.setClusterId(clusterId);
-    }
-
-    @Override
-    public void setNodeId(String nodeId)
-    {
-        super.setNodeId(nodeId);
-    }
-}
diff --git a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSessionDataStore.java b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSessionDataStore.java
new file mode 100644
index 0000000..acea527
--- /dev/null
+++ b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSessionDataStore.java
@@ -0,0 +1,98 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+
+package org.eclipse.jetty.nosql;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.eclipse.jetty.server.session.AbstractSessionDataStore;
+import org.eclipse.jetty.server.session.SessionData;
+
+
+/**
+ * NoSqlSessionDataStore
+ *
+ *
+ */
+public abstract class NoSqlSessionDataStore extends AbstractSessionDataStore
+{
+    
+    public class NoSqlSessionData extends SessionData
+    {
+        private Object _version;
+        private Set<String> _dirtyAttributes = new HashSet<String>();
+        
+
+        /**
+         * @param id
+         * @param cpath
+         * @param vhost
+         * @param created
+         * @param accessed
+         * @param lastAccessed
+         * @param maxInactiveMs
+         */
+        public NoSqlSessionData(String id, String cpath, String vhost, long created, long accessed, long lastAccessed, long maxInactiveMs)
+        {
+            super(id, cpath, vhost, created, accessed, lastAccessed, maxInactiveMs);
+        }
+        
+        public void setVersion (Object v)
+        {
+            _version = v;
+        }
+        
+        public Object getVersion ()
+        {
+            return _version;
+        }
+
+        @Override
+        public void setDirty(String name)
+        {
+            super.setDirty(name);
+            _dirtyAttributes.add(name);
+        }
+        
+        
+        public Set<String> takeDirtyAttributes()
+        {
+            Set<String> copy = new HashSet<>(_dirtyAttributes);
+            _dirtyAttributes.clear();
+            return copy;
+            
+        }
+        
+        public Set<String> getAllAttributeNames ()
+        {
+            return new HashSet<String>(_attributes.keySet());
+        }
+    }
+
+
+    @Override
+    public SessionData newSessionData(String id, long created, long accessed, long lastAccessed, long maxInactiveMs)
+    {
+        return new NoSqlSessionData(id, _context.getCanonicalContextPath(), _context.getVhost(), created, accessed, lastAccessed, maxInactiveMs);
+    }
+    
+    
+
+}
diff --git a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSessionManager.java b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSessionManager.java
deleted file mode 100644
index 137d238..0000000
--- a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/NoSqlSessionManager.java
+++ /dev/null
@@ -1,432 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
-//  ------------------------------------------------------------------------
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-package org.eclipse.jetty.nosql;
-
-import java.util.ArrayList;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-import javax.servlet.http.HttpServletRequest;
-
-import org.eclipse.jetty.server.SessionManager;
-import org.eclipse.jetty.server.session.AbstractSession;
-import org.eclipse.jetty.server.session.AbstractSessionManager;
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
-
-/**
- * NoSqlSessionManager
- *
- * Base class for SessionManager implementations using nosql frameworks
- */
-public abstract class NoSqlSessionManager extends AbstractSessionManager implements SessionManager
-{
-    private final static Logger __log = Log.getLogger("org.eclipse.jetty.server.session");
-
-    protected final ConcurrentMap<String,NoSqlSession> _sessions=new ConcurrentHashMap<String,NoSqlSession>();
-
-    private int _stalePeriod=0;
-    private int _savePeriod=0;
-    private int _idlePeriod=-1;
-    private boolean _invalidateOnStop;
-    private boolean _preserveOnStop = true;
-    private boolean _saveAllAttributes;
-    
-    /* ------------------------------------------------------------ */
-    /**
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#doStart()
-     */
-    @Override
-    public void doStart() throws Exception
-    {
-        super.doStart();
-       
-    }
-    
-    /* ------------------------------------------------------------ */
-    @Override
-    protected void addSession(AbstractSession session)
-    {
-        if (isRunning())
-        {
-            //add into memory
-            _sessions.put(session.getClusterId(),(NoSqlSession)session);
-            //add into db
-            ((NoSqlSession)session).save(true);
-        }
-    }
-
-    /* ------------------------------------------------------------ */
-    @Override
-    public AbstractSession getSession(String idInCluster)
-    {
-        NoSqlSession session = _sessions.get(idInCluster);
-        __log.debug("getSession {} ", session );
-        
-        if (session==null)
-        {
-            //session not in this node's memory, load it
-            session=loadSession(idInCluster);
-            
-            if (session!=null)
-            {
-                //session exists, check another request thread hasn't loaded it too
-                NoSqlSession race=_sessions.putIfAbsent(idInCluster,session);
-                if (race!=null)
-                {
-                    session.willPassivate();
-                    session.clearAttributes();
-                    session=race;
-                }
-                else
-                    __log.debug("session loaded ", idInCluster);
-                
-                //check if the session we just loaded has actually expired, maybe while we weren't running
-                if (getMaxInactiveInterval() > 0 && session.getAccessed() > 0 && ((getMaxInactiveInterval()*1000L)+session.getAccessed()) < System.currentTimeMillis())
-                {
-                    __log.debug("session expired ", idInCluster);
-                    expire(idInCluster);
-                    session = null;
-                }
-            }
-            else
-                __log.debug("session does not exist {}", idInCluster);
-        }
-
-        return session;
-    }
-    
-    /* ------------------------------------------------------------ */
-    @Override
-    protected void shutdownSessions() throws Exception
-    {
-        //If we are stopping, and we're preserving sessions, then we want to
-        //save all of the sessions (including those that have been added during this method call)
-        //and then just remove them from memory.
-        
-        //If we don't wish to preserve sessions and we're stopping, then we should invalidate
-        //the session (which may remove it).
-        long gracefulStopMs = getContextHandler().getServer().getStopTimeout();
-        long stopTime = 0;
-        if (gracefulStopMs > 0)
-            stopTime = System.nanoTime() + (TimeUnit.NANOSECONDS.convert(gracefulStopMs, TimeUnit.MILLISECONDS));        
-        
-        ArrayList<NoSqlSession> sessions=new ArrayList<NoSqlSession>(_sessions.values());
-
-        // loop while there are sessions, and while there is stop time remaining, or if no stop time, just 1 loop
-        while (sessions.size() > 0 && ((stopTime > 0 && (System.nanoTime() < stopTime)) || (stopTime == 0)))
-        {
-            for (NoSqlSession session : sessions)
-            {
-                if (isPreserveOnStop())
-                {
-                    //we don't want to delete the session, so save the session
-                    //and remove from memory
-                    session.save(false);
-                    _sessions.remove(session.getClusterId());
-                }
-                else
-                {
-                  //invalidate the session so listeners will be called and also removes the session
-                  session.invalidate();
-                }
-            }
-            
-            //check if we should terminate our loop if we're not using the stop timer
-            if (stopTime == 0)
-            {
-                break;
-            }
-            // Get any sessions that were added by other requests during processing and go around the loop again
-            sessions=new ArrayList<NoSqlSession>(_sessions.values());
-        }
-    }
-    
-
-    /* ------------------------------------------------------------ */
-    @Override
-    protected AbstractSession newSession(HttpServletRequest request)
-    {
-        return new NoSqlSession(this,request);
-    }
-
-    /* ------------------------------------------------------------ */
-    /** Remove the session from the in-memory list for this context.
-     * Also remove the context sub-document for this session id from the db.
-     * @see org.eclipse.jetty.server.session.AbstractSessionManager#removeSession(java.lang.String)
-     */
-    @Override
-    protected boolean removeSession(String idInCluster)
-    {
-        NoSqlSession session = _sessions.remove(idInCluster);
-
-        try
-        {
-            if (session != null)
-            {
-                return remove(session);
-            }
-        }
-        catch (Exception e)
-        {
-            __log.warn("Problem deleting session {}", idInCluster,e);
-        }
-
-        return session != null;
-
-    }
-
-    /* ------------------------------------------------------------ */
-    protected void expire( String idInCluster )
-    {
-        //get the session from memory
-        NoSqlSession session = _sessions.get(idInCluster);
-
-        try
-        {
-            if (session == null)
-            {
-                //we need to expire the session with its listeners, so load it
-                session = loadSession(idInCluster);
-            }
-
-            if (session != null)
-                session.timeout();
-        }
-        catch (Exception e)
-        {
-            __log.warn("Problem expiring session {}", idInCluster,e);
-        }
-    }
-
-
-    public void invalidateSession (String idInCluster)
-    {
-        NoSqlSession session = _sessions.get(idInCluster);
-        try
-        {
-            __log.debug("invalidating session {}", idInCluster);
-            if (session != null)
-            {
-                session.invalidate();
-            }
-        }
-        catch (Exception e)
-        {
-            __log.warn("Problem invalidating session {}", idInCluster,e); 
-        }
-    }
-
-    
-    /* ------------------------------------------------------------ */
-    /**
-     * The State Period is the maximum time in seconds that an in memory session is allows to be stale:
-     * <ul>  
-     * <li>If this period is exceeded, the DB will be checked to see if a more recent version is available.</li>
-     * <li>If the state period is set to a value &lt; 0, then no staleness check will be made.</li>
-     * <li>If the state period is set to 0, then a staleness check is made whenever the active request count goes from 0 to 1.</li>
-     * </ul>
-     * @return the stalePeriod in seconds
-     */
-    public int getStalePeriod()
-    {
-        return _stalePeriod;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * The State Period is the maximum time in seconds that an in memory session is allows to be stale:
-     * <ul>  
-     * <li>If this period is exceeded, the DB will be checked to see if a more recent version is available.</li>
-     * <li>If the state period is set to a value &lt; 0, then no staleness check will be made.</li>
-     * <li>If the state period is set to 0, then a staleness check is made whenever the active request count goes from 0 to 1.</li>
-     * </ul>
-     * @param stalePeriod the stalePeriod in seconds
-     */
-    public void setStalePeriod(int stalePeriod)
-    {
-        _stalePeriod = stalePeriod;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * The Save Period is the time in seconds between saves of a dirty session to the DB.  
-     * When this period is exceeded, the a dirty session will be written to the DB: <ul>
-     * <li>a save period of -2 means the session is written to the DB whenever setAttribute is called.</li>
-     * <li>a save period of -1 means the session is never saved to the DB other than on a shutdown</li>
-     * <li>a save period of 0 means the session is written to the DB whenever the active request count goes from 1 to 0.</li>
-     * <li>a save period of 1 means the session is written to the DB whenever the active request count goes from 1 to 0 and the session is dirty.</li>
-     * <li>a save period of &gt; 1 means the session is written after that period in seconds of being dirty.</li>
-     * </ul>
-     * @return the savePeriod -2,-1,0,1 or the period in seconds &gt;=2 
-     */
-    public int getSavePeriod()
-    {
-        return _savePeriod;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * The Save Period is the time in seconds between saves of a dirty session to the DB.  
-     * When this period is exceeded, the a dirty session will be written to the DB: <ul>
-     * <li>a save period of -2 means the session is written to the DB whenever setAttribute is called.</li>
-     * <li>a save period of -1 means the session is never saved to the DB other than on a shutdown</li>
-     * <li>a save period of 0 means the session is written to the DB whenever the active request count goes from 1 to 0.</li>
-     * <li>a save period of 1 means the session is written to the DB whenever the active request count goes from 1 to 0 and the session is dirty.</li>
-     * <li>a save period of &gt; 1 means the session is written after that period in seconds of being dirty.</li>
-     * </ul>
-     * @param savePeriod the savePeriod -2,-1,0,1 or the period in seconds &gt;=2 
-     */
-    public void setSavePeriod(int savePeriod)
-    {
-        _savePeriod = savePeriod;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * The Idle Period is the time in seconds before an in memory session is passivated.
-     * When this period is exceeded, the session will be passivated and removed from memory.  If the session was dirty, it will be written to the DB.
-     * If the idle period is set to a value &lt; 0, then the session is never idled.
-     * If the save period is set to 0, then the session is idled whenever the active request count goes from 1 to 0.
-     * @return the idlePeriod
-     */
-    public int getIdlePeriod()
-    {
-        return _idlePeriod;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * The Idle Period is the time in seconds before an in memory session is passivated.
-     * When this period is exceeded, the session will be passivated and removed from memory.  If the session was dirty, it will be written to the DB.
-     * If the idle period is set to a value &lt; 0, then the session is never idled.
-     * If the save period is set to 0, then the session is idled whenever the active request count goes from 1 to 0.
-     * @param idlePeriod the idlePeriod in seconds
-     */
-    public void setIdlePeriod(int idlePeriod)
-    {
-        _idlePeriod = idlePeriod;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * Invalidate sessions when the session manager is stopped otherwise save them to the DB.
-     * @return the invalidateOnStop
-     */
-    public boolean isInvalidateOnStop()
-    {
-        return _invalidateOnStop;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * Preserve sessions when the session manager is stopped otherwise remove them from the DB.
-     * @return the removeOnStop
-     */
-    public boolean isPreserveOnStop()
-    {
-        return _preserveOnStop;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * Invalidate sessions when the session manager is stopped otherwise save them to the DB.
-     * @param invalidateOnStop the invalidateOnStop to set
-     */
-    public void setInvalidateOnStop(boolean invalidateOnStop)
-    {
-        _invalidateOnStop = invalidateOnStop;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * Preserve sessions when the session manager is stopped otherwise remove them from the DB.
-     * @param preserveOnStop the preserveOnStop to set
-     */
-    public void setPreserveOnStop(boolean preserveOnStop)
-    {
-        _preserveOnStop = preserveOnStop;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * Save all attributes of a session or only update the dirty attributes.
-     * @return the saveAllAttributes
-     */
-    public boolean isSaveAllAttributes()
-    {
-        return _saveAllAttributes;
-    }
-
-    /* ------------------------------------------------------------ */
-    /**
-     * Save all attributes of a session or only update the dirty attributes.
-     * @param saveAllAttributes the saveAllAttributes to set
-     */
-    public void setSaveAllAttributes(boolean saveAllAttributes)
-    {
-        _saveAllAttributes = saveAllAttributes;
-    }
-    
-    /* ------------------------------------------------------------ */
-    @Override
-    public void renewSessionId(String oldClusterId, String oldNodeId, String newClusterId, String newNodeId)
-    {
-        
-        // Take the old session out of the list of sessions
-        // Change to the new id
-        // Put it back into the list of sessions
-        // Update permanent storage
-
-        synchronized (this)
-        {
-            try
-            {
-                NoSqlSession session = _sessions.remove(oldClusterId);
-                update (session, newClusterId, newNodeId);
-                session.setClusterId(newClusterId);
-                session.setNodeId(newNodeId);
-                _sessions.put(newClusterId, session);
-            }
-            catch (Exception e)
-            {
-                __log.warn(e);
-            }
-        }
-        super.renewSessionId(oldClusterId, oldNodeId, newClusterId, newNodeId);
-    }
-
-    
-    /* ------------------------------------------------------------ */
-    abstract protected NoSqlSession loadSession(String clusterId);
-    
-    /* ------------------------------------------------------------ */
-    abstract protected Object save(NoSqlSession session,Object version, boolean activateAfterSave);
-
-    /* ------------------------------------------------------------ */
-    abstract protected Object refresh(NoSqlSession session, Object version);
-
-    /* ------------------------------------------------------------ */
-    abstract protected boolean remove(NoSqlSession session);
-
-    /* ------------------------------------------------------------ */
-    abstract protected void update(NoSqlSession session, String newClusterId, String newNodeId) throws Exception;
-    
-}
diff --git a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionDataStore.java b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionDataStore.java
new file mode 100644
index 0000000..9aea8cb
--- /dev/null
+++ b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionDataStore.java
@@ -0,0 +1,643 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+
+package org.eclipse.jetty.nosql.mongodb;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.nosql.NoSqlSessionDataStore;
+import org.eclipse.jetty.server.session.SessionData;
+import org.eclipse.jetty.util.ClassLoadingObjectInputStream;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
+import com.mongodb.DBObject;
+import com.mongodb.WriteConcern;
+import com.mongodb.WriteResult;
+
+/**
+ * MongoSessionDataStore
+ *
+ *  The document model is an outer object that contains the elements:
+ * <ul>
+ *  <li>"id"      : session_id </li>
+ *  <li>"created" : create_time </li>
+ *  <li>"accessed": last_access_time </li>
+ *  <li>"maxIdle" : max_idle_time setting as session was created </li>
+ *  <li>"expiry"  : time at which session should expire </li>
+ *  <li>"valid"   : session_valid </li>
+ *  <li>"context" : a nested object containing 1 nested object per context for which the session id is in use
+ * </ul>
+ * Each of the nested objects inside the "context" element contains:
+ * <ul>
+ *  <li>unique_context_name : nested object containing name:value pairs of the session attributes for that context</li>
+ * </ul>
+ * <p>
+ * One of the name:value attribute pairs will always be the special attribute "__metadata__". The value 
+ * is an object representing a version counter which is incremented every time the attributes change.
+ * </p>
+ * <p>
+ * For example:
+ * <pre>
+ * { "_id"       : ObjectId("52845534a40b66410f228f23"), 
+ *    "accessed" :  NumberLong("1384818548903"), 
+ *    "maxIdle"  : 1,
+ *    "context"  : { "::_contextA" : { "A"            : "A", 
+ *                                     "__metadata__" : { "version" : NumberLong(2) } 
+ *                                   },
+ *                   "::_contextB" : { "B"            : "B", 
+ *                                     "__metadata__" : { "version" : NumberLong(1) } 
+ *                                   } 
+ *                 }, 
+ *    "created"  : NumberLong("1384818548903"),
+ *    "expiry"   : NumberLong("1384818549903"),
+ *    "id"       : "w01ijx2vnalgv1sqrpjwuirprp7", 
+ *    "valid"    : true 
+ * }
+ * </pre>
+ * <p>
+ * In MongoDB, the nesting level is indicated by "." separators for the key name. Thus to
+ * interact with a session attribute, the key is composed of:
+ * <code>"context".unique_context_name.attribute_name</code>
+ *  Eg  <code>"context"."::/contextA"."A"</code>
+ */
+public class MongoSessionDataStore extends NoSqlSessionDataStore
+{
+    
+    private final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
+    
+    
+    /**
+     * Special attribute for a session that is context-specific
+     */
+    private final static String __METADATA = "__metadata__";
+
+    /**
+     * Name of nested document field containing 1 sub document per context for which the session id is in use
+     */
+    private final static String __CONTEXT = "context";   
+    
+    /**
+     * Special attribute per session per context, incremented each time attributes are modified
+     */
+    public final static String __VERSION = __METADATA + ".version";
+    
+    /**
+     * Last access time of session
+     */
+    public final static String __ACCESSED = "accessed";
+    
+    /**
+     * Time this session will expire, based on last access time and maxIdle
+     */
+    public final static String __EXPIRY = "expiry";
+    
+    /**
+     * The max idle time of a session (smallest value across all contexts which has a session with the same id)
+     */
+    public final static String __MAX_IDLE = "maxIdle";
+    
+    /**
+     * Time of session creation
+     */
+    private final static String __CREATED = "created";
+    
+    /**
+     * Whether or not session is valid
+     */
+    public final static String __VALID = "valid";
+    
+    /**
+     * Session id
+     */
+    public final static String __ID = "id";
+    
+    
+    
+    /**
+     * Utility value of 1 for a session version for this context
+     */
+    private DBObject _version_1;
+    
+    /**
+     * Access to MongoDB
+     */
+    private DBCollection _dbSessions;
+    
+    
+    private long _gracePeriodMs = 1000L * 60 * 60; //default grace period is 1hr
+    
+    public void setDBCollection (DBCollection collection)
+    {
+        _dbSessions = collection;
+    }
+    
+    
+    /**
+     * @return
+     */
+    public DBCollection getDBCollection ()
+    {
+        return _dbSessions;
+    }
+    
+    /**
+     * @return
+     */
+    public int getGracePeriodSec ()
+    {
+        return (int)(_gracePeriodMs == 0L? 0 : _gracePeriodMs/1000L);
+    }
+    
+    /**
+     * @param sec
+     */
+    public void setGracePeriodSec (int sec)
+    {
+        if (sec < 0)
+            _gracePeriodMs = 0;
+        else
+            _gracePeriodMs = sec * 1000L;
+    }
+    
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#load(org.eclipse.jetty.server.session.SessionKey)
+     */
+    @Override
+    public SessionData load(String id) throws Exception
+    {
+        final AtomicReference<SessionData> reference = new AtomicReference<SessionData>();
+        final AtomicReference<Exception> exception = new AtomicReference<Exception>();
+        Runnable r = new Runnable()
+        {
+            public void run ()
+            {
+                try
+                {
+                    DBObject sessionDocument = _dbSessions.findOne(new BasicDBObject(__ID, id));
+
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("id={} loaded={}", id, sessionDocument);
+                    
+                    if (sessionDocument == null)
+                        return;
+
+                    Boolean valid = (Boolean)sessionDocument.get(__VALID);   
+
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("id={} valid={}", id, valid);
+                    if (valid == null || !valid)
+                        return;
+
+
+                    Object version = getNestedValue(sessionDocument, getContextSubfield(__VERSION));
+
+                    Long created = (Long)sessionDocument.get(__CREATED);
+                    Long accessed = (Long)sessionDocument.get(__ACCESSED);
+                    Long maxInactive = (Long)sessionDocument.get(__MAX_IDLE);
+                    Long expiry = (Long)sessionDocument.get(__EXPIRY);
+
+                    NoSqlSessionData data = null;
+
+                    // get the session for the context
+                    DBObject sessionSubDocumentForContext = (DBObject)getNestedValue(sessionDocument,getContextField());
+
+                    if (LOG.isDebugEnabled()) LOG.debug("attrs {}", sessionSubDocumentForContext);
+
+                    if (sessionSubDocumentForContext != null)
+                    {
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("Session {} present for context {}", id, _context);
+
+                        //only load a session if it exists for this context
+                        data = (NoSqlSessionData)newSessionData(id, created, accessed, accessed, maxInactive);
+                        data.setVersion(version);
+                        data.setExpiry(expiry);
+                        data.setContextPath(_context.getCanonicalContextPath());
+                        data.setVhost(_context.getVhost());
+
+                        HashMap<String, Object> attributes = new HashMap<>();
+                        for (String name : sessionSubDocumentForContext.keySet())
+                        {
+                            //skip special metadata attribute which is not one of the actual session attributes
+                            if ( __METADATA.equals(name) )
+                                continue;         
+                            String attr = decodeName(name);
+                            Object value = decodeValue(sessionSubDocumentForContext.get(name));
+                            attributes.put(attr,value);
+                        }
+
+                        data.putAllAttributes(attributes);
+                    }
+                    else
+                    {
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("Session  {} not present for context {}", id, _context);        
+                    }
+
+                    reference.set(data);
+                }
+                catch (Exception e)
+                {
+                    exception.set(e);
+                }
+            }
+        };
+        
+        _context.run(r);
+        
+        if (exception.get() != null)
+            throw exception.get();
+        
+        return reference.get();
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#delete(org.eclipse.jetty.server.session.SessionKey)
+     */
+    @Override
+    public boolean delete(String id) throws Exception
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("Remove:session {} for context ",id, _context);
+
+        /*
+         * Check if the session exists and if it does remove the context
+         * associated with this session
+         */
+        BasicDBObject mongoKey = new BasicDBObject(__ID, id);
+        
+        DBObject sessionDocument = _dbSessions.findOne(mongoKey,_version_1);
+
+        if (sessionDocument != null)
+        {
+            DBObject c = (DBObject)getNestedValue(sessionDocument, __CONTEXT);
+            if (c == null)
+            {
+                //delete whole doc
+                _dbSessions.remove(mongoKey);
+                return false;
+            }
+
+            Set<String> contexts = c.keySet();
+            if (contexts.isEmpty())
+            {
+                //delete whole doc
+                _dbSessions.remove(mongoKey);
+                return false;
+            }
+
+            if (contexts.size() == 1 && contexts.iterator().next().equals(getCanonicalContextId()))
+            {
+                //delete whole doc
+                 _dbSessions.remove(mongoKey);
+                return true;
+            }
+            
+            //just remove entry for my context
+            BasicDBObject remove = new BasicDBObject();
+            BasicDBObject unsets = new BasicDBObject();
+            unsets.put(getContextField(),1);
+            remove.put("$unset",unsets);
+            _dbSessions.update(mongoKey,remove,false,false,WriteConcern.SAFE);
+            
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+        
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#getExpired(java.util.Set)
+     */
+    @Override
+    public Set<String> getExpired(Set<String> candidates)
+    {
+        long upperBound = System.currentTimeMillis();
+        Set<String> expiredSessions = new HashSet<>();
+        
+        //firstly ask mongo to verify if these candidate ids have expired
+        BasicDBObject query = new BasicDBObject();     
+        query.put(__ID,new BasicDBObject("$in", candidates));
+        query.put(__EXPIRY, new BasicDBObject("$gt", 0));
+        query.put(__EXPIRY, new BasicDBObject("$lt", upperBound));   
+
+        DBCursor verifiedExpiredSessions = null;
+        try  
+        {
+            verifiedExpiredSessions = _dbSessions.find(query, new BasicDBObject(__ID, 1));
+            for ( DBObject session : verifiedExpiredSessions )  
+            {
+                String id = (String)session.get(__ID);
+                if (LOG.isDebugEnabled()) LOG.debug("{} Mongo confirmed expired session {}", _context,id);
+                expiredSessions.add(id);
+            }            
+        }
+        finally
+        {
+            if (verifiedExpiredSessions != null) verifiedExpiredSessions.close();
+        }
+
+
+        //now ask mongo to find sessions that expired a while ago    
+        upperBound = upperBound - (3 * _gracePeriodMs);
+        query.clear();
+        query.put(__EXPIRY, new BasicDBObject("$gt", 0));
+        query.put(__EXPIRY, new BasicDBObject("$lt", upperBound));
+
+        DBCursor oldExpiredSessions = null;
+        try
+        {
+            oldExpiredSessions = _dbSessions.find(query, new BasicDBObject(__ID, 1));
+            for (DBObject session : oldExpiredSessions)
+            {
+                String id = (String)session.get(__ID);
+                if (LOG.isDebugEnabled()) LOG.debug("{} Mongo found old expired session {}", _context, id);
+                expiredSessions.add(id);
+            }
+
+        }
+        finally
+        {
+            oldExpiredSessions.close();
+        }
+        
+        return expiredSessions;
+    }
+
+    /** 
+     * @see org.eclipse.jetty.server.session.AbstractSessionDataStore#doStore(org.eclipse.jetty.server.session.SessionKey, org.eclipse.jetty.server.session.SessionData, boolean)
+     */
+    @Override
+    public void doStore(String id, SessionData data, boolean isNew) throws Exception
+    {
+        NoSqlSessionData nsqd = (NoSqlSessionData)data;
+        
+        // Form query for upsert
+        BasicDBObject key = new BasicDBObject(__ID, id);
+
+        // Form updates
+        BasicDBObject update = new BasicDBObject();
+        boolean upsert = false;
+        BasicDBObject sets = new BasicDBObject();
+        BasicDBObject unsets = new BasicDBObject();
+        
+        Object version = ((NoSqlSessionData)data).getVersion();
+        
+        // New session
+        if (isNew)
+        {
+            upsert = true;
+            version = new Long(1);
+            sets.put(__CREATED,nsqd.getCreated());
+            sets.put(__VALID,true);
+
+            sets.put(getContextSubfield(__VERSION),version);
+            sets.put(__MAX_IDLE, nsqd.getMaxInactiveMs());
+            sets.put(__EXPIRY, nsqd.getExpiry());
+            nsqd.setVersion(version);
+        }
+        else
+        {
+            version = new Long(((Number)version).longValue() + 1);
+            nsqd.setVersion(version);
+            update.put("$inc",_version_1); 
+            //if max idle time and/or expiry is smaller for this context, then choose that for the whole session doc
+            BasicDBObject fields = new BasicDBObject();
+            fields.append(__MAX_IDLE, true);
+            fields.append(__EXPIRY, true);
+            DBObject o = _dbSessions.findOne(new BasicDBObject("id", id), fields);
+            if (o != null)
+            {
+                Long currentMaxIdle = (Long)o.get(__MAX_IDLE);
+                Long currentExpiry = (Long)o.get(__EXPIRY);
+                if (currentMaxIdle != null && nsqd.getMaxInactiveMs() > 0 && nsqd.getMaxInactiveMs() < currentMaxIdle)
+                    sets.put(__MAX_IDLE, nsqd.getMaxInactiveMs());
+                if (currentExpiry != null && nsqd.getExpiry() > 0 && nsqd.getExpiry() != currentExpiry)
+                    sets.put(__EXPIRY, nsqd.getExpiry());
+            }
+            else
+                LOG.warn("Session {} not found, can't update", id);
+        }
+
+        sets.put(__ACCESSED, nsqd.getAccessed());
+
+        Set<String> names = nsqd.takeDirtyAttributes();
+
+        if (isNew)
+        {
+            names.addAll(nsqd.getAllAttributeNames()); // note dirty may include removed names
+        }
+
+
+        for (String name : names)
+        {
+
+            Object value = data.getAttribute(name);
+            if (value == null)
+                unsets.put(getContextField() + "." + encodeName(name),1);
+            else
+                sets.put(getContextField() + "." + encodeName(name),encodeName(value));
+        }
+
+        // Do the upsert
+        if (!sets.isEmpty())
+            update.put("$set",sets);
+        if (!unsets.isEmpty())
+            update.put("$unset",unsets);
+
+        _dbSessions.update(key,update,upsert,false,WriteConcern.SAFE);
+
+
+        if (LOG.isDebugEnabled())
+            LOG.debug("Save:db.sessions.update( {}, {} )", key, update); 
+    }
+
+
+    
+    
+    @Override
+    protected void doStart() throws Exception
+    {
+        if (_dbSessions == null)
+            throw new IllegalStateException("DBCollection not set");
+        
+        _version_1 = new BasicDBObject(getContextSubfield(__VERSION),1);
+        
+        super.doStart();
+    }
+
+    @Override
+    protected void doStop() throws Exception
+    {
+        // TODO Auto-generated method stub
+        super.doStop();
+    }
+
+    /*------------------------------------------------------------ */
+    private String getContextField ()
+    {
+        return __CONTEXT + "." + getCanonicalContextId();
+    }
+    
+   
+    private String getCanonicalContextId ()
+    {
+        return canonicalizeVHost(_context.getVhost()) + ":" + _context.getCanonicalContextPath();
+    }
+    
+    private String canonicalizeVHost (String vhost)
+    {
+        if (vhost == null)
+            return "";
+        
+        return vhost.replace('.', '_');
+    }
+  
+    
+    private String getContextSubfield (String attr)
+    {
+        return getContextField () +"."+ attr;
+    }
+    
+
+    /*------------------------------------------------------------ */
+    protected Object decodeValue(final Object valueToDecode) throws IOException, ClassNotFoundException
+    {
+        if (valueToDecode == null || valueToDecode instanceof Number || valueToDecode instanceof String || valueToDecode instanceof Boolean || valueToDecode instanceof Date)
+        {
+            return valueToDecode;
+        }
+        else if (valueToDecode instanceof byte[])
+        {
+            final byte[] decodeObject = (byte[])valueToDecode;
+            final ByteArrayInputStream bais = new ByteArrayInputStream(decodeObject);
+            final ClassLoadingObjectInputStream objectInputStream = new ClassLoadingObjectInputStream(bais);
+            return objectInputStream.readUnshared();
+        }
+        else if (valueToDecode instanceof DBObject)
+        {
+            Map<String, Object> map = new HashMap<String, Object>();
+            for (String name : ((DBObject)valueToDecode).keySet())
+            {
+                String attr = decodeName(name);
+                map.put(attr,decodeValue(((DBObject)valueToDecode).get(name)));
+            }
+            return map;
+        }
+        else
+        {
+            throw new IllegalStateException(valueToDecode.getClass().toString());
+        }
+    }
+    /*------------------------------------------------------------ */
+    protected String decodeName(String name)
+    {
+        return name.replace("%2E",".").replace("%25","%");
+    }
+    
+
+    /*------------------------------------------------------------ */
+    protected String encodeName(String name)
+    {
+        return name.replace("%","%25").replace(".","%2E");
+    }
+
+
+    /*------------------------------------------------------------ */
+    protected Object encodeName(Object value) throws IOException
+    {
+        if (value instanceof Number || value instanceof String || value instanceof Boolean || value instanceof Date)
+        {
+            return value;
+        }
+        else if (value.getClass().equals(HashMap.class))
+        {
+            BasicDBObject o = new BasicDBObject();
+            for (Map.Entry<?, ?> entry : ((Map<?, ?>)value).entrySet())
+            {
+                if (!(entry.getKey() instanceof String))
+                {
+                    o = null;
+                    break;
+                }
+                o.append(encodeName(entry.getKey().toString()),encodeName(entry.getValue()));
+            }
+
+            if (o != null)
+                return o;
+        }
+        
+        ByteArrayOutputStream bout = new ByteArrayOutputStream();
+        ObjectOutputStream out = new ObjectOutputStream(bout);
+        out.reset();
+        out.writeUnshared(value);
+        out.flush();
+        return bout.toByteArray();
+    }
+    
+    /*------------------------------------------------------------ */
+    /**
+     * Dig through a given dbObject for the nested value
+     */
+    private Object getNestedValue(DBObject dbObject, String nestedKey)
+    {
+        String[] keyChain = nestedKey.split("\\.");
+
+        DBObject temp = dbObject;
+
+        for (int i = 0; i < keyChain.length - 1; ++i)
+        {
+            temp = (DBObject)temp.get(keyChain[i]);
+
+            if ( temp == null )
+            {
+                return null;
+            }
+        }
+
+        return temp.get(keyChain[keyChain.length - 1]);
+    }
+
+
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#isPassivating()
+     */
+    @Override
+    public boolean isPassivating()
+    {
+        return true;
+    }
+
+}
diff --git a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionIdManager.java b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionIdManager.java
index e078b9a..9cad234 100644
--- a/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionIdManager.java
+++ b/jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionIdManager.java
@@ -20,93 +20,40 @@
 
 
 import java.net.UnknownHostException;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.Random;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpSession;
 
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.SessionManager;
-import org.eclipse.jetty.server.handler.ContextHandler;
 import org.eclipse.jetty.server.session.AbstractSessionIdManager;
-import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.server.session.Session;
 import org.eclipse.jetty.util.ConcurrentHashSet;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-import org.eclipse.jetty.util.thread.Scheduler;
 
 import com.mongodb.BasicDBObject;
 import com.mongodb.BasicDBObjectBuilder;
 import com.mongodb.DBCollection;
-import com.mongodb.DBCursor;
 import com.mongodb.DBObject;
 import com.mongodb.Mongo;
 import com.mongodb.MongoException;
 
 /**
- * Based partially on the JDBCSessionIdManager.
- * <p>
- * Theory is that we really only need the session id manager for the local 
- * instance so we have something to scavenge on, namely the list of known ids
- * <p>
- * This class has a timer that runs a periodic scavenger thread to query
- *  for all id's known to this node whose precalculated expiry time has passed.
- * <p>
- * These found sessions are then run through the invalidateAll(id) method that 
- * is a bit hinky but is supposed to notify all handlers this id is now DOA and 
- * ought to be cleaned up.  this ought to result in a save operation on the session
- * that will change the valid field to false (this conjecture is unvalidated atm)
+ * Manager of session ids based on sessions stored in Mongo.
+ * 
  */
 public class MongoSessionIdManager extends AbstractSessionIdManager
 {
-    private final static Logger __log = Log.getLogger("org.eclipse.jetty.server.session");
+    private final static Logger LOG = Log.getLogger("org.eclipse.jetty.server.session");
 
-    final static DBObject __version_1 = new BasicDBObject(MongoSessionManager.__VERSION,1);
-    final static DBObject __valid_false = new BasicDBObject(MongoSessionManager.__VALID,false);
-    final static DBObject __valid_true = new BasicDBObject(MongoSessionManager.__VALID,true);
+    final static DBObject __version_1 = new BasicDBObject(MongoSessionDataStore.__VERSION,1);
+    final static DBObject __valid_false = new BasicDBObject(MongoSessionDataStore.__VALID,false);
+    final static DBObject __valid_true = new BasicDBObject(MongoSessionDataStore.__VALID,true);
+    final static DBObject __expiry = new BasicDBObject(MongoSessionDataStore.__EXPIRY, 1);
 
-    final static long __defaultScavengePeriod = 30 * 60 * 1000; // every 30 minutes
-   
     
     final DBCollection _sessions;
-    protected Server _server;
-    private Scheduler _scheduler;
-    private boolean _ownScheduler;
-    private Scheduler.Task _scavengerTask;
-    private Scheduler.Task _purgerTask;
- 
-
-    
-    private long _scavengePeriod = __defaultScavengePeriod;
-    
-
-    /** 
-     * purge process is enabled by default
-     */
-    private boolean _purge = true;
-
-    /**
-     * purge process would run daily by default
-     */
-    private long _purgeDelay = 24 * 60 * 60 * 1000; // every day
-    
-    /**
-     * how long do you want to persist sessions that are no longer
-     * valid before removing them completely
-     */
-    private long _purgeInvalidAge = 24 * 60 * 60 * 1000; // default 1 day
-
-    /**
-     * how long do you want to leave sessions that are still valid before
-     * assuming they are dead and removing them
-     */
-    private long _purgeValidAge = 7 * 24 * 60 * 60 * 1000; // default 1 week
 
     
     /**
@@ -114,57 +61,7 @@
      */
     protected final Set<String> _sessionsIds = new ConcurrentHashSet<>();
 
-    /**
-     * The maximum number of items to return from a purge query.
-     */
-    private int _purgeLimit = 0;
-
-    private int _scavengeBlockSize;
-    
-    
-    /**
-     * Scavenger
-     *
-     */
-    protected class Scavenger implements Runnable
-    {
-        @Override
-        public void run()
-        {
-            try
-            {
-                scavenge();
-            }
-            finally
-            {
-                if (_scheduler != null && _scheduler.isRunning())
-                    _scavengerTask = _scheduler.schedule(this, _scavengePeriod, TimeUnit.MILLISECONDS);
-            }
-        } 
-    }
-    
-    
-    /**
-     * Purger
-     *
-     */
-    protected class Purger implements Runnable
-    {
-        @Override
-        public void run()
-        {
-            try
-            {
-                purge();
-            }
-            finally
-            {
-                if (_scheduler != null && _scheduler.isRunning())
-                    _purgerTask = _scheduler.schedule(this, _purgeDelay, TimeUnit.MILLISECONDS);
-            }
-        }
-    }
-    
+   
     
     
 
@@ -177,9 +74,8 @@
     /* ------------------------------------------------------------ */
     public MongoSessionIdManager(Server server, DBCollection sessions)
     {
-        super(new Random());
+        super(server, new Random());
         
-        _server = server;
         _sessions = sessions;
 
         _sessions.ensureIndex(
@@ -193,187 +89,10 @@
         // so that we can take advantage of index prefixes
         // http://docs.mongodb.org/manual/core/index-compound/#compound-index-prefix
         _sessions.ensureIndex(
-                BasicDBObjectBuilder.start().add(MongoSessionManager.__VALID, 1).add(MongoSessionManager.__ACCESSED, 1).get(),
+                BasicDBObjectBuilder.start().add(MongoSessionDataStore.__VALID, 1).add(MongoSessionDataStore.__ACCESSED, 1).get(),
                 BasicDBObjectBuilder.start().add("sparse", false).add("background", true).get());
     }
  
-    /* ------------------------------------------------------------ */
-    /**
-     * Scavenge is a process that periodically checks the tracked session
-     * ids of this given instance of the session id manager to see if they 
-     * are past the point of expiration.
-     */
-    protected void scavenge()
-    {
-        long now = System.currentTimeMillis();
-        __log.debug("SessionIdManager:scavenge:at {}", now);        
-        /*
-         * run a query returning results that:
-         *  - are in the known list of sessionIds
-         *  - the expiry time has passed
-         *  
-         *  we limit the query to return just the __ID so we are not sucking back full sessions
-         *  
-         *  break scavenge query into blocks for faster mongo queries
-         */
-        Set<String> block = new HashSet<String>();
-            
-        Iterator<String> itor = _sessionsIds.iterator();
-        while (itor.hasNext())
-        {
-            block.add(itor.next());
-            if ((_scavengeBlockSize > 0) && (block.size() == _scavengeBlockSize))
-            {
-                //got a block
-                scavengeBlock (now, block);
-                //reset for next run
-                block.clear();
-            }
-        }
-        
-        //non evenly divisble block size, or doing it all at once
-        if (!block.isEmpty())
-            scavengeBlock(now, block);
-    }
-    
-    
-    /* ------------------------------------------------------------ */
-    /**
-     * Check a block of ses