diff --git a/src/Microsoft.AspNet.Server.Kestrel/KestrelEngine.cs b/src/Microsoft.AspNet.Server.Kestrel/KestrelEngine.cs
index ae0c87590d..21b4df4523 100644
--- a/src/Microsoft.AspNet.Server.Kestrel/KestrelEngine.cs
+++ b/src/Microsoft.AspNet.Server.Kestrel/KestrelEngine.cs
@@ -91,19 +91,33 @@ namespace Microsoft.AspNet.Server.Kestrel
         public IDisposable CreateServer(string scheme, string host, int port, Func<Frame, Task> application)
         {
             var listeners = new List<Listener>();
-            foreach (var thread in Threads)
+
+            try
             {
-                var listener = new Listener(Memory);
-                listener.StartAsync(scheme, host, port, thread, application).Wait();
-                listeners.Add(listener);
+                foreach (var thread in Threads)
+                {
+                    var listener = new Listener(Memory);
+
+                    listeners.Add(listener);
+                    listener.StartAsync(scheme, host, port, thread, application).Wait();
+                }
+                return new Disposable(() =>
+                {
+                    foreach (var listener in listeners)
+                    {
+                        listener.Dispose();
+                    }
+                });
             }
-            return new Disposable(() =>
+            catch
             {
                 foreach (var listener in listeners)
                 {
                     listener.Dispose();
                 }
-            });
+
+                throw;
+            }
         }
     }
 }