Add an option to Kestrel to disable threadpool dispatching

samples/SampleApp/Startup.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Server.Kestrel.Internal;
 using Microsoft.Extensions.Logging;
 
 namespace SampleApp
@@ -68,6 +69,8 @@ namespace SampleApp
                 .UseStartup<Startup>()
                 .Build();
 
+            host.ServerFeatures.Get<InternalKestrelServerOptions>().ThreadPoolDispatching = false;
+
             host.Run();
         }
     }

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameOfT.cs

@@ -17,8 +17,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
     {
         private readonly IHttpApplication<TContext> _application;
 
-        public Frame(IHttpApplication<TContext> application,
+        public Frame(IHttpApplication<TContext> application, ConnectionContext context)
-                     ConnectionContext context)
             : base(context)
         {
             _application = application;

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/InlineLoggingThreadPool.cs

@@ -0,0 +1,78 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure
+{
+    public class InlineLoggingThreadPool : IThreadPool
+    {
+        private readonly IKestrelTrace _log;
+
+        public InlineLoggingThreadPool(IKestrelTrace log)
+        {
+            _log = log;
+        }
+
+        public void Run(Action action)
+        {
+            try
+            {
+                action();
+            }
+            catch (Exception e)
+            {
+                _log.LogError(0, e, "InlineLoggingThreadPool.Run");
+            }
+        }
+
+        public void Complete(TaskCompletionSource<object> tcs)
+        {
+            try
+            {
+                tcs.TrySetResult(null);
+            }
+            catch (Exception e)
+            {
+                _log.LogError(0, e, "InlineLoggingThreadPool.Complete");
+            }
+        }
+
+        public void Cancel(TaskCompletionSource<object> tcs)
+        {
+            try
+            {
+                tcs.TrySetCanceled();
+            }
+            catch (Exception e)
+            {
+                _log.LogError(0, e, "InlineLoggingThreadPool.Cancel");
+            }
+        }
+
+        public void Error(TaskCompletionSource<object> tcs, Exception ex)
+        {
+            try
+            {
+                tcs.TrySetException(ex);
+            }
+            catch (Exception e)
+            {
+                _log.LogError(0, e, "InlineLoggingThreadPool.Error");
+            }
+        }
+
+        public void UnsafeRun(WaitCallback action, object state)
+        {
+            action(state);
+        }
+
+        public void Schedule(Action action)
+        {
+            Run(action);
+        }
+    }
+}

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/KestrelThread.cs

@@ -20,7 +20,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal
     /// <summary>
     /// Summary description for KestrelThread
     /// </summary>
-    public class KestrelThread: IScheduler
+    public class KestrelThread : IScheduler
     {
         public const long HeartbeatMilliseconds = 1000;
 

src/Microsoft.AspNetCore.Server.Kestrel/Internal/InternalKestrelServerOptions.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Internal
+{
+    public class InternalKestrelServerOptions
+    {
+        // This will likely be replace with transport-specific configuration.
+        // https://github.com/aspnet/KestrelHttpServer/issues/828
+        public bool ThreadPoolDispatching { get; set; } = true;
+    }
+}

src/Microsoft.AspNetCore.Server.Kestrel/KestrelServer.cs

@@ -45,17 +45,21 @@ namespace Microsoft.AspNetCore.Server.Kestrel
             }
 
             Options = options.Value ?? new KestrelServerOptions();
+            InternalOptions = new InternalKestrelServerOptions();
             _applicationLifetime = applicationLifetime;
             _logger = loggerFactory.CreateLogger(typeof(KestrelServer).GetTypeInfo().Namespace);
             Features = new FeatureCollection();
             _serverAddresses = new ServerAddressesFeature();
             Features.Set<IServerAddressesFeature>(_serverAddresses);
+            Features.Set(InternalOptions);
         }
 
         public IFeatureCollection Features { get; }
 
         public KestrelServerOptions Options { get; }
 
+        private InternalKestrelServerOptions InternalOptions { get; }
+
         public void Start<TContext>(IHttpApplication<TContext> application)
         {
             try
@@ -76,6 +80,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel
 
                 var dateHeaderValueManager = new DateHeaderValueManager();
                 var trace = new KestrelTrace(_logger);
+
+                IThreadPool threadPool;
+                if (InternalOptions.ThreadPoolDispatching)
+                {
+                    threadPool = new LoggingThreadPool(trace);
+                }
+                else
+                {
+                    threadPool = new InlineLoggingThreadPool(trace);
+                }
+
                 var engine = new KestrelEngine(new ServiceContext
                 {
                     FrameFactory = context =>
@@ -84,7 +99,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel
                     },
                     AppLifetime = _applicationLifetime,
                     Log = trace,
-                    ThreadPool = new LoggingThreadPool(trace),
+                    ThreadPool = threadPool,
                     DateHeaderValueManager = dateHeaderValueManager,
                     ServerOptions = Options
                 });

test/Microsoft.AspNetCore.Server.KestrelTests/ConnectionTests.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Server.Kestrel;
 using Microsoft.AspNetCore.Server.Kestrel.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Internal.Http;
+using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure;
 using Microsoft.AspNetCore.Server.Kestrel.Internal.Networking;
 using Microsoft.AspNetCore.Server.KestrelTests.TestHelpers;
 using Microsoft.AspNetCore.Testing;
@@ -22,42 +23,37 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
         public async Task DoesNotEndConnectionOnZeroRead()
         {
             var mockLibuv = new MockLibuv();
+            var serviceContext = new TestServiceContext
+            {
+                FrameFactory = connectionContext => new Frame<HttpContext>(
+                    new DummyApplication(httpContext => TaskCache.CompletedTask), connectionContext)
+            };
 
-            using (var engine = new KestrelEngine(mockLibuv, new TestServiceContext()))
+            // Ensure ProcessRequestAsync runs inline with the ReadCallback
+            serviceContext.ThreadPool = new InlineLoggingThreadPool(serviceContext.Log);
+
+            using (var engine = new KestrelEngine(mockLibuv, serviceContext))
             {
                 engine.Start(count: 1);
 
-                var trace = new TestKestrelTrace();
+                var listenerContext = new ListenerContext(serviceContext)
-                var serviceContext = new TestServiceContext
-                {
-                    FrameFactory = connectionContext => new Frame<HttpContext>(
-                        new DummyApplication(httpContext => TaskCache.CompletedTask), connectionContext),
-                };
-                var context = new ListenerContext(serviceContext)
                 {
                     ListenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0)),
                     Thread = engine.Threads[0]
                 };
 
                 Connection connection = null;
-                await context.Thread.PostAsync(_ =>
+                await listenerContext.Thread.PostAsync(_ =>
                 {
-                    var socket = new MockSocket(mockLibuv, Thread.CurrentThread.ManagedThreadId, trace);
+                    var socket = new MockSocket(mockLibuv, Thread.CurrentThread.ManagedThreadId, serviceContext.Log);
-                    connection = new Connection(context, socket);
+                    connection = new Connection(listenerContext, socket);
                     connection.Start();
 
                     Libuv.uv_buf_t ignored;
                     mockLibuv.AllocCallback(socket.InternalGetHandle(), 2048, out ignored);
+                    // This runs the ProcessRequestAsync inline
                     mockLibuv.ReadCallback(socket.InternalGetHandle(), 0, ref ignored);
-                    
-                }, (object)null);
 
-                // Wait until ProcessRequestAsync runs
-                // TODO: Remove when we get non dispatching support
-                await Task.Delay(1000);
-
-                await context.Thread.PostAsync(_ =>
-                {
                     var readAwaitable = connection.Input.Reader.ReadAsync();
 
                     Assert.False(readAwaitable.IsCompleted);

test/Microsoft.AspNetCore.Server.KestrelTests/SocketOutputTests.cs

@@ -69,7 +69,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var socketOutput = new SocketOutput(kestrelThread, socket, new MockConnection(options), "0", trace, ltp);
 
                 // At least one run of this test should have a MaxResponseBufferSize < 1 MB.
@@ -112,7 +112,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var options = new KestrelServerOptions { Limits = { MaxResponseBufferSize = null } };
                 var socketOutput = new SocketOutput(kestrelThread, socket, new MockConnection(options), "0", trace, ltp);
 
@@ -163,7 +163,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var options = new KestrelServerOptions { Limits = { MaxResponseBufferSize = 0 } };
                 var socketOutput = new SocketOutput(kestrelThread, socket, new MockConnection(options), "0", trace, ltp);
 
@@ -226,7 +226,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var mockConnection = new MockConnection(options);
                 var socketOutput = new SocketOutput(kestrelThread, socket, mockConnection, "0", trace, ltp);
 
@@ -298,7 +298,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var mockConnection = new MockConnection(options);
                 var socketOutput = new SocketOutput(kestrelThread, socket, mockConnection, "0", trace, ltp);
 
@@ -381,7 +381,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
 
                 using (var mockConnection = new MockConnection(options))
                 {
@@ -498,7 +498,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
 
                 using (var mockConnection = new MockConnection(options))
                 {
@@ -591,7 +591,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var mockConnection = new MockConnection(options);
                 var socketOutput = new SocketOutput(kestrelThread, socket, mockConnection, "0", trace, ltp);
 
@@ -671,7 +671,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var socketOutput = new SocketOutput(kestrelThread, socket, new MockConnection(options), "0", trace, ltp);
 
                 // block 1
@@ -724,7 +724,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var mockConnection = new MockConnection(options);
                 var socketOutput = new SocketOutput(kestrelThread, socket, mockConnection, "0", trace, ltp);
 
@@ -796,7 +796,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var socketOutput = new SocketOutput(kestrelThread, socket, new MockConnection(new KestrelServerOptions()), "0", trace, ltp);
 
                 mockLibuv.KestrelThreadBlocker.Reset();
@@ -842,7 +842,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
 
                 var socket = new MockSocket(mockLibuv, kestrelThread.Loop.ThreadId, new TestKestrelTrace());
                 var trace = new KestrelTrace(new TestKestrelTrace());
-                var ltp = new SynchronousThreadPool();
+                var ltp = new InlineLoggingThreadPool(trace);
                 var connection = new MockConnection(new KestrelServerOptions());
                 var socketOutput = new SocketOutput(kestrelThread, socket, connection, "0", trace, ltp);
 

test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/SynchronousThreadPool.cs

@@ -1,43 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure;
-
-namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers
-{
-    public class SynchronousThreadPool : IThreadPool
-    {
-        public void Complete(TaskCompletionSource<object> tcs)
-        {
-            tcs.TrySetResult(null);
-        }
-
-        public void Cancel(TaskCompletionSource<object> tcs)
-        {
-            tcs.TrySetCanceled();
-        }
-
-        public void Error(TaskCompletionSource<object> tcs, Exception ex)
-        {
-            tcs.TrySetException(ex);
-        }
-
-        public void Run(Action action)
-        {
-            action();
-        }
-
-        public void UnsafeRun(WaitCallback action, object state)
-        {
-            action(state);
-        }
-
-        public void Schedule(Action action)
-        {
-            action();
-        }
-    }
-}

test/shared/TestServiceContext.cs

@@ -4,7 +4,6 @@
 using System;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Server.Kestrel;
-using Microsoft.AspNetCore.Server.Kestrel.Adapter;
 using Microsoft.AspNetCore.Server.Kestrel.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Internal.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure;
@@ -22,8 +21,11 @@ namespace Microsoft.AspNetCore.Testing
             ThreadPool = new LoggingThreadPool(Log);
             DateHeaderValueManager = new DateHeaderValueManager(systemClock: new MockSystemClock());
             DateHeaderValue = DateHeaderValueManager.GetDateHeaderValues().String;
-            ServerOptions = new KestrelServerOptions { AddServerHeader = false };
+            ServerOptions = new KestrelServerOptions
-            ServerOptions.ShutdownTimeout = TimeSpan.FromSeconds(5);
+            {
+                AddServerHeader = false,
+                ShutdownTimeout = TimeSpan.FromSeconds(5)
+            };
         }
 
         public string DateHeaderValue { get; }