Overhead Benchmark for Request Throttling (#10907)

* initial implementation; no tests

* benchmark project

* Better benchmarks

* overhead test for congested queue

* Addressed feedback

src/Middleware/Middleware.sln

@@ -293,6 +293,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Metada
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Authorization", "..\Security\Authorization\Core\src\Microsoft.AspNetCore.Authorization.csproj", "{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.RequestThrottling.Microbenchmarks", "RequestThrottling\perf\Microbenchmarks\Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.csproj", "{737B26B4-CFC6-4B44-9070-DD36334E85B3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1599,6 +1601,18 @@ Global
 		{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}.Release|x64.Build.0 = Release|Any CPU
 		{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}.Release|x86.ActiveCfg = Release|Any CPU
 		{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}.Release|x86.Build.0 = Release|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x64.Build.0 = Debug|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x86.Build.0 = Debug|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x64.ActiveCfg = Release|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x64.Build.0 = Release|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x86.ActiveCfg = Release|Any CPU
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1725,6 +1739,7 @@ Global
 		{353AA2B0-1013-486C-B5BD-9379385CA403} = {8C9AA8A2-9D1F-4450-9F8D-56BAB6F3D343}
 		{7E2EA6E2-31FE-418A-9AE4-955A4C708AE7} = {ACA6DDB9-7592-47CE-A740-D15BF307E9E0}
 		{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF} = {ACA6DDB9-7592-47CE-A740-D15BF307E9E0}
+		{737B26B4-CFC6-4B44-9070-DD36334E85B3} = {8C9AA8A2-9D1F-4450-9F8D-56BAB6F3D343}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {83786312-A93B-4BB4-AB06-7C6913A59AFA}

src/Middleware/RequestThrottling/RequestThrottling.slnf

@@ -18,7 +18,8 @@
       "HttpsPolicy\\src\\Microsoft.AspNetCore.HttpsPolicy.csproj",
       "RequestThrottling\\sample\\RequestThrottlingSample.csproj",
       "RequestThrottling\\src\\Microsoft.AspNetCore.RequestThrottling.csproj",
-      "RequestThrottling\\test\\Microsoft.AspNetCore.RequestThrottling.Tests.csproj"
+      "RequestThrottling\\test\\Microsoft.AspNetCore.RequestThrottling.Tests.csproj",
+      "RequestThrottling\\perf\\Microbenchmarks\\Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.csproj"
     ]
   }
 }

src/Middleware/RequestThrottling/perf/Microbenchmarks/AssemblyInfo.cs

@@ -0,0 +1 @@
+[assembly: BenchmarkDotNet.Attributes.AspNetCoreBenchmark]

src/Middleware/RequestThrottling/perf/Microbenchmarks/Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!--<StartupObject>Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.Test</StartupObject>-->
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Reference Include="BenchmarkDotNet" />
+    <Reference Include="Microsoft.AspNetCore.BenchmarkRunner.Sources" />
+    <Reference Include="Microsoft.AspNetCore.RequestThrottling" />
+  </ItemGroup>
+</Project>

src/Middleware/RequestThrottling/perf/Microbenchmarks/QueueEmptyOverhead.cs

@@ -0,0 +1,71 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
+{
+    public class QueueEmptyOverhead
+    {
+        private const int _numRequests = 20000;
+
+        private RequestThrottlingMiddleware _middleware;
+        private RequestDelegate _restOfServer;
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            _restOfServer = YieldsThreadInternally ? (RequestDelegate)YieldsThread : (RequestDelegate)CompletesImmediately;
+
+            var options = new RequestThrottlingOptions
+            {
+                MaxConcurrentRequests = 8,
+                RequestQueueLimit = _numRequests
+            };
+
+            _middleware = new RequestThrottlingMiddleware(
+                    next: _restOfServer,
+                    loggerFactory: NullLoggerFactory.Instance,
+                    options: Options.Create(options)
+                );
+        }
+
+        [Params(false, true)]
+        public bool YieldsThreadInternally;
+
+        [Benchmark(OperationsPerInvoke = _numRequests)]
+        public async Task Baseline()
+        {
+            for (int i = 0; i < _numRequests; i++)
+            {
+                await _restOfServer(null);
+            }
+        }
+
+        [Benchmark(OperationsPerInvoke = _numRequests)]
+        public async Task WithEmptyQueueOverhead()
+        {
+            for (int i = 0; i < _numRequests; i++)
+            {
+                await _middleware.Invoke(null);
+            }
+        }
+
+        private static async Task YieldsThread(HttpContext context)
+        {
+            await Task.Yield();
+        }
+
+        private static Task CompletesImmediately(HttpContext context)
+        {
+            return Task.CompletedTask;
+        }
+    }
+}

src/Middleware/RequestThrottling/perf/Microbenchmarks/QueueFullOverhead.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
+{
+    public class QueueFullOverhead
+    {
+        private const int _numRequests = 2000;
+        private int _requestCount = 0;
+        private ManualResetEventSlim _mres = new ManualResetEventSlim();
+
+        private RequestThrottlingMiddleware _middleware;
+
+        [Params(8)]
+        public int MaxConcurrentRequests;
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            var options = new RequestThrottlingOptions
+            {
+                MaxConcurrentRequests = MaxConcurrentRequests,
+                RequestQueueLimit = _numRequests
+            };
+
+            _middleware = new RequestThrottlingMiddleware(
+                    next: (RequestDelegate)_incrementAndCheck,
+                    loggerFactory: NullLoggerFactory.Instance,
+                    options: Options.Create(options)
+                );
+        }
+
+        [IterationSetup]
+        public void Setup()
+        {
+            _requestCount = 0;
+            _mres.Reset();
+        }
+
+        private async Task _incrementAndCheck(HttpContext context)
+        {
+            if (Interlocked.Increment(ref _requestCount) == _numRequests)
+            {
+                _mres.Set();
+            }
+
+            await Task.Yield();
+        }
+
+        [Benchmark(OperationsPerInvoke = _numRequests)]
+        public void Baseline()
+        {
+            for (int i = 0; i < _numRequests; i++)
+            {
+                _ = _incrementAndCheck(null);
+            }
+
+            _mres.Wait();
+        }
+
+        [Benchmark(OperationsPerInvoke = _numRequests)]
+        public void QueueingAll()
+        {
+            for (int i = 0; i < _numRequests; i++)
+            {
+                _ = _middleware.Invoke(null);
+            }
+
+            _mres.Wait();
+        }
+    }
+}

src/Middleware/RequestThrottling/ref/Microsoft.AspNetCore.RequestThrottling.netcoreapp3.0.cs

@@ -13,6 +13,7 @@ namespace Microsoft.AspNetCore.RequestThrottling
     public partial class RequestThrottlingMiddleware
     {
         public RequestThrottlingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate next, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, Microsoft.Extensions.Options.IOptions<Microsoft.AspNetCore.RequestThrottling.RequestThrottlingOptions> options) { }
+        public int ActiveRequestCount { get { throw null; } }
         [System.Diagnostics.DebuggerStepThroughAttribute]
         public System.Threading.Tasks.Task Invoke(Microsoft.AspNetCore.Http.HttpContext context) { throw null; }
     }

src/Middleware/RequestThrottling/sample/RequestThrottlingSample.csproj

@@ -1,13 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
   </PropertyGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(BenchmarksTargetFramework)' == ''">
     <Reference Include="Microsoft.Extensions.Logging.Console" />
     <Reference Include="Microsoft.AspNetCore.RequestThrottling" />
     <Reference Include="Microsoft.AspNetCore.Server.Kestrel" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(BenchmarksTargetFramework)' != ''">
+    <PackageReference Include="Microsoft.AspNetCore.RequestThrottling" Version="$(MicrosoftAspNetCoreAppPackageVersion)" />
+
+    <FrameworkReference Update="Microsoft.AspNetCore.App" RuntimeFrameworkVersion="$(MicrosoftAspNetCoreAppPackageVersion)" />
+    <FrameworkReference Update="Microsoft.NETCore.App" RuntimeFrameworkVersion="$(MicrosoftNETCoreAppPackageVersion)" />
+  </ItemGroup>
+
 </Project>

src/Middleware/RequestThrottling/src/Internal/IRequestQueue.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information
+
+using System;
+using System.Threading.Tasks;
+
+namespace Microsoft.AspNetCore.RequestThrottling.Internal
+{
+    interface IRequestQueue : IDisposable
+    {
+        int TotalRequests { get; }
+
+        Task<bool> TryEnterQueueAsync();
+
+        void Release();
+    }
+}

src/Middleware/RequestThrottling/src/Internal/TailDrop.cs

@@ -7,7 +7,7 @@ using System.Threading.Tasks;
 
 namespace Microsoft.AspNetCore.RequestThrottling.Internal
 {
-    internal class RequestQueue : IDisposable
+    internal class TailDrop : IRequestQueue
     {
         private readonly int _maxConcurrentRequests;
         private readonly int _requestQueueLimit;
@@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.RequestThrottling.Internal
         private object _totalRequestsLock = new object();
         public int TotalRequests { get; private set; }
 
-        public RequestQueue(int maxConcurrentRequests, int requestQueueLimit)
+        public TailDrop(int maxConcurrentRequests, int requestQueueLimit)
         {
             _maxConcurrentRequests = maxConcurrentRequests;
             _requestQueueLimit = requestQueueLimit;

src/Middleware/RequestThrottling/src/Microsoft.AspNetCore.RequestThrottling.csproj

@@ -1,10 +1,11 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <Description>ASP.NET Core middleware for queuing incoming HTTP requests, to avoid threadpool starvation.</Description>
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <PackageTags>aspnetcore;queue;queuing</PackageTags>
+    <IsShippingPackage>true</IsShippingPackage>
   </PropertyGroup>
 
   <ItemGroup>

src/Middleware/RequestThrottling/src/RequestThrottlingMiddleware.cs

@@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.RequestThrottling
     /// </summary>
     public class RequestThrottlingMiddleware
     {
-        private readonly RequestQueue _requestQueue;
+        private readonly IRequestQueue _requestQueue;
         private readonly RequestDelegate _next;
         private readonly RequestThrottlingOptions _requestThrottlingOptions;
         private readonly ILogger _logger;
@@ -35,9 +35,9 @@ namespace Microsoft.AspNetCore.RequestThrottling
             {
                 throw new ArgumentException("The value of 'options.MaxConcurrentRequests' must be specified.", nameof(options));
             }
-            if (_requestThrottlingOptions.MaxConcurrentRequests < 0)
+            if (_requestThrottlingOptions.MaxConcurrentRequests <= 0)
             {
-                throw new ArgumentException("The value of 'options.MaxConcurrentRequests' must be a positive integer.", nameof(options));
+                throw new ArgumentOutOfRangeException(nameof(options), "The value of `options.MaxConcurrentRequests` must be a positive integer.");
             }
             if (_requestThrottlingOptions.RequestQueueLimit < 0)
             {
@@ -51,9 +51,16 @@ namespace Microsoft.AspNetCore.RequestThrottling
 
             _next = next;
             _logger = loggerFactory.CreateLogger<RequestThrottlingMiddleware>();
-            _requestQueue = new RequestQueue(
+
-                _requestThrottlingOptions.MaxConcurrentRequests.Value,
+            if (_requestThrottlingOptions.ServerAlwaysBlocks)
-                _requestThrottlingOptions.RequestQueueLimit);
+            {
+                // note: this option for testing only. Blocks all requests from entering the server.
+                _requestQueue = new TailDrop(0, _requestThrottlingOptions.RequestQueueLimit);
+            }
+            else
+            {
+                _requestQueue = new TailDrop(_requestThrottlingOptions.MaxConcurrentRequests.Value, _requestThrottlingOptions.RequestQueueLimit);
+            }
         }
 
         /// <summary>
@@ -64,25 +71,25 @@ namespace Microsoft.AspNetCore.RequestThrottling
         public async Task Invoke(HttpContext context)
         {
             var waitInQueueTask = _requestQueue.TryEnterQueueAsync();
-            if (waitInQueueTask.IsCompletedSuccessfully && !waitInQueueTask.Result)
+
+            if (waitInQueueTask.IsCompletedSuccessfully && waitInQueueTask.Result)
+            {
+                RequestThrottlingLog.RequestRunImmediately(_logger, ActiveRequestCount);
+            }
+            else
+            {
+                RequestThrottlingLog.RequestEnqueued(_logger, ActiveRequestCount);
+                await waitInQueueTask;
+                RequestThrottlingLog.RequestDequeued(_logger, ActiveRequestCount);
+            }
+
+            if (!waitInQueueTask.Result)
             {
                 RequestThrottlingLog.RequestRejectedQueueFull(_logger);
                 context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
                 await _requestThrottlingOptions.OnRejected(context);
                 return;
             }
-            else if (!waitInQueueTask.IsCompletedSuccessfully)
-            {
-                RequestThrottlingLog.RequestEnqueued(_logger, ActiveRequestCount);
-                var result = await waitInQueueTask;
-                RequestThrottlingLog.RequestDequeued(_logger, ActiveRequestCount);
-
-                Debug.Assert(result);
-            }
-            else
-            {
-                RequestThrottlingLog.RequestRunImmediately(_logger, ActiveRequestCount);
-            }
 
             try
             {
@@ -98,7 +105,7 @@ namespace Microsoft.AspNetCore.RequestThrottling
         /// The number of requests currently on the server.
         /// Cannot exceeed the sum of <see cref="RequestThrottlingOptions.RequestQueueLimit"> and </see>/><see cref="RequestThrottlingOptions.MaxConcurrentRequests"/>.
         /// </summary>
-        internal int ActiveRequestCount
+        public int ActiveRequestCount
         {
             get => _requestQueue.TotalRequests;
         }

src/Middleware/RequestThrottling/src/RequestThrottlingOptions.cs

@@ -33,5 +33,10 @@ namespace Microsoft.AspNetCore.RequestThrottling
         {
             return Task.CompletedTask;
         };
+
+        /// <summary>
+        /// For internal testing only. If true, no requests will enter the server.
+        /// </summary>
+        internal bool ServerAlwaysBlocks { get; set; } = false;
     }
 }

src/Middleware/RequestThrottling/test/MiddlewareTests.cs

@@ -84,8 +84,7 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
         [Fact]
         public async void RequestsBlockedIfQueueFull()
         {
-            var middleware = TestUtils.CreateTestMiddleware(
+            var middleware = TestUtils.CreateBlockingTestMiddleware(
-                maxConcurrentRequests: 0,
                 requestQueueLimit: 0,
                 next: httpContext =>
                 {
@@ -99,9 +98,7 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
         [Fact]
         public async void FullQueueResultsIn503Error()
         {
-            var middleware = TestUtils.CreateTestMiddleware(
+            var middleware = TestUtils.CreateBlockingTestMiddleware(requestQueueLimit: 0);
-                maxConcurrentRequests: 0,
-                requestQueueLimit: 0);
 
             var context = new DefaultHttpContext();
             await middleware.Invoke(context).OrTimeout();
@@ -112,7 +109,7 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
         public void MultipleRequestsFillUpQueue()
         {
             var middleware = TestUtils.CreateTestMiddleware(
-                maxConcurrentRequests: 0,
+                maxConcurrentRequests: 1,
                 requestQueueLimit: 10,
                 next: httpContext =>
                 {
@@ -133,8 +130,7 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
         {
             bool onRejectedInvoked = false;
 
-            var middleware = TestUtils.CreateTestMiddleware(
+            var middleware = TestUtils.CreateBlockingTestMiddleware(
-                maxConcurrentRequests: 0,
                 requestQueueLimit: 0,
                 onRejected: httpContext =>
                 {

src/Middleware/RequestThrottling/test/TestUtils.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System.Threading.Tasks;
-using Microsoft.AspNetCore.RequestThrottling;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
@@ -20,6 +19,23 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
                 RequestQueueLimit = requestQueueLimit
             };
 
+            return BuildFromOptions(options, onRejected, next);
+        }
+
+        public static RequestThrottlingMiddleware CreateBlockingTestMiddleware(int requestQueueLimit = 5000, RequestDelegate onRejected = null, RequestDelegate next = null)
+        {
+            var options = new RequestThrottlingOptions
+            {
+                MaxConcurrentRequests = 999,
+                RequestQueueLimit = requestQueueLimit,
+                ServerAlwaysBlocks = true
+            };
+
+            return BuildFromOptions(options, onRejected, next);
+        }
+
+        private static RequestThrottlingMiddleware BuildFromOptions(RequestThrottlingOptions options, RequestDelegate onRejected, RequestDelegate next)
+        {
             if (onRejected != null)
             {
                 options.OnRejected = onRejected;
@@ -32,6 +48,6 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
                 );
         }
 
-        internal static RequestQueue CreateRequestQueue(int maxConcurrentRequests) => new RequestQueue(maxConcurrentRequests, 5000);
+        internal static IRequestQueue CreateRequestQueue(int maxConcurrentRequests) => new TailDrop(maxConcurrentRequests, 5000);
     }
 }