Dylan/stack policy (#11293)

* Initial StackPolicy implementation

src/Middleware/RequestThrottling/perf/Microbenchmarks/QueueEmptyOverhead.cs

@@ -13,7 +13,8 @@ namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
     {
         private const int _numRequests = 20000;
 
-        private RequestThrottlingMiddleware _middleware;
+        private RequestThrottlingMiddleware _middlewareFIFO;
+        private RequestThrottlingMiddleware _middlewareLIFO;
         private RequestDelegate _restOfServer;
 
         [GlobalSetup]
@@ -21,9 +22,14 @@ namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
         {
             _restOfServer = YieldsThreadInternally ? (RequestDelegate)YieldsThread : (RequestDelegate)CompletesImmediately;
 
-            _middleware = TestUtils.CreateTestMiddleware_TailDrop(
+            _middlewareFIFO = TestUtils.CreateTestMiddleware_TailDrop(
                 maxConcurrentRequests: 1,
-                requestQueueLimit: 0,
+                requestQueueLimit: 100,
+                next: _restOfServer);
+
+            _middlewareLIFO = TestUtils.CreateTestMiddleware_StackPolicy(
+                maxConcurrentRequests: 1,
+                requestQueueLimit: 100,
                 next: _restOfServer);
         }
 
@@ -40,11 +46,20 @@ namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
         }
 
         [Benchmark(OperationsPerInvoke = _numRequests)]
-        public async Task WithEmptyQueueOverhead()
+        public async Task WithEmptyQueueOverhead_FIFO()
         {
             for (int i = 0; i < _numRequests; i++)
             {
-                await _middleware.Invoke(null);
+                await _middlewareFIFO.Invoke(null);
+            }
+        }
+
+        [Benchmark(OperationsPerInvoke = _numRequests)]
+        public async Task WithEmptyQueueOverhead_LIFO()
+        {
+            for (int i = 0; i < _numRequests; i++)
+            {
+                await _middlewareLIFO.Invoke(null);
             }
         }
 

src/Middleware/RequestThrottling/perf/Microbenchmarks/QueueFullOverhead.cs

@@ -11,11 +11,12 @@ namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
 {
     public class QueueFullOverhead
     {
-        private const int _numRequests = 2000;
+        private const int _numRequests = 200;
         private int _requestCount = 0;
         private ManualResetEventSlim _mres = new ManualResetEventSlim();
 
-        private RequestThrottlingMiddleware _middleware;
+        private RequestThrottlingMiddleware _middleware_FIFO;
+        private RequestThrottlingMiddleware _middleware_LIFO;
 
         [Params(8)]
         public int MaxConcurrentRequests;
@@ -23,11 +24,15 @@ namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
         [GlobalSetup]
         public void GlobalSetup()
         {
-            _middleware = TestUtils.CreateTestMiddleware_TailDrop(
+            _middleware_FIFO = TestUtils.CreateTestMiddleware_TailDrop(
                 maxConcurrentRequests: MaxConcurrentRequests,
                 requestQueueLimit: _numRequests,
-                next: IncrementAndCheck
-                );
+                next: IncrementAndCheck);
+
+            _middleware_LIFO = TestUtils.CreateTestMiddleware_StackPolicy(
+                maxConcurrentRequests: MaxConcurrentRequests,
+                requestQueueLimit: _numRequests,
+                next: IncrementAndCheck);
         }
 
         [IterationSetup]
@@ -59,14 +64,26 @@ namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
         }
 
         [Benchmark(OperationsPerInvoke = _numRequests)]
-        public void QueueingAll()
+        public void QueueingAll_FIFO()
         {
             for (int i = 0; i < _numRequests; i++)
             {
-                _ = _middleware.Invoke(null);
+                _ = _middleware_FIFO.Invoke(null);
             }
 
             _mres.Wait();
         }
+
+        [Benchmark(OperationsPerInvoke = _numRequests)]
+        public void QueueingAll_LIFO()
+        {
+            for (int i = 0; i < _numRequests; i++)
+            {
+                _ = _middleware_LIFO.Invoke(null);
+            }
+
+            _mres.Wait();
+        }
+
     }
 }

src/Middleware/RequestThrottling/ref/Microsoft.AspNetCore.RequestThrottling.netcoreapp3.0.cs

@@ -28,11 +28,11 @@ namespace Microsoft.AspNetCore.RequestThrottling
         public Microsoft.AspNetCore.Http.RequestDelegate OnRejected { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
     }
 }
-namespace Microsoft.AspNetCore.RequestThrottling.Policies
+namespace Microsoft.AspNetCore.RequestThrottling.QueuePolicies
 {
-    public partial class TailDropOptions
+    public partial class QueuePolicyOptions
     {
-        public TailDropOptions() { }
+        public QueuePolicyOptions() { }
         public int MaxConcurrentRequests { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
         public int RequestQueueLimit { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
     }
@@ -41,6 +41,7 @@ namespace Microsoft.Extensions.DependencyInjection
 {
     public static partial class QueuePolicyServiceCollectionExtensions
     {
-        public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddTailDropQueue(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action<Microsoft.AspNetCore.RequestThrottling.Policies.TailDropOptions> configure) { throw null; }
+        public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddStackQueue(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action<Microsoft.AspNetCore.RequestThrottling.QueuePolicies.QueuePolicyOptions> configure) { throw null; }
+        public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddTailDropQueue(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action<Microsoft.AspNetCore.RequestThrottling.QueuePolicies.QueuePolicyOptions> configure) { throw null; }
     }
 }

src/Middleware/RequestThrottling/sample/RequestThrottlingSample.csproj

@@ -8,6 +8,7 @@
     <Reference Include="Microsoft.Extensions.Logging.Console" />
     <Reference Include="Microsoft.AspNetCore.RequestThrottling" />
     <Reference Include="Microsoft.AspNetCore.Server.Kestrel" />
+    <Reference Include="Microsoft.Extensions.Configuration.CommandLine" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(BenchmarksTargetFramework)' != ''">

src/Middleware/RequestThrottling/sample/Startup.cs

@@ -1,11 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -14,14 +16,16 @@ namespace RequestThrottlingSample
 {
     public class Startup
     {
+        static IConfiguration _config;
+
         // This method gets called by the runtime. Use this method to add services to the container.
         // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddTailDropQueue((options) =>
             {
-                options.MaxConcurrentRequests = 4;
-                options.RequestQueueLimit = 0;
+                options.MaxConcurrentRequests = Math.Max(1, _config.GetValue<int>("maxCores"));
+                options.RequestQueueLimit = Math.Max(1, _config.GetValue<int>("maxQueue"));
             });
 
             services.AddLogging();
@@ -30,17 +34,21 @@ namespace RequestThrottlingSample
         public void Configure(IApplicationBuilder app, ILoggerFactory loggerFactory)
         {
             app.UseRequestThrottling();
-
             app.Run(async context =>
             {
-                await context.Response.WriteAsync("Hello Request Throttling! If you refresh this page a bunch, it will 503.");
-                await Task.Delay(1000);
+                await context.Response.WriteAsync("Hello Request Throttling! If you rapidly refresh this page, it will 503.");
+                await Task.Delay(400);
             });
         }
 
         // Entry point for the application.
         public static void Main(string[] args)
         {
+            _config = new ConfigurationBuilder()
+                            .AddEnvironmentVariables(prefix: "ASPNETCORE_")
+                            .AddCommandLine(args)
+                            .Build();
+
             var host = new WebHostBuilder()
                 .UseKestrel()
                 .UseContentRoot(Directory.GetCurrentDirectory()) // for the cert file

src/Middleware/RequestThrottling/src/Policies/QueuePolicyServiceCollectionExtensions.cs

@@ -1,34 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Text;
-using System.Xml.Schema;
-using Microsoft.AspNetCore.RequestThrottling;
-using Microsoft.AspNetCore.RequestThrottling.Policies;
-using Microsoft.Extensions.DependencyInjection.Extensions;
-
-namespace Microsoft.Extensions.DependencyInjection
-{
-    /// <summary>
-    /// Contains methods for adding Q
-    /// </summary>
-    public static class QueuePolicyServiceCollectionExtensions
-    {
-        /// <summary>
-        /// Tells <see cref="RequestThrottlingMiddleware"/> to use a TailDrop queue as its queueing strategy.
-        /// </summary>
-        /// <param name="services">The <see cref="IServiceCollection"/> to add services to.</param>
-        /// <param name="configure">Set the options used by the queue.
-        /// Mandatory, since <see cref="TailDropOptions.MaxConcurrentRequests"></see> must be provided.</param>
-        /// <returns></returns>
-        public static IServiceCollection AddTailDropQueue(this IServiceCollection services, Action<TailDropOptions> configure)
-        {
-            services.Configure<TailDropOptions>(configure);
-            services.AddSingleton<IQueuePolicy, TailDrop>();
-            return services;
-        }
-    }
-}

src/Middleware/RequestThrottling/src/QueuePolicies/QueuePolicyOptions.cs

@@ -2,12 +2,12 @@ using System;
 using System.Collections.Generic;
 using System.Text;
 
-namespace Microsoft.AspNetCore.RequestThrottling.Policies
+namespace Microsoft.AspNetCore.RequestThrottling.QueuePolicies
 {
     /// <summary>
-    /// Specifies options for the <see cref="TailDrop"/>
+    /// Specifies options for the <see cref="IQueuePolicy"/>
     /// </summary>
-    public class TailDropOptions
+    public class QueuePolicyOptions
     {
         /// <summary>
         /// Maximum number of concurrent requests. Any extras will be queued on the server. 

src/Middleware/RequestThrottling/src/QueuePolicies/QueuePolicyServiceCollectionExtensions.cs

@@ -0,0 +1,43 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.AspNetCore.RequestThrottling;
+using Microsoft.AspNetCore.RequestThrottling.QueuePolicies;
+
+namespace Microsoft.Extensions.DependencyInjection
+{
+    /// <summary>
+    /// Contains methods for specifying which queue the middleware should use.
+    /// </summary>
+    public static class QueuePolicyServiceCollectionExtensions
+    {
+        /// <summary>
+        /// Tells <see cref="RequestThrottlingMiddleware"/> to use a FIFO queue as its queueing strategy.
+        /// </summary>
+        /// <param name="services">The <see cref="IServiceCollection"/> to add services to.</param>
+        /// <param name="configure">Set the options used by the queue.
+        /// Mandatory, since <see cref="QueuePolicyOptions.MaxConcurrentRequests"></see> must be provided.</param>
+        /// <returns></returns>
+        public static IServiceCollection AddTailDropQueue(this IServiceCollection services, Action<QueuePolicyOptions> configure)
+        {
+            services.Configure(configure);
+            services.AddSingleton<IQueuePolicy, TailDropQueuePolicy>();
+            return services;
+        }
+
+        /// <summary>
+        /// Tells <see cref="RequestThrottlingMiddleware"/> to use a LIFO stack as its queueing strategy.
+        /// </summary>
+        /// <param name="services">The <see cref="IServiceCollection"/> to add services to.</param>
+        /// <param name="configure">Set the options used by the queue.
+        /// Mandatory, since <see cref="QueuePolicyOptions.MaxConcurrentRequests"></see> must be provided.</param>
+        /// <returns></returns>
+        public static IServiceCollection AddStackQueue(this IServiceCollection services, Action<QueuePolicyOptions> configure)
+        {
+            services.Configure(configure);
+            services.AddSingleton<IQueuePolicy, StackQueuePolicy>();
+            return services;
+        }
+    }
+}

src/Middleware/RequestThrottling/src/QueuePolicies/StackQueuePolicy.cs

@@ -0,0 +1,104 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.AspNetCore.RequestThrottling.QueuePolicies
+{
+    internal class StackQueuePolicy : IQueuePolicy
+    {
+        private readonly List<TaskCompletionSource<bool>> _buffer;
+        private readonly int _maxQueueCapacity;
+        private readonly int _maxConcurrentRequests;
+        private bool _hasReachedCapacity;
+        private int _head;
+        private int _queueLength;
+
+        private static readonly Task<bool> _trueTask = Task.FromResult(true);
+
+        private readonly object _bufferLock = new Object();
+
+        private int _freeServerSpots;
+
+        public StackQueuePolicy(IOptions<QueuePolicyOptions> options)
+        {
+            _buffer = new List<TaskCompletionSource<bool>>();
+            _maxQueueCapacity = options.Value.RequestQueueLimit;
+            _maxConcurrentRequests = options.Value.MaxConcurrentRequests;
+            _freeServerSpots = options.Value.MaxConcurrentRequests;
+        }
+
+        public Task<bool> TryEnterAsync()
+        {
+            lock (_bufferLock)
+            {
+                if (_freeServerSpots > 0)
+                {
+                    _freeServerSpots--;
+                    return _trueTask;
+                }
+
+                // if queue is full, cancel oldest request
+                if (_queueLength == _maxQueueCapacity)
+                {
+                    _hasReachedCapacity = true;
+                    _buffer[_head].SetResult(false);
+                    _queueLength--;
+                }
+
+                // enqueue request with a tcs
+                var tcs = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+                if (_hasReachedCapacity || _queueLength < _buffer.Count)
+                {
+                    _buffer[_head] = tcs;
+                }
+                else
+                {
+                    _buffer.Add(tcs);
+                }
+                _queueLength++;
+
+                // increment _head for next time
+                _head++;
+                if (_head == _maxQueueCapacity)
+                {
+                    _head = 0;
+                }
+                return tcs.Task;
+            }
+        }
+
+        public void OnExit()
+        {
+            lock (_bufferLock)
+            {
+                if (_queueLength == 0)
+                {
+                    _freeServerSpots++;
+
+                    if (_freeServerSpots > _maxConcurrentRequests)
+                    {
+                        _freeServerSpots--;
+                        throw new InvalidOperationException("OnExit must only be called once per successful call to TryEnterAsync");
+                    }
+
+                    return;
+                }
+
+                // step backwards and launch a new task
+                if (_head == 0)
+                {
+                    _head = _maxQueueCapacity - 1;
+                }
+                else
+                {
+                    _head--;
+                }
+
+                _buffer[_head].SetResult(true);
+                _buffer[_head] = null;
+                _queueLength--;
+            }
+        }
+    }
+}

src/Middleware/RequestThrottling/src/QueuePolicies/TailDropQueuePolicy.cs

@@ -6,9 +6,9 @@ using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Options;
 
-namespace Microsoft.AspNetCore.RequestThrottling.Policies
+namespace Microsoft.AspNetCore.RequestThrottling.QueuePolicies
 {
-    internal class TailDrop : IQueuePolicy, IDisposable
+    internal class TailDropQueuePolicy : IQueuePolicy, IDisposable
     {
         private readonly int _maxConcurrentRequests;
         private readonly int _requestQueueLimit;
@@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.RequestThrottling.Policies
         private object _totalRequestsLock = new object();
         public int TotalRequests { get; private set; }
 
-        public TailDrop(IOptions<TailDropOptions> options)
+        public TailDropQueuePolicy(IOptions<QueuePolicyOptions> options)
         {
             _maxConcurrentRequests = options.Value.MaxConcurrentRequests;
             if (_maxConcurrentRequests <= 0)

src/Middleware/RequestThrottling/src/RequestThrottlingMiddleware.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Xml.Schema;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;

src/Middleware/RequestThrottling/test/PolicyTests/StackQueueTests.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.RequestThrottling.QueuePolicies;
+using Microsoft.Extensions.Options;
+using Xunit;
+
+namespace Microsoft.AspNetCore.RequestThrottling.Tests.PolicyTests
+{
+    public static class StackQueueTests
+    {
+        [Fact]
+        public static void BaseFunctionality()
+        {
+            var stack = new StackQueuePolicy(Options.Create(new QueuePolicyOptions {
+                MaxConcurrentRequests = 0,
+                RequestQueueLimit = 2,
+            }));
+
+            var task1 = stack.TryEnterAsync();
+
+            Assert.False(task1.IsCompleted);
+
+            stack.OnExit();
+
+            Assert.True(task1.IsCompleted && task1.Result);
+        }
+
+        [Fact]
+        public static void OldestRequestOverwritten()
+        {
+            var stack = new StackQueuePolicy(Options.Create(new QueuePolicyOptions {
+                MaxConcurrentRequests = 0,
+                RequestQueueLimit = 3,
+            }));
+
+            var task1 = stack.TryEnterAsync();
+            Assert.False(task1.IsCompleted);
+            var task2 = stack.TryEnterAsync();
+            Assert.False(task2.IsCompleted);
+            var task3 = stack.TryEnterAsync();
+            Assert.False(task3.IsCompleted);
+
+            var task4 = stack.TryEnterAsync();
+            Assert.False(task4.IsCompleted);
+
+            Assert.True(task1.IsCompleted);
+            Assert.False(task1.Result);
+        }
+
+        [Fact]
+        public static void RespectsMaxConcurrency()
+        {
+            var stack = new StackQueuePolicy(Options.Create(new QueuePolicyOptions {
+                MaxConcurrentRequests = 2,
+                RequestQueueLimit = 2,
+            }));
+
+            var task1 = stack.TryEnterAsync();
+            Assert.True(task1.IsCompleted);
+
+            var task2 = stack.TryEnterAsync();
+            Assert.True(task2.IsCompleted);
+
+            var task3 = stack.TryEnterAsync();
+            Assert.False(task3.IsCompleted);
+        }
+
+        [Fact]
+        public static void ExitRequestsPreserveSemaphoreState()
+        {
+            var stack = new StackQueuePolicy(Options.Create(new QueuePolicyOptions {
+                MaxConcurrentRequests = 1,
+                RequestQueueLimit = 2,
+            }));
+
+            var task1 = stack.TryEnterAsync();
+            Assert.True(task1.IsCompleted && task1.Result);
+
+            var task2 = stack.TryEnterAsync();
+            Assert.False(task2.IsCompleted);
+
+            stack.OnExit();  // t1 exits, should free t2 to return
+            Assert.True(task2.IsCompleted && task2.Result);
+
+            stack.OnExit();  // t2 exists, there's now a free spot in server
+
+            var task3 = stack.TryEnterAsync();
+            Assert.True(task3.IsCompleted && task3.Result);
+        }
+
+        [Fact]
+        public static void StaleRequestsAreProperlyOverwritten()
+        {
+            var stack = new StackQueuePolicy(Options.Create(new QueuePolicyOptions
+            {
+                MaxConcurrentRequests = 0,
+                RequestQueueLimit = 4,
+            }));
+
+            var task1 = stack.TryEnterAsync();
+            stack.OnExit();
+            Assert.True(task1.IsCompleted);
+
+            var task2 = stack.TryEnterAsync();
+            stack.OnExit();
+            Assert.True(task2.IsCompleted);
+        }
+
+        [Fact]
+        public static async Task OneTryEnterAsyncOneOnExit()
+        {
+            var stack = new StackQueuePolicy(Options.Create(new QueuePolicyOptions
+            {
+                MaxConcurrentRequests = 1,
+                RequestQueueLimit = 4,
+            }));
+
+            Assert.Throws<InvalidOperationException>(() => stack.OnExit());
+
+            await stack.TryEnterAsync();
+
+            stack.OnExit();
+
+            Assert.Throws<InvalidOperationException>(() => stack.OnExit());
+        }
+    } 
+}

src/Middleware/RequestThrottling/test/TestUtils.cs

@@ -5,7 +5,7 @@ using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.RequestThrottling.Policies;
+using Microsoft.AspNetCore.RequestThrottling.QueuePolicies;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 
@@ -37,15 +37,35 @@ namespace Microsoft.AspNetCore.RequestThrottling.Tests
                 );
         }
 
-        internal static TailDrop CreateTailDropQueue(int maxConcurrentRequests, int requestQueueLimit = 5000)
+        public static RequestThrottlingMiddleware CreateTestMiddleware_StackPolicy(int maxConcurrentRequests, int requestQueueLimit, RequestDelegate onRejected = null, RequestDelegate next = null)
         {
-            var options = Options.Create(new TailDropOptions
+            return CreateTestMiddleware(
+                queue: CreateStackPolicy(maxConcurrentRequests, requestQueueLimit),
+                onRejected: onRejected,
+                next: next
+                );
+        }
+
+        internal static StackQueuePolicy CreateStackPolicy(int maxConcurrentRequests, int requestsQueuelimit = 100)
+        {
+            var options = Options.Create(new QueuePolicyOptions
+            {
+                MaxConcurrentRequests = maxConcurrentRequests,
+                RequestQueueLimit = requestsQueuelimit
+            });
+
+            return new StackQueuePolicy(options);
+        }
+
+        internal static TailDropQueuePolicy CreateTailDropQueue(int maxConcurrentRequests, int requestQueueLimit = 100)
+        {
+            var options = Options.Create(new QueuePolicyOptions
             {
                 MaxConcurrentRequests = maxConcurrentRequests,
                 RequestQueueLimit = requestQueueLimit
             });
 
-            return new TailDrop(options);
+            return new TailDropQueuePolicy(options);
         }
     }