diff --git a/benchmarks/Kestrel.Performance/FrameFeatureCollection.cs b/benchmarks/Kestrel.Performance/FrameFeatureCollection.cs
index 2e157f41d7..5bb4f8603a 100644
--- a/benchmarks/Kestrel.Performance/FrameFeatureCollection.cs
+++ b/benchmarks/Kestrel.Performance/FrameFeatureCollection.cs
@@ -85,6 +85,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             var frameContext = new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = new PipeFactory()
             };
 
diff --git a/benchmarks/Kestrel.Performance/FrameParsingOverheadBenchmark.cs b/benchmarks/Kestrel.Performance/FrameParsingOverheadBenchmark.cs
index f27fb8c947..dd905485c4 100644
--- a/benchmarks/Kestrel.Performance/FrameParsingOverheadBenchmark.cs
+++ b/benchmarks/Kestrel.Performance/FrameParsingOverheadBenchmark.cs
@@ -3,6 +3,7 @@
 
 using System.IO.Pipelines;
 using BenchmarkDotNet.Attributes;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -29,6 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             var frameContext = new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = new PipeFactory(),
                 TimeoutControl = new MockTimeoutControl()
             };
diff --git a/benchmarks/Kestrel.Performance/FrameWritingBenchmark.cs b/benchmarks/Kestrel.Performance/FrameWritingBenchmark.cs
index 82f943f525..bd280dec1d 100644
--- a/benchmarks/Kestrel.Performance/FrameWritingBenchmark.cs
+++ b/benchmarks/Kestrel.Performance/FrameWritingBenchmark.cs
@@ -7,6 +7,7 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -107,6 +108,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             var frame = new TestFrame<object>(application: null, context: new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = pipeFactory,
                 Application = pair.Application,
                 Transport = pair.Transport
diff --git a/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs b/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs
index b4f14656e4..406c7a9f87 100644
--- a/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs
+++ b/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs
@@ -3,6 +3,7 @@
 
 using System.IO.Pipelines;
 using BenchmarkDotNet.Attributes;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -33,6 +34,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             var frameContext = new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = PipeFactory,
                 TimeoutControl = new MockTimeoutControl()
             };
diff --git a/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs b/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs
index 54f7907ce3..c8d26ed2dc 100644
--- a/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs
+++ b/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs
@@ -6,6 +6,7 @@ using System.Runtime.CompilerServices;
 using System.Text;
 using BenchmarkDotNet.Attributes;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Http.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
@@ -177,6 +178,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             var frameContext = new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = new PipeFactory()
             };
 
diff --git a/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs b/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs
index 52ffb6bb77..ab5c5dd260 100644
--- a/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs
+++ b/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs
@@ -7,6 +7,7 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -123,6 +124,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             var frame = new TestFrame<object>(application: null, context: new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = pipeFactory,
                 TimeoutControl = new MockTimeoutControl(),
                 Application = pair.Application,
diff --git a/src/Kestrel.Core/Features/IDecrementConcurrentConnectionCountFeature.cs b/src/Kestrel.Core/Features/IDecrementConcurrentConnectionCountFeature.cs
new file mode 100644
index 0000000000..d34b1d1439
--- /dev/null
+++ b/src/Kestrel.Core/Features/IDecrementConcurrentConnectionCountFeature.cs
@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Features
+{
+    /// <summary>
+    /// A connection feature allowing middleware to stop counting connections towards <see cref="KestrelServerLimits.MaxConcurrentConnections"/>.
+    /// This is used by Kestrel internally to stop counting upgraded connections towards this limit.
+    /// </summary>
+    public interface IDecrementConcurrentConnectionCountFeature
+    {
+        /// <summary>
+        /// Idempotent method to stop counting a connection towards <see cref="KestrelServerLimits.MaxConcurrentConnections"/>.
+        /// </summary>
+        void ReleaseConnection();
+    }
+}
diff --git a/src/Kestrel.Core/Internal/ConnectionLimitBuilderExtensions.cs b/src/Kestrel.Core/Internal/ConnectionLimitBuilderExtensions.cs
deleted file mode 100644
index 2c02fa664c..0000000000
--- a/src/Kestrel.Core/Internal/ConnectionLimitBuilderExtensions.cs
+++ /dev/null
@@ -1,16 +0,0 @@
-using Microsoft.AspNetCore.Protocols;
-
-namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
-{
-    public static class ConnectionLimitBuilderExtensions
-    {
-        public static IConnectionBuilder UseConnectionLimit(this IConnectionBuilder builder, ServiceContext serviceContext)
-        {
-            return builder.Use(next =>
-            {
-                var middleware = new ConnectionLimitMiddleware(next, serviceContext);
-                return middleware.OnConnectionAsync;
-            });
-        }
-    }
-}
diff --git a/src/Kestrel.Core/Internal/ConnectionLimitMiddleware.cs b/src/Kestrel.Core/Internal/ConnectionLimitMiddleware.cs
index cd058bc235..fd6823773f 100644
--- a/src/Kestrel.Core/Internal/ConnectionLimitMiddleware.cs
+++ b/src/Kestrel.Core/Internal/ConnectionLimitMiddleware.cs
@@ -1,32 +1,74 @@
-using System.Threading.Tasks;
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
 using Microsoft.AspNetCore.Protocols;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
 {
     public class ConnectionLimitMiddleware
     {
-        private readonly ServiceContext _serviceContext;
         private readonly ConnectionDelegate _next;
+        private readonly ResourceCounter _concurrentConnectionCounter;
+        private readonly IKestrelTrace _trace;
 
-        public ConnectionLimitMiddleware(ConnectionDelegate next, ServiceContext serviceContext)
+        public ConnectionLimitMiddleware(ConnectionDelegate next, long connectionLimit, IKestrelTrace trace)
+            : this(next, ResourceCounter.Quota(connectionLimit), trace)
         {
-            _next = next;
-            _serviceContext = serviceContext;
         }
 
-        public Task OnConnectionAsync(ConnectionContext connection)
+        // For Testing
+        internal ConnectionLimitMiddleware(ConnectionDelegate next, ResourceCounter concurrentConnectionCounter, IKestrelTrace trace)
         {
-            if (!_serviceContext.ConnectionManager.NormalConnectionCount.TryLockOne())
+            _next = next;
+            _concurrentConnectionCounter = concurrentConnectionCounter;
+            _trace = trace;
+        }
+
+        public async Task OnConnectionAsync(ConnectionContext connection)
+        {
+            if (!_concurrentConnectionCounter.TryLockOne())
             {
                 KestrelEventSource.Log.ConnectionRejected(connection.ConnectionId);
-                _serviceContext.Log.ConnectionRejected(connection.ConnectionId);
+                _trace.ConnectionRejected(connection.ConnectionId);
                 connection.Transport.Input.Complete();
                 connection.Transport.Output.Complete();
-                return Task.CompletedTask;
+                return;
             }
 
-            return _next(connection);
+            var releasor = new ConnectionReleasor(_concurrentConnectionCounter);
+
+            try
+            {
+                connection.Features.Set<IDecrementConcurrentConnectionCountFeature>(releasor);
+                await _next(connection);
+            }
+            finally
+            {
+                releasor.ReleaseConnection();
+            }
+        }
+
+        private class ConnectionReleasor : IDecrementConcurrentConnectionCountFeature
+        {
+            private readonly ResourceCounter _concurrentConnectionCounter;
+            private bool _connectionReleased;
+
+            public ConnectionReleasor(ResourceCounter normalConnectionCounter)
+            {
+                _concurrentConnectionCounter = normalConnectionCounter;
+            }
+
+            public void ReleaseConnection()
+            {
+                if (!_connectionReleased)
+                {
+                    _connectionReleased = true;
+                    _concurrentConnectionCounter.ReleaseOne();
+                }
+            }
         }
     }
 }
diff --git a/src/Kestrel.Core/Internal/FrameConnection.cs b/src/Kestrel.Core/Internal/FrameConnection.cs
index f2e3a3e172..9232c515da 100644
--- a/src/Kestrel.Core/Internal/FrameConnection.cs
+++ b/src/Kestrel.Core/Internal/FrameConnection.cs
@@ -141,7 +141,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
                     adaptedPipelineTask = adaptedPipeline.RunAsync(stream);
                 }
 
-                if (_frame.ConnectionFeatures?.Get<ITlsApplicationProtocolFeature>()?.ApplicationProtocol == "h2" &&
+                if (_frame.ConnectionFeatures.Get<ITlsApplicationProtocolFeature>()?.ApplicationProtocol == "h2" &&
                     Interlocked.CompareExchange(ref _http2ConnectionState, Http2ConnectionStarted, Http2ConnectionNotStarted) == Http2ConnectionNotStarted)
                 {
                     await _http2Connection.ProcessAsync(httpApplication);
@@ -167,10 +167,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
                 {
                     _context.ServiceContext.ConnectionManager.UpgradedConnectionCount.ReleaseOne();
                 }
-                else
-                {
-                    _context.ServiceContext.ConnectionManager.NormalConnectionCount.ReleaseOne();
-                }
 
                 KestrelEventSource.Log.ConnectionStop(this);
             }
@@ -181,6 +177,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
             _frame = new Frame<TContext>(httpApplication, new FrameContext
             {
                 ConnectionId = _context.ConnectionId,
+                ConnectionFeatures = _context.ConnectionFeatures,
                 PipeFactory = PipeFactory,
                 LocalEndPoint = LocalEndPoint,
                 RemoteEndPoint = RemoteEndPoint,
@@ -255,10 +252,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
         {
             Debug.Assert(_frame != null, $"{nameof(_frame)} is null");
 
-            var features = new FeatureCollection();
             var connectionAdapters = _context.ConnectionAdapters;
             var stream = new RawStream(_context.Transport.Input, _context.Transport.Output);
-            var adapterContext = new ConnectionAdapterContext(features, stream);
+            var adapterContext = new ConnectionAdapterContext(_frame.ConnectionFeatures, stream);
             _adaptedConnections = new List<IAdaptedConnection>(connectionAdapters.Count);
 
             try
@@ -267,7 +263,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
                 {
                     var adaptedConnection = await connectionAdapters[i].OnConnectionAsync(adapterContext);
                     _adaptedConnections.Add(adaptedConnection);
-                    adapterContext = new ConnectionAdapterContext(features, adaptedConnection.ConnectionStream);
+                    adapterContext = new ConnectionAdapterContext(_frame.ConnectionFeatures, adaptedConnection.ConnectionStream);
                 }
             }
             catch (Exception ex)
@@ -276,10 +272,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
 
                 return null;
             }
-            finally
-            {
-                _frame.ConnectionFeatures = features;
-            }
 
             return adapterContext.ConnectionStream;
         }
diff --git a/src/Kestrel.Core/Internal/FrameConnectionContext.cs b/src/Kestrel.Core/Internal/FrameConnectionContext.cs
index 5f5c5c2dfa..c8beb6ab45 100644
--- a/src/Kestrel.Core/Internal/FrameConnectionContext.cs
+++ b/src/Kestrel.Core/Internal/FrameConnectionContext.cs
@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.IO.Pipelines;
 using System.Net;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Adapter.Internal;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
@@ -13,6 +14,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
         public string ConnectionId { get; set; }
         public long FrameConnectionId { get; set; }
         public ServiceContext ServiceContext { get; set; }
+        public IFeatureCollection ConnectionFeatures { get; set; }
         public IList<IConnectionAdapter> ConnectionAdapters { get; set; }
         public PipeFactory PipeFactory { get; set; }
         public IPEndPoint LocalEndPoint { get; set; }
diff --git a/src/Kestrel.Core/Internal/Http/Frame.FeatureCollection.cs b/src/Kestrel.Core/Internal/Http/Frame.FeatureCollection.cs
index f2457d0696..c922e2d1cf 100644
--- a/src/Kestrel.Core/Internal/Http/Frame.FeatureCollection.cs
+++ b/src/Kestrel.Core/Internal/Http/Frame.FeatureCollection.cs
@@ -251,13 +251,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
         object IFeatureCollection.this[Type key]
         {
-            get => FastFeatureGet(key) ?? ConnectionFeatures?[key];
+            get => FastFeatureGet(key) ?? ConnectionFeatures[key];
             set => FastFeatureSet(key, value);
         }
 
         TFeature IFeatureCollection.Get<TFeature>()
         {
-            return (TFeature)(FastFeatureGet(typeof(TFeature)) ?? ConnectionFeatures?[typeof(TFeature)]);
+            return (TFeature)(FastFeatureGet(typeof(TFeature)) ?? ConnectionFeatures[typeof(TFeature)]);
         }
 
         void IFeatureCollection.Set<TFeature>(TFeature instance)
@@ -294,7 +294,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
             _wasUpgraded = true;
 
-            ServiceContext.ConnectionManager.NormalConnectionCount.ReleaseOne();
+            ConnectionFeatures.Get<IDecrementConcurrentConnectionCountFeature>()?.ReleaseConnection();
 
             StatusCode = StatusCodes.Status101SwitchingProtocols;
             ReasonPhrase = "Switching Protocols";
diff --git a/src/Kestrel.Core/Internal/Http/Frame.cs b/src/Kestrel.Core/Internal/Http/Frame.cs
index 3c56334f26..6195d79d5e 100644
--- a/src/Kestrel.Core/Internal/Http/Frame.cs
+++ b/src/Kestrel.Core/Internal/Http/Frame.cs
@@ -107,7 +107,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         private IPEndPoint LocalEndPoint => _frameContext.LocalEndPoint;
         private IPEndPoint RemoteEndPoint => _frameContext.RemoteEndPoint;
 
-        public IFeatureCollection ConnectionFeatures { get; set; }
+        public IFeatureCollection ConnectionFeatures => _frameContext.ConnectionFeatures;
         public IPipeReader Input => _frameContext.Transport.Input;
         public OutputProducer Output { get; }
         public ITimeoutControl TimeoutControl => _frameContext.TimeoutControl;
diff --git a/src/Kestrel.Core/Internal/Http/FrameContext.cs b/src/Kestrel.Core/Internal/Http/FrameContext.cs
index 5a52c03ea5..399c9ac6fe 100644
--- a/src/Kestrel.Core/Internal/Http/FrameContext.cs
+++ b/src/Kestrel.Core/Internal/Http/FrameContext.cs
@@ -3,6 +3,7 @@
 
 using System.IO.Pipelines;
 using System.Net;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Protocols;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
@@ -12,6 +13,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
     {
         public string ConnectionId { get; set; }
         public ServiceContext ServiceContext { get; set; }
+        public IFeatureCollection ConnectionFeatures { get; set; }
         public PipeFactory PipeFactory { get; set; }
         public IPEndPoint RemoteEndPoint { get; set; }
         public IPEndPoint LocalEndPoint { get; set; }
diff --git a/src/Kestrel.Core/Internal/HttpConnectionMiddleware.cs b/src/Kestrel.Core/Internal/HttpConnectionMiddleware.cs
index b47ea9d603..62c51fb0f5 100644
--- a/src/Kestrel.Core/Internal/HttpConnectionMiddleware.cs
+++ b/src/Kestrel.Core/Internal/HttpConnectionMiddleware.cs
@@ -43,6 +43,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
                 ConnectionId = connectionContext.ConnectionId,
                 FrameConnectionId = frameConnectionId,
                 ServiceContext = _serviceContext,
+                ConnectionFeatures = connectionContext.Features,
                 PipeFactory = connectionContext.PipeFactory,
                 ConnectionAdapters = _connectionAdapters,
                 Transport = connectionContext.Transport,
diff --git a/src/Kestrel.Core/Internal/Infrastructure/FrameConnectionManager.cs b/src/Kestrel.Core/Internal/Infrastructure/FrameConnectionManager.cs
index ed4c2d44ae..dc4d969ad7 100644
--- a/src/Kestrel.Core/Internal/Infrastructure/FrameConnectionManager.cs
+++ b/src/Kestrel.Core/Internal/Infrastructure/FrameConnectionManager.cs
@@ -11,23 +11,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
         private readonly ConcurrentDictionary<long, FrameConnectionReference> _connectionReferences = new ConcurrentDictionary<long, FrameConnectionReference>();
         private readonly IKestrelTrace _trace;
 
-        public FrameConnectionManager(IKestrelTrace trace, long? normalConnectionLimit, long? upgradedConnectionLimit)
-            : this(trace, GetCounter(normalConnectionLimit), GetCounter(upgradedConnectionLimit))
+        public FrameConnectionManager(IKestrelTrace trace, long? upgradedConnectionLimit)
+            : this(trace, GetCounter(upgradedConnectionLimit))
         {
         }
 
-        public FrameConnectionManager(IKestrelTrace trace, ResourceCounter normalConnections, ResourceCounter upgradedConnections)
+        public FrameConnectionManager(IKestrelTrace trace, ResourceCounter upgradedConnections)
         {
-            NormalConnectionCount = normalConnections;
             UpgradedConnectionCount = upgradedConnections;
             _trace = trace;
         }
 
-        /// <summary>
-        /// TCP connections processed by Kestrel.
-        /// </summary>
-        public ResourceCounter NormalConnectionCount { get; }
-
         /// <summary>
         /// Connections that have been switched to a different protocol.
         /// </summary>
diff --git a/src/Kestrel.Core/KestrelServer.cs b/src/Kestrel.Core/KestrelServer.cs
index a1f5864f97..528ea80e7a 100644
--- a/src/Kestrel.Core/KestrelServer.cs
+++ b/src/Kestrel.Core/KestrelServer.cs
@@ -70,7 +70,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
             var trace = new KestrelTrace(logger);
             var connectionManager = new FrameConnectionManager(
                 trace,
-                serverOptions.Limits.MaxConcurrentConnections,
                 serverOptions.Limits.MaxConcurrentUpgradedConnections);
 
             var systemClock = new SystemClock();
@@ -135,16 +134,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
 
                 async Task OnBind(ListenOptions endpoint)
                 {
-                    // Add the connection limit middleware
-                    endpoint.UseConnectionLimit(ServiceContext);
-
-                    // Configure the user delegate
-                    endpoint.Configure(endpoint);
-
                     // Add the HTTP middleware as the terminal connection middleware
                     endpoint.UseHttpServer(endpoint.ConnectionAdapters, ServiceContext, application);
 
-                    var connectionHandler = new ConnectionHandler(ServiceContext, endpoint.Build()); 
+                    var connectionDelegate = endpoint.Build();
+
+                    // Add the connection limit middleware
+                    if (Options.Limits.MaxConcurrentConnections.HasValue)
+                    {
+                        connectionDelegate = new ConnectionLimitMiddleware(connectionDelegate, Options.Limits.MaxConcurrentConnections.Value, Trace).OnConnectionAsync;
+                    }
+
+                    var connectionHandler = new ConnectionHandler(ServiceContext, connectionDelegate);
                     var transport = _transportFactory.Create(endpoint, connectionHandler);
                     _transports.Add(transport);
 
diff --git a/src/Kestrel.Core/KestrelServerLimits.cs b/src/Kestrel.Core/KestrelServerLimits.cs
index eb061cf500..6cd16e9a6b 100644
--- a/src/Kestrel.Core/KestrelServerLimits.cs
+++ b/src/Kestrel.Core/KestrelServerLimits.cs
@@ -201,7 +201,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
         }
 
         /// <summary>
-        /// Gets or sets the maximum number of open HTTP/S connections. When set to null, the number of connections is unlimited.
+        /// Gets or sets the maximum number of open connections. When set to null, the number of connections is unlimited.
         /// </summary>
         /// <remarks>
         /// <para>
diff --git a/src/Kestrel.Core/KestrelServerOptions.cs b/src/Kestrel.Core/KestrelServerOptions.cs
index 6a33639710..136b983533 100644
--- a/src/Kestrel.Core/KestrelServerOptions.cs
+++ b/src/Kestrel.Core/KestrelServerOptions.cs
@@ -100,7 +100,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
                 throw new ArgumentNullException(nameof(configure));
             }
 
-            var listenOptions = new ListenOptions(endPoint) { KestrelServerOptions = this, Configure = configure };
+            var listenOptions = new ListenOptions(endPoint) { KestrelServerOptions = this };
+            configure(listenOptions);
             ListenOptions.Add(listenOptions);
         }
 
@@ -131,7 +132,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
                 throw new ArgumentNullException(nameof(configure));
             }
 
-            var listenOptions = new ListenOptions(socketPath) { KestrelServerOptions = this, Configure = configure };
+            var listenOptions = new ListenOptions(socketPath) { KestrelServerOptions = this };
+            configure(listenOptions);
             ListenOptions.Add(listenOptions);
         }
 
@@ -154,7 +156,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
                 throw new ArgumentNullException(nameof(configure));
             }
 
-            var listenOptions = new ListenOptions(handle) { KestrelServerOptions = this, Configure = configure };
+            var listenOptions = new ListenOptions(handle) { KestrelServerOptions = this };
+            configure(listenOptions);
             ListenOptions.Add(listenOptions);
         }
     }
diff --git a/src/Kestrel.Core/ListenOptions.cs b/src/Kestrel.Core/ListenOptions.cs
index f1f66209a2..f56b3c98be 100644
--- a/src/Kestrel.Core/ListenOptions.cs
+++ b/src/Kestrel.Core/ListenOptions.cs
@@ -131,8 +131,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
 
         public IServiceProvider ApplicationServices => KestrelServerOptions?.ApplicationServices;
 
-        internal Action<ListenOptions> Configure { get; set; } = _ => { };
-
         /// <summary>
         /// Gets the name of this endpoint to display on command-line when the web server starts.
         /// </summary>
diff --git a/test/Kestrel.Core.Tests/FrameConnectionManagerTests.cs b/test/Kestrel.Core.Tests/FrameConnectionManagerTests.cs
index a6e6bb4879..a476c32ad1 100644
--- a/test/Kestrel.Core.Tests/FrameConnectionManagerTests.cs
+++ b/test/Kestrel.Core.Tests/FrameConnectionManagerTests.cs
@@ -18,7 +18,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         {
             var connectionId = "0";
             var trace = new Mock<IKestrelTrace>();
-            var frameConnectionManager = new FrameConnectionManager(trace.Object, ResourceCounter.Unlimited, ResourceCounter.Unlimited);
+            var frameConnectionManager = new FrameConnectionManager(trace.Object, ResourceCounter.Unlimited);
 
             // Create FrameConnection in inner scope so it doesn't get rooted by the current frame.
             UnrootedConnectionsGetRemovedFromHeartbeatInnerScope(connectionId, frameConnectionManager, trace);
diff --git a/test/Kestrel.Core.Tests/FrameConnectionTests.cs b/test/Kestrel.Core.Tests/FrameConnectionTests.cs
index b2418bf61a..847910d677 100644
--- a/test/Kestrel.Core.Tests/FrameConnectionTests.cs
+++ b/test/Kestrel.Core.Tests/FrameConnectionTests.cs
@@ -2,11 +2,10 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.IO.Pipelines;
 using System.Collections.Generic;
-using System.Linq;
+using System.IO.Pipelines;
 using System.Threading;
-using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Adapter.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
@@ -31,6 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             {
                 ConnectionId = "0123456789",
                 ConnectionAdapters = new List<IConnectionAdapter>(),
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = _pipeFactory,
                 FrameConnectionId = long.MinValue,
                 Application = pair.Application,
diff --git a/test/Kestrel.Core.Tests/FrameResponseHeadersTests.cs b/test/Kestrel.Core.Tests/FrameResponseHeadersTests.cs
index 7c3dfa0f08..0bc399629c 100644
--- a/test/Kestrel.Core.Tests/FrameResponseHeadersTests.cs
+++ b/test/Kestrel.Core.Tests/FrameResponseHeadersTests.cs
@@ -2,10 +2,11 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.IO.Pipelines;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO.Pipelines;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 using Microsoft.AspNetCore.Testing;
 using Microsoft.Extensions.Primitives;
@@ -23,6 +24,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             var frameContext = new FrameContext
             {
                 ServiceContext = new TestServiceContext(),
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = factory,
                 Application = pair.Application,
                 Transport = pair.Transport,
diff --git a/test/Kestrel.Core.Tests/FrameTests.cs b/test/Kestrel.Core.Tests/FrameTests.cs
index 57c8287284..eb7317c498 100644
--- a/test/Kestrel.Core.Tests/FrameTests.cs
+++ b/test/Kestrel.Core.Tests/FrameTests.cs
@@ -63,6 +63,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             _frameContext = new FrameContext
             {
                 ServiceContext = _serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = _pipelineFactory,
                 TimeoutControl = _timeoutControl.Object,
                 Application = pair.Application,
diff --git a/test/Kestrel.Core.Tests/KestrelServerOptionsTests.cs b/test/Kestrel.Core.Tests/KestrelServerOptionsTests.cs
index b34f92a1d4..2597862f80 100644
--- a/test/Kestrel.Core.Tests/KestrelServerOptionsTests.cs
+++ b/test/Kestrel.Core.Tests/KestrelServerOptionsTests.cs
@@ -18,9 +18,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
                 d.NoDelay = false;
             });
 
-            // Execute the callback
-            o1.ListenOptions[1].Configure(o1.ListenOptions[1]);
-
             Assert.True(o1.ListenOptions[0].NoDelay);
             Assert.False(o1.ListenOptions[1].NoDelay);
         }
diff --git a/test/Kestrel.Core.Tests/TestInput.cs b/test/Kestrel.Core.Tests/TestInput.cs
index 220f9393cd..9033c90d24 100644
--- a/test/Kestrel.Core.Tests/TestInput.cs
+++ b/test/Kestrel.Core.Tests/TestInput.cs
@@ -4,6 +4,7 @@
 using System;
 using System.IO.Pipelines;
 using System.Text;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.AspNetCore.Testing;
@@ -27,6 +28,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             FrameContext = new FrameContext
             {
                 ServiceContext = new TestServiceContext(),
+                ConnectionFeatures = new FeatureCollection(),
                 Application = Application,
                 Transport = Transport,
                 PipeFactory = _pipelineFactory,
diff --git a/test/Kestrel.FunctionalTests/ConnectionLimitTests.cs b/test/Kestrel.FunctionalTests/ConnectionLimitTests.cs
index ffcde60a9d..0e63b8e88b 100644
--- a/test/Kestrel.FunctionalTests/ConnectionLimitTests.cs
+++ b/test/Kestrel.FunctionalTests/ConnectionLimitTests.cs
@@ -2,12 +2,13 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.IO;
-using System.Net.Sockets;
+using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.AspNetCore.Server.Kestrel.Tests;
 using Microsoft.AspNetCore.Testing;
@@ -23,15 +24,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
             var requestTcs = new TaskCompletionSource<object>();
             var releasedTcs = new TaskCompletionSource<object>();
             var lockedTcs = new TaskCompletionSource<bool>();
-            var (serviceContext, counter) = SetupMaxConnections(max: 1);
+            var counter = new EventRaisingResourceCounter(ResourceCounter.Quota(1));
             counter.OnLock += (s, e) => lockedTcs.TrySetResult(e);
             counter.OnRelease += (s, e) => releasedTcs.TrySetResult(null);
 
-            using (var server = new TestServer(async context =>
+            using (var server = CreateServerWithMaxConnections(async context =>
             {
                 await context.Response.WriteAsync("Hello");
                 await requestTcs.Task;
-            }, serviceContext))
+            }, counter))
             using (var connection = server.CreateConnection())
             {
                 await connection.SendEmptyGetAsKeepAlive(); ;
@@ -46,8 +47,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
         [Fact]
         public async Task UpgradedConnectionsCountsAgainstDifferentLimit()
         {
-            var (serviceContext, _) = SetupMaxConnections(max: 1);
-            using (var server = new TestServer(async context =>
+            using (var server = CreateServerWithMaxConnections(async context =>
             {
                 var feature = context.Features.Get<IHttpUpgradeFeature>();
                 if (feature.IsUpgradableRequest)
@@ -59,7 +59,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                         await Task.Delay(100);
                     }
                 }
-            }, serviceContext))
+            }, max: 1))
             using (var disposables = new DisposableStack<TestConnection>())
             {
                 var upgraded = server.CreateConnection();
@@ -81,7 +81,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                     {
                         // this may throw IOException, depending on how fast Kestrel closes the socket
                         await rejected.SendEmptyGetAsKeepAlive();
-                    } catch { }
+                    }
+                    catch { }
 
                     // connection should close without sending any data
                     await rejected.WaitForConnectionClose().TimeoutAfter(TimeSpan.FromSeconds(15));
@@ -93,14 +94,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
         public async Task RejectsConnectionsWhenLimitReached()
         {
             const int max = 10;
-            var (serviceContext, _) = SetupMaxConnections(max);
             var requestTcs = new TaskCompletionSource<object>();
 
-            using (var server = new TestServer(async context =>
+            using (var server = CreateServerWithMaxConnections(async context =>
             {
                 await context.Response.WriteAsync("Hello");
                 await requestTcs.Task;
-            }, serviceContext))
+            }, max))
             using (var disposables = new DisposableStack<TestConnection>())
             {
                 for (var i = 0; i < max; i++)
@@ -141,7 +141,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
             var openedTcs = new TaskCompletionSource<object>();
             var closedTcs = new TaskCompletionSource<object>();
 
-            var (serviceContext, counter) = SetupMaxConnections(uint.MaxValue);
+            var counter = new EventRaisingResourceCounter(ResourceCounter.Quota(uint.MaxValue));
 
             counter.OnLock += (o, e) =>
             {
@@ -159,7 +159,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                 }
             };
 
-            using (var server = new TestServer(_ => Task.CompletedTask, serviceContext))
+            using (var server = CreateServerWithMaxConnections(_ => Task.CompletedTask, counter))
             {
                 // open a bunch of connections in parallel
                 Parallel.For(0, count, async i =>
@@ -187,12 +187,24 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
             }
         }
 
-        private (TestServiceContext serviceContext, EventRaisingResourceCounter counter) SetupMaxConnections(long max)
+        private TestServer CreateServerWithMaxConnections(RequestDelegate app, long max)
         {
-            var counter = new EventRaisingResourceCounter(ResourceCounter.Quota(max));
             var serviceContext = new TestServiceContext();
-            serviceContext.ConnectionManager = new FrameConnectionManager(serviceContext.Log, counter, ResourceCounter.Unlimited);
-            return (serviceContext, counter);
+            serviceContext.ServerOptions.Limits.MaxConcurrentConnections = max;
+            return new TestServer(app, serviceContext);
+        }
+
+        private TestServer CreateServerWithMaxConnections(RequestDelegate app, ResourceCounter concurrentConnectionCounter)
+        {
+            var serviceContext = new TestServiceContext();
+            var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0));
+            listenOptions.Use(next =>
+            {
+                var middleware = new ConnectionLimitMiddleware(next, concurrentConnectionCounter, serviceContext.Log);
+                return middleware.OnConnectionAsync;
+            });
+
+            return new TestServer(app, serviceContext, listenOptions);
         }
     }
 }
diff --git a/test/Kestrel.FunctionalTests/TestHelpers/TestServer.cs b/test/Kestrel.FunctionalTests/TestHelpers/TestServer.cs
index 6e85ab30fa..5e36ddf4af 100644
--- a/test/Kestrel.FunctionalTests/TestHelpers/TestServer.cs
+++ b/test/Kestrel.FunctionalTests/TestHelpers/TestServer.cs
@@ -14,7 +14,6 @@ using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal;
 using Microsoft.AspNetCore.Testing;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
diff --git a/test/Kestrel.FunctionalTests/UpgradeTests.cs b/test/Kestrel.FunctionalTests/UpgradeTests.cs
index f16c7c6466..d2578dd393 100644
--- a/test/Kestrel.FunctionalTests/UpgradeTests.cs
+++ b/test/Kestrel.FunctionalTests/UpgradeTests.cs
@@ -260,7 +260,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
             const int limit = 10;
             var upgradeTcs = new TaskCompletionSource<object>();
             var serviceContext = new TestServiceContext();
-            serviceContext.ConnectionManager = new FrameConnectionManager(serviceContext.Log, ResourceCounter.Unlimited, ResourceCounter.Quota(limit));
+            serviceContext.ConnectionManager = new FrameConnectionManager(serviceContext.Log, ResourceCounter.Quota(limit));
 
             using (var server = new TestServer(async context =>
             {
diff --git a/test/Kestrel.Transport.Libuv.Tests/LibuvOutputConsumerTests.cs b/test/Kestrel.Transport.Libuv.Tests/LibuvOutputConsumerTests.cs
index cd604be6f1..09a88c883d 100644
--- a/test/Kestrel.Transport.Libuv.Tests/LibuvOutputConsumerTests.cs
+++ b/test/Kestrel.Transport.Libuv.Tests/LibuvOutputConsumerTests.cs
@@ -6,6 +6,7 @@ using System.Collections.Concurrent;
 using System.IO.Pipelines;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
@@ -699,6 +700,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.Tests
             var frame = new Frame<object>(null, new FrameContext
             {
                 ServiceContext = serviceContext,
+                ConnectionFeatures = new FeatureCollection(),
                 PipeFactory = _pipeFactory,
                 TimeoutControl = Mock.Of<ITimeoutControl>(),
                 Application = pair.Application,
diff --git a/test/shared/TestServiceContext.cs b/test/shared/TestServiceContext.cs
index 61487222d7..5661d3678f 100644
--- a/test/shared/TestServiceContext.cs
+++ b/test/shared/TestServiceContext.cs
@@ -38,7 +38,7 @@ namespace Microsoft.AspNetCore.Testing
             ThreadPool = new LoggingThreadPool(Log);
             SystemClock = new MockSystemClock();
             DateHeaderValueManager = new DateHeaderValueManager(SystemClock);
-            ConnectionManager = new FrameConnectionManager(Log, ResourceCounter.Unlimited, ResourceCounter.Unlimited);
+            ConnectionManager = new FrameConnectionManager(Log, ResourceCounter.Unlimited);
             HttpParserFactory = frameAdapter => new HttpParser<FrameAdapter>(frameAdapter.Frame.ServiceContext.Log.IsEnabled(LogLevel.Information));
             ServerOptions = new KestrelServerOptions
             {