Ensure CircuitRegistry evicts CircuitHost entries after configured (#11157)

* Ensure CircuitRegistry evicts CircuitHost entries after configured
duration

* Use an active expiration token to trigger expiration
* Add logging during host state transitions

Fixes https://github.com/aspnet/AspNetCore/issues/9893

src/Components/Server/src/Circuits/CircuitHost.cs

@@ -194,6 +194,8 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         private async Task OnCircuitOpenedAsync(CancellationToken cancellationToken)
         {
+            Log.CircuitOpened(_logger, Circuit.Id);
+
             for (var i = 0; i < _circuitHandlers.Length; i++)
             {
                 var circuitHandler = _circuitHandlers[i];
@@ -210,6 +212,8 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         public async Task OnConnectionUpAsync(CancellationToken cancellationToken)
         {
+            Log.ConnectionUp(_logger, Circuit.Id, Client.ConnectionId);
+
             try
             {
                 await HandlerLock.WaitAsync(cancellationToken);
@@ -235,6 +239,8 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         public async Task OnConnectionDownAsync(CancellationToken cancellationToken)
         {
+            Log.ConnectionDown(_logger, Circuit.Id, Client.ConnectionId);
+
             try
             {
                 await HandlerLock.WaitAsync(cancellationToken);
@@ -265,6 +271,8 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         private async Task OnCircuitDownAsync()
         {
+            Log.CircuitClosed(_logger, Circuit.Id);
+
             for (var i = 0; i < _circuitHandlers.Length; i++)
             {
                 var circuitHandler = _circuitHandlers[i];
@@ -283,13 +291,13 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
         {
             Log.DisposingCircuit(_logger, CircuitId);
 
-            await Renderer.InvokeAsync((Func<Task>)(async () =>
+            await Renderer.InvokeAsync(async () =>
             {
                 await OnConnectionDownAsync(CancellationToken.None);
                 await OnCircuitDownAsync();
                 Renderer.Dispose();
                 _scope.Dispose();
-            }));
+            });
         }
 
         private void AssertInitialized()
@@ -314,11 +322,19 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
         {
             private static readonly Action<ILogger, Type, string, string, Exception> _unhandledExceptionInvokingCircuitHandler;
             private static readonly Action<ILogger, string, Exception> _disposingCircuit;
+            private static readonly Action<ILogger, string, Exception> _onCircuitOpened;
+            private static readonly Action<ILogger, string, string, Exception> _onConnectionUp;
+            private static readonly Action<ILogger, string, string, Exception> _onConnectionDown;
+            private static readonly Action<ILogger, string, Exception> _onCircuitClosed;
 
             private static class EventIds
             {
                 public static readonly EventId ExceptionInvokingCircuitHandlerMethod = new EventId(100, "ExceptionInvokingCircuitHandlerMethod");
                 public static readonly EventId DisposingCircuit = new EventId(101, "DisposingCircuitHost");
+                public static readonly EventId OnCircuitOpened = new EventId(102, "OnCircuitOpened");
+                public static readonly EventId OnConnectionUp = new EventId(103, "OnConnectionUp");
+                public static readonly EventId OnConnectionDown = new EventId(104, "OnConnectionDown");
+                public static readonly EventId OnCircuitClosed = new EventId(105, "OnCircuitClosed");
             }
 
             static Log()
@@ -332,6 +348,26 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
                     LogLevel.Trace,
                     EventIds.DisposingCircuit,
                     "Disposing circuit with identifier {CircuitId}");
+
+                _onCircuitOpened = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.OnCircuitOpened,
+                    "Opening circuit with id {CircuitId}.");
+
+                _onConnectionUp = LoggerMessage.Define<string, string>(
+                    LogLevel.Debug,
+                    EventIds.OnConnectionUp,
+                    "Circuit id {CircuitId} connected using connection {ConnectionId}.");
+
+                _onConnectionDown = LoggerMessage.Define<string, string>(
+                    LogLevel.Debug,
+                    EventIds.OnConnectionDown,
+                    "Circuit id {CircuitId} disconnected from connection {ConnectionId}.");
+
+                _onCircuitClosed = LoggerMessage.Define<string>(
+                   LogLevel.Debug,
+                   EventIds.OnCircuitClosed,
+                   "Closing circuit with id {CircuitId}.");
             }
 
             public static void UnhandledExceptionInvokingCircuitHandler(ILogger logger, CircuitHandler handler, string handlerMethod, Exception exception)
@@ -345,6 +381,17 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
             }
 
             public static void DisposingCircuit(ILogger logger, string circuitId) => _disposingCircuit(logger, circuitId, null);
+
+            public static void CircuitOpened(ILogger logger, string circuitId) => _onCircuitOpened(logger, circuitId, null);
+
+            public static void ConnectionUp(ILogger logger, string circuitId, string connectionId) =>
+                _onConnectionUp(logger, circuitId, connectionId, null);
+
+            public static void ConnectionDown(ILogger logger, string circuitId, string connectionId) =>
+                _onConnectionDown(logger, circuitId, connectionId, null);
+
+            public static void CircuitClosed(ILogger logger, string circuitId) =>
+                _onCircuitClosed(logger, circuitId, null);
         }
     }
 }

src/Components/Server/src/Circuits/CircuitRegistry.cs

@@ -10,6 +10,7 @@ using Microsoft.AspNetCore.SignalR;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
 
 namespace Microsoft.AspNetCore.Components.Server.Circuits
 {
@@ -82,6 +83,8 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         public virtual Task DisconnectAsync(CircuitHost circuitHost, string connectionId)
         {
+            Log.CircuitDisconnectStarted(_logger, circuitHost.CircuitId, connectionId);
+
             Task circuitHandlerTask;
             lock (CircuitRegistryLock)
             {
@@ -104,44 +107,60 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         protected virtual bool DisconnectCore(CircuitHost circuitHost, string connectionId)
         {
-            if (!ConnectedCircuits.TryGetValue(circuitHost.CircuitId, out circuitHost))
+            var circuitId = circuitHost.CircuitId;
+            if (!ConnectedCircuits.TryGetValue(circuitId, out circuitHost))
             {
+                Log.CircuitNotActive(_logger, circuitId);
+
                 // Guard: The circuit might already have been marked as inactive.
                 return false;
             }
 
             if (!string.Equals(circuitHost.Client.ConnectionId, connectionId, StringComparison.Ordinal))
             {
+                Log.CircuitConnectedToDifferentConnection(_logger, circuitId, circuitHost.Client.ConnectionId);
+
                 // The circuit is associated with a different connection. One way this could happen is when
                 // the client reconnects with a new connection before the OnDisconnect for the older
                 // connection is executed. Do nothing
                 return false;
             }
 
-            var result = ConnectedCircuits.TryRemove(circuitHost.CircuitId, out circuitHost);
+            var result = ConnectedCircuits.TryRemove(circuitId, out circuitHost);
             Debug.Assert(result, "This operation operates inside of a lock. We expect the previously inspected value to be still here.");
 
             circuitHost.Client.SetDisconnected();
             RegisterDisconnectedCircuit(circuitHost);
+
+            Log.CircuitMarkedDisconnected(_logger, circuitId);
+
             return true;
         }
 
         public void RegisterDisconnectedCircuit(CircuitHost circuitHost)
         {
+            var cancellationTokenSource = new CancellationTokenSource(_options.DisconnectedCircuitRetentionPeriod);
             var entryOptions = new MemoryCacheEntryOptions
             {
-                AbsoluteExpiration = DateTimeOffset.UtcNow.Add(_options.DisconnectedCircuitRetentionPeriod),
                 Size = 1,
                 PostEvictionCallbacks = { _postEvictionCallback },
+                ExpirationTokens =
+                {
+                    new CancellationChangeToken(cancellationTokenSource.Token),
+                },
             };
 
-            DisconnectedCircuits.Set(circuitHost.CircuitId, circuitHost, entryOptions);
+            var entry = new DisconnectedCircuitEntry(circuitHost, cancellationTokenSource);
+            DisconnectedCircuits.Set(circuitHost.CircuitId, entry, entryOptions);
         }
 
         public virtual async Task<CircuitHost> ConnectAsync(string circuitId, IClientProxy clientProxy, string connectionId, CancellationToken cancellationToken)
         {
+            Log.CircuitConnectStarted(_logger, circuitId);
+
             if (!_circuitIdFactory.ValidateCircuitId(circuitId))
             {
+                Log.InvalidCircuitId(_logger, circuitId);
                 return null;
             }
 
@@ -159,6 +178,7 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
                 if (circuitHost == null)
                 {
+                    Log.FailedToReconnectToCircuit(_logger, circuitId);
                     // Failed to connect. Nothing to do here.
                     return null;
                 }
@@ -181,6 +201,7 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
                     await circuitHost.OnConnectionUpAsync(cancellationToken);
                 });
 
+                Log.ReconnectionSucceeded(_logger, circuitId);
             }
 
             await circuitHandlerTask;
@@ -190,36 +211,45 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
         protected virtual (CircuitHost circuitHost, bool previouslyConnected) ConnectCore(string circuitId, IClientProxy clientProxy, string connectionId)
         {
-            if (ConnectedCircuits.TryGetValue(circuitId, out var circuitHost))
+            if (ConnectedCircuits.TryGetValue(circuitId, out var connectedCircuitHost))
             {
+                Log.ConnectingToActiveCircuit(_logger, circuitId, connectionId);
+
                 // The host is still active i.e. the server hasn't detected the client disconnect.
                 // However the client reconnected establishing a new connection.
-                circuitHost.Client.Transfer(clientProxy, connectionId);
+                connectedCircuitHost.Client.Transfer(clientProxy, connectionId);
-                return (circuitHost, true);
+                return (connectedCircuitHost, true);
             }
 
-            if (DisconnectedCircuits.TryGetValue(circuitId, out circuitHost))
+            if (DisconnectedCircuits.TryGetValue(circuitId, out DisconnectedCircuitEntry disconnectedEntry))
             {
+                Log.ConnectingToDisconnectedCircuit(_logger, circuitId, connectionId);
+
                 // The host was in disconnected state. Transfer it to ConnectedCircuits so that it's no longer considered disconnected.
+                // First discard the CancellationTokenSource so that the cache entry does not expire.
+                DisposeTokenSource(disconnectedEntry);
+
                 DisconnectedCircuits.Remove(circuitId);
-                ConnectedCircuits.TryAdd(circuitId, circuitHost);
+                ConnectedCircuits.TryAdd(circuitId, disconnectedEntry.CircuitHost);
 
-                circuitHost.Client.Transfer(clientProxy, connectionId);
+                disconnectedEntry.CircuitHost.Client.Transfer(clientProxy, connectionId);
-
+                return (disconnectedEntry.CircuitHost, false);
-                return (circuitHost, false);
             }
 
             return default;
         }
 
-        private void OnEntryEvicted(object key, object value, EvictionReason reason, object state)
+        protected virtual void OnEntryEvicted(object key, object value, EvictionReason reason, object state)
         {
             switch (reason)
             {
                 case EvictionReason.Expired:
+                case EvictionReason.TokenExpired:
                 case EvictionReason.Capacity:
                     // Kick off the dispose in the background.
-                    _ = DisposeCircuitHost((CircuitHost)value);
+                    var disconnectedEntry = (DisconnectedCircuitEntry)value;
+                    Log.CircuitEvicted(_logger, disconnectedEntry.CircuitHost.CircuitId, reason);
+                    _ = DisposeCircuitEntry(disconnectedEntry);
                     break;
 
                 case EvictionReason.Removed:
@@ -232,11 +262,13 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
             }
         }
 
-        private async Task DisposeCircuitHost(CircuitHost circuitHost)
+        private async Task DisposeCircuitEntry(DisconnectedCircuitEntry entry)
         {
+            DisposeTokenSource(entry);
+
             try
             {
-                await circuitHost.DisposeAsync();
+                await entry.CircuitHost.DisposeAsync();
             }
             catch (Exception ex)
             {
@@ -244,30 +276,168 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
             }
         }
 
+        private void DisposeTokenSource(DisconnectedCircuitEntry entry)
+        {
+            try
+            {
+                entry.TokenSource.Dispose();
+            }
+            catch (Exception ex)
+            {
+                Log.ExceptionDisposingTokenSource(_logger, ex);
+            }
+        }
+
+        private readonly struct DisconnectedCircuitEntry
+        {
+            public DisconnectedCircuitEntry(CircuitHost circuitHost, CancellationTokenSource tokenSource)
+            {
+                CircuitHost = circuitHost;
+                TokenSource = tokenSource;
+            }
+
+            public CircuitHost CircuitHost { get; }
+            public CancellationTokenSource TokenSource { get; }
+        }
+
         private static class Log
         {
-            private static readonly Action<ILogger, string, Exception> _unhandledExceptionDisposingCircuitHost;
+            private static readonly Action<ILogger, string, Exception> _exceptionDisposingCircuitHost;
+            private static readonly Action<ILogger, string, Exception> _unhandledExceptionDisposingTokenSource;
+            private static readonly Action<ILogger, string, Exception> _circuitReconnectStarted;
+            private static readonly Action<ILogger, string, Exception> _invalidCircuitId;
+            private static readonly Action<ILogger, string, string, Exception> _connectingToActiveCircuit;
+            private static readonly Action<ILogger, string, string, Exception> _connectingToDisconnectedCircuit;
+            private static readonly Action<ILogger, string, Exception> _failedToReconnectToCircuit;
+            private static readonly Action<ILogger, string, Exception> _reconnectionSucceeded;
+            private static readonly Action<ILogger, string, string, Exception> _circuitDisconnectStarted;
+            private static readonly Action<ILogger, string, Exception> _circuitNotActive;
+            private static readonly Action<ILogger, string, string, Exception> _circuitConnectedToDifferentConnection;
+            private static readonly Action<ILogger, string, Exception> _circuitMarkedDisconnected;
+            private static readonly Action<ILogger, string, EvictionReason, Exception> _circuitEvicted;
 
             private static class EventIds
             {
                 public static readonly EventId ExceptionDisposingCircuit = new EventId(100, "ExceptionDisposingCircuit");
+                public static readonly EventId ExceptionDisposingTokenSource = new EventId(101, "ExceptionDisposingTokenSource");
+                public static readonly EventId AttemptingToReconnect = new EventId(102, "AttemptingToReconnect");
+                public static readonly EventId InvalidCircuitId = new EventId(103, "InvalidCircuitId");
+                public static readonly EventId ConnectingToActiveCircuit = new EventId(104, "ConnectingToActiveCircuit");
+                public static readonly EventId ConnectingToDisconnectedCircuit = new EventId(105, "ConnectingToDisconnectedCircuit");
+                public static readonly EventId FailedToReconnectToCircuit = new EventId(106, "FailedToReconnectToCircuit");
+                public static readonly EventId CircuitDisconnectStarted = new EventId(107, "CircuitDisconnectStarted");
+                public static readonly EventId CircuitNotActive = new EventId(108, "CircuitNotActive");
+                public static readonly EventId CircuitConnectedToDifferentConnection = new EventId(109, "CircuitConnectedToDifferentConnection");
+                public static readonly EventId CircuitMarkedDisconnected = new EventId(110, "CircuitMarkedDisconnected");
+                public static readonly EventId CircuitEvicted = new EventId(111, "CircuitEvicted");
             }
 
             static Log()
             {
-                _unhandledExceptionDisposingCircuitHost = LoggerMessage.Define<string>(
+                _exceptionDisposingCircuitHost = LoggerMessage.Define<string>(
                     LogLevel.Error,
                     EventIds.ExceptionDisposingCircuit,
                     "Unhandled exception disposing circuit host: {Message}");
+
+                _unhandledExceptionDisposingTokenSource = LoggerMessage.Define<string>(
+                    LogLevel.Trace,
+                    EventIds.ExceptionDisposingTokenSource,
+                    "Exception thrown when disposing token source: {Message}");
+
+                _circuitReconnectStarted = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.AttemptingToReconnect,
+                    "Attempting to reconnect to Circuit with id {CircuitId}.");
+
+                _invalidCircuitId = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.InvalidCircuitId,
+                    "Failed to validate circuit id {CircuitId}.");
+
+                _connectingToActiveCircuit = LoggerMessage.Define<string, string>(
+                    LogLevel.Debug,
+                    EventIds.ConnectingToActiveCircuit,
+                    "Transferring active circuit {CircuitId} to connection {ConnectionId}.");
+
+                _connectingToDisconnectedCircuit = LoggerMessage.Define<string, string>(
+                    LogLevel.Debug,
+                    EventIds.ConnectingToDisconnectedCircuit,
+                    "Transfering disconnected circuit {CircuitId} to connection {ConnectionId}.");
+
+                _failedToReconnectToCircuit = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.FailedToReconnectToCircuit,
+                    "Failed to reconnect to a circuit with id {CircuitId}.");
+
+                _reconnectionSucceeded = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.FailedToReconnectToCircuit,
+                    "Reconnect to circuit with id {CircuitId} succeeded.");
+
+                _circuitDisconnectStarted = LoggerMessage.Define<string, string>(
+                    LogLevel.Debug,
+                    EventIds.CircuitDisconnectStarted,
+                    "Attempting to disconnect circuit with id {CircuitId} from connection {ConnectionId}.");
+
+                _circuitNotActive = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.CircuitNotActive,
+                    "Failed to disconnect circuit with id {CircuitId}. The circuit is not active.");
+
+                _circuitConnectedToDifferentConnection = LoggerMessage.Define<string, string>(
+                    LogLevel.Debug,
+                    EventIds.CircuitConnectedToDifferentConnection,
+                    "Failed to disconnect circuit with id {CircuitId}. The circuit is connected to {ConnectionId}.");
+
+                _circuitMarkedDisconnected = LoggerMessage.Define<string>(
+                    LogLevel.Debug,
+                    EventIds.CircuitMarkedDisconnected,
+                    "Circuit with id {CircuitId} is disconnected.");
+
+                _circuitEvicted = LoggerMessage.Define<string, EvictionReason>(
+                    LogLevel.Debug,
+                    EventIds.CircuitEvicted,
+                    "Circuit with id {CircuitId} evicted due to {EvictionReason}.");
             }
 
-            public static void UnhandledExceptionDisposingCircuitHost(ILogger logger, Exception exception)
+            public static void UnhandledExceptionDisposingCircuitHost(ILogger logger, Exception exception) =>
-            {
+                _exceptionDisposingCircuitHost(logger, exception.Message, exception);
-                _unhandledExceptionDisposingCircuitHost(
+
-                    logger,
+            public static void ExceptionDisposingTokenSource(ILogger logger, Exception exception) =>
-                    exception.Message,
+                _unhandledExceptionDisposingTokenSource(logger, exception.Message, exception);
-                    exception);
+
-            }
+            public static void CircuitConnectStarted(ILogger logger, string circuitId) =>
+                _circuitReconnectStarted(logger, circuitId, null);
+
+            public static void InvalidCircuitId(ILogger logger, string circuitId) =>
+                _invalidCircuitId(logger, circuitId, null);
+
+            public static void ConnectingToActiveCircuit(ILogger logger, string circuitId, string connectionId) =>
+                _connectingToActiveCircuit(logger, circuitId, connectionId, null);
+
+            public static void ConnectingToDisconnectedCircuit(ILogger logger, string circuitId, string connectionId) =>
+                _connectingToDisconnectedCircuit(logger, circuitId, connectionId, null);
+
+            public static void FailedToReconnectToCircuit(ILogger logger, string circuitId) =>
+                _failedToReconnectToCircuit(logger, circuitId, null);
+
+            public static void ReconnectionSucceeded(ILogger logger, string circuitId) =>
+                _reconnectionSucceeded(logger, circuitId, null);
+
+            public static void CircuitDisconnectStarted(ILogger logger, string circuitId, string connectionId) =>
+                _circuitDisconnectStarted(logger, circuitId, connectionId, null);
+
+            public static void CircuitNotActive(ILogger logger, string circuitId) =>
+                _circuitNotActive(logger, circuitId, null);
+
+            public static void CircuitConnectedToDifferentConnection(ILogger logger, string circuitId, string connectionId) =>
+                _circuitConnectedToDifferentConnection(logger, circuitId, connectionId, null);
+
+            public static void CircuitMarkedDisconnected(ILogger logger, string circuitId) =>
+                _circuitMarkedDisconnected(logger, circuitId, null);
+
+            public static void CircuitEvicted(ILogger logger, string circuitId, EvictionReason evictionReason) =>
+               _circuitEvicted(logger, circuitId, evictionReason, null);
         }
     }
 }

src/Components/Server/test/Circuits/CircuitRegistryTest.cs

@@ -2,10 +2,12 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.DataProtection;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.AspNetCore.Testing;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
@@ -64,7 +66,7 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
             var registry = CreateRegistry(circuitIdFactory);
             var circuitHost = TestCircuitHost.Create(circuitIdFactory.CreateCircuitId());
-            registry.DisconnectedCircuits.Set(circuitHost.CircuitId, circuitHost, new MemoryCacheEntryOptions { Size = 1 });
+            registry.RegisterDisconnectedCircuit(circuitHost);
 
             var newClient = Mock.Of<IClientProxy>();
             var newConnectionId = "new-id";
@@ -90,7 +92,7 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
             var registry = CreateRegistry(circuitIdFactory);
             var handler = new Mock<CircuitHandler> { CallBase = true };
             var circuitHost = TestCircuitHost.Create(circuitIdFactory.CreateCircuitId(), handlers: new[] { handler.Object });
-            registry.DisconnectedCircuits.Set(circuitHost.CircuitId, circuitHost, new MemoryCacheEntryOptions { Size = 1 });
+            registry.RegisterDisconnectedCircuit(circuitHost);
 
             var newClient = Mock.Of<IClientProxy>();
             var newConnectionId = "new-id";
@@ -313,16 +315,108 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
             Assert.False(registry.DisconnectedCircuits.TryGetValue(circuitHost.CircuitId, out _));
         }
 
+        [Fact]
+        public void CircuitRegistryUsesConfiguredMaxRetainedDisconnectedCircuitsValue()
+        {
+            // Arrange
+            var circuitIdFactory = TestCircuitIdFactory.CreateTestFactory();
+            var maxCircuits = 3;
+            var circuitOptions = new CircuitOptions
+            {
+                MaxRetainedDisconnectedCircuits = maxCircuits,
+            };
+            var registry = new TestCircuitRegistry(circuitIdFactory, circuitOptions);
+            var hosts = Enumerable.Range(0, maxCircuits + 2)
+                .Select(_ => TestCircuitHost.Create())
+                .ToArray();
+
+            // Act
+            for (var i = 0; i < hosts.Length; i++)
+            {
+                registry.RegisterDisconnectedCircuit(hosts[i]);
+            }
+
+            // Assert
+            for (var i = 0; i < maxCircuits; i++)
+            {
+                Assert.True(registry.DisconnectedCircuits.TryGetValue(hosts[i].CircuitId, out var _));
+            }
+
+            // Additional circuits do not get registered.
+            Assert.False(registry.DisconnectedCircuits.TryGetValue(hosts[maxCircuits].CircuitId, out var _));
+            Assert.False(registry.DisconnectedCircuits.TryGetValue(hosts[maxCircuits + 1].CircuitId, out var _));
+        }
+
+        [Fact]
+        public async Task DisconnectedCircuitIsRemovedAfterConfiguredTimeout()
+        {
+            // Arrange
+            var circuitIdFactory = TestCircuitIdFactory.CreateTestFactory();
+            var circuitOptions = new CircuitOptions
+            {
+                DisconnectedCircuitRetentionPeriod = TimeSpan.FromSeconds(3),
+            };
+            var registry = new TestCircuitRegistry(circuitIdFactory, circuitOptions);
+            var tcs = new TaskCompletionSource<object>();
+
+            registry.OnAfterEntryEvicted = () =>
+            {
+                tcs.TrySetResult(new object());
+            };
+            var circuitHost = TestCircuitHost.Create();
+
+            registry.RegisterDisconnectedCircuit(circuitHost);
+
+            // Act
+            // Verify it's present in the dictionary.
+            Assert.True(registry.DisconnectedCircuits.TryGetValue(circuitHost.CircuitId, out var _));
+            await Task.Run(() => tcs.Task.TimeoutAfter(TimeSpan.FromSeconds(10)));
+            Assert.False(registry.DisconnectedCircuits.TryGetValue(circuitHost.CircuitId, out var _));
+        }
+
+        [Fact]
+        public async Task ReconnectBeforeTimeoutDoesNotGetEntryToBeEvicted()
+        {
+            // Arrange
+            var circuitIdFactory = TestCircuitIdFactory.CreateTestFactory();
+            var circuitOptions = new CircuitOptions
+            {
+                DisconnectedCircuitRetentionPeriod = TimeSpan.FromSeconds(8),
+            };
+            var registry = new TestCircuitRegistry(circuitIdFactory, circuitOptions);
+            var tcs = new TaskCompletionSource<object>();
+
+            registry.OnAfterEntryEvicted = () =>
+            {
+                tcs.TrySetResult(new object());
+            };
+            var circuitHost = TestCircuitHost.Create(circuitIdFactory.CreateCircuitId());
+
+            registry.RegisterDisconnectedCircuit(circuitHost);
+            await registry.ConnectAsync(circuitHost.CircuitId, Mock.Of<IClientProxy>(), "new-connection", default);
+
+            // Act
+            await Task.Run(() => tcs.Task.TimeoutAfter(TimeSpan.FromSeconds(10)));
+
+            // Verify it's still connected
+            Assert.True(registry.ConnectedCircuits.TryGetValue(circuitHost.CircuitId, out var cacheValue));
+            Assert.Same(circuitHost, cacheValue);
+            // Nothing should be disconnected.
+            Assert.False(registry.DisconnectedCircuits.TryGetValue(circuitHost.CircuitId, out var _));
+        }
+
         private class TestCircuitRegistry : CircuitRegistry
         {
-            public TestCircuitRegistry(CircuitIdFactory factory)
+            public TestCircuitRegistry(CircuitIdFactory factory, CircuitOptions circuitOptions = null)
-                : base(Options.Create(new CircuitOptions()), NullLogger<CircuitRegistry>.Instance, factory)
+                : base(Options.Create(circuitOptions ?? new CircuitOptions()), NullLogger<CircuitRegistry>.Instance, factory)
             {
             }
 
             public ManualResetEventSlim BeforeConnect { get; set; }
             public ManualResetEventSlim BeforeDisconnect { get; set; }
 
+            public Action OnAfterEntryEvicted { get; set; }
+
             protected override (CircuitHost, bool) ConnectCore(string circuitId, IClientProxy clientProxy, string connectionId)
             {
                 if (BeforeConnect != null)
@@ -342,6 +436,12 @@ namespace Microsoft.AspNetCore.Components.Server.Circuits
 
                 return base.DisconnectCore(circuitHost, connectionId);
             }
+
+            protected override void OnEntryEvicted(object key, object value, EvictionReason reason, object state)
+            {
+                base.OnEntryEvicted(key, value, reason, state);
+                OnAfterEntryEvicted();
+            }
         }
 
         private static CircuitRegistry CreateRegistry(CircuitIdFactory factory = null)