diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs
index 447a437ca3..70e5bce881 100644
--- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs
+++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs
@@ -382,6 +382,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
         {
             lock (_onStartingSync)
             {
+                if (HasResponseStarted)
+                {
+                    ThrowResponseAlreadyStartedException(nameof(OnStarting));
+                }
+
                 if (_onStarting == null)
                 {
                     _onStarting = new List<KeyValuePair<Func<object, Task>, object>>();
diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/FrameTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/FrameTests.cs
index 876b0279f5..7b1af2b94f 100644
--- a/test/Microsoft.AspNetCore.Server.KestrelTests/FrameTests.cs
+++ b/test/Microsoft.AspNetCore.Server.KestrelTests/FrameTests.cs
@@ -440,6 +440,26 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
             Assert.Throws<InvalidOperationException>(() => ((IHttpResponseFeature)frame).ReasonPhrase = "Reason phrase");
         }
 
+        [Fact]
+        public void ThrowsWhenOnStartingIsSetAfterResponseStarted()
+        {
+            // Arrange
+            var connectionContext = new ConnectionContext()
+            {
+                DateHeaderValueManager = new DateHeaderValueManager(),
+                ServerAddress = ServerAddress.FromUrl("http://localhost:5000"),
+                ServerOptions = new KestrelServerOptions(),
+                SocketOutput = new MockSocketOuptut()
+            };
+            var frame = new Frame<object>(application: null, context: connectionContext);
+            frame.InitializeHeaders();
+            frame.Write(new ArraySegment<byte>(new byte[1]));
+
+            // Act/Assert
+            Assert.True(frame.HasResponseStarted);
+            Assert.Throws<InvalidOperationException>(() => ((IHttpResponseFeature)frame).OnStarting(_ => TaskUtilities.CompletedTask, null));
+        }
+
         [Fact]
         public void InitializeHeadersResetsRequestHeaders()
         {