From 01b35bc3913204ebbd308b4719a20168466d0183 Mon Sep 17 00:00:00 2001 From: "Chris Ross (ASP.NET)" Date: Thu, 20 Sep 2018 15:27:33 -0700 Subject: [PATCH] Make HTTP/2 connection and stream windows configurable #2814 --- src/Kestrel.Core/Http2Limits.cs | 46 +++ .../Internal/Http2/Http2Connection.cs | 14 +- .../Internal/Http2/Http2Stream.cs | 14 +- .../Internal/Http2/Http2StreamContext.cs | 1 + .../Http2/Http2ConnectionTests.cs | 350 ++++++++++++------ .../Http2/Http2TestBase.cs | 7 +- .../HttpProtocolSelectionTests.cs | 3 +- 7 files changed, 317 insertions(+), 118 deletions(-) diff --git a/src/Kestrel.Core/Http2Limits.cs b/src/Kestrel.Core/Http2Limits.cs index e2914ac150..c1c4b0746c 100644 --- a/src/Kestrel.Core/Http2Limits.cs +++ b/src/Kestrel.Core/Http2Limits.cs @@ -15,6 +15,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core private int _headerTableSize = (int)Http2PeerSettings.DefaultHeaderTableSize; private int _maxFrameSize = (int)Http2PeerSettings.DefaultMaxFrameSize; private int _maxRequestHeaderFieldSize = 8192; + private int _initialConnectionWindowSize = 1024 * 128; // Larger than the default 64kb, and larger than any one single stream. + private int _initialStreamWindowSize = 1024 * 96; // Larger than the default 64kb /// /// Limits the number of concurrent request streams per HTTP/2 connection. Excess streams will be refused. @@ -95,5 +97,49 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core _maxRequestHeaderFieldSize = value; } } + + /// + /// Indicates how much request body data the server is willing to receive and buffer at a time aggregated across all + /// requests (streams) per connection. Note requests are also limited by + /// + /// Value must be greater than or equal to 65,535 and less than 2^31, defaults to 128 kb. + /// + /// + public int InitialConnectionWindowSize + { + get => _initialConnectionWindowSize; + set + { + if (value < Http2PeerSettings.DefaultInitialWindowSize || value > Http2PeerSettings.MaxWindowSize) + { + throw new ArgumentOutOfRangeException(nameof(value), value, + CoreStrings.FormatArgumentOutOfRange(Http2PeerSettings.DefaultInitialWindowSize, Http2PeerSettings.MaxWindowSize)); + } + + _initialConnectionWindowSize = value; + } + } + + /// + /// Indicates how much request body data the server is willing to receive and buffer at a time per stream. + /// Note connections are also limited by + /// + /// Value must be greater than or equal to 65,535 and less than 2^31, defaults to 96 kb. + /// + /// + public int InitialStreamWindowSize + { + get => _initialStreamWindowSize; + set + { + if (value < Http2PeerSettings.DefaultInitialWindowSize || value > Http2PeerSettings.MaxWindowSize) + { + throw new ArgumentOutOfRangeException(nameof(value), value, + CoreStrings.FormatArgumentOutOfRange(Http2PeerSettings.DefaultInitialWindowSize, Http2PeerSettings.MaxWindowSize)); + } + + _initialStreamWindowSize = value; + } + } } } diff --git a/src/Kestrel.Core/Internal/Http2/Http2Connection.cs b/src/Kestrel.Core/Internal/Http2/Http2Connection.cs index 9bbf1a9236..756b564351 100644 --- a/src/Kestrel.Core/Internal/Http2/Http2Connection.cs +++ b/src/Kestrel.Core/Internal/Http2/Http2Connection.cs @@ -63,7 +63,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 private readonly Http2ConnectionContext _context; private readonly Http2FrameWriter _frameWriter; private readonly HPackDecoder _hpackDecoder; - private readonly InputFlowControl _inputFlowControl = new InputFlowControl(Http2PeerSettings.DefaultInitialWindowSize, Http2PeerSettings.DefaultInitialWindowSize / 2); + private readonly InputFlowControl _inputFlowControl; private readonly OutputFlowControl _outputFlowControl = new OutputFlowControl(Http2PeerSettings.DefaultInitialWindowSize); private readonly Http2PeerSettings _serverSettings = new Http2PeerSettings(); @@ -96,6 +96,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 _serverSettings.HeaderTableSize = (uint)http2Limits.HeaderTableSize; _hpackDecoder = new HPackDecoder(http2Limits.HeaderTableSize, http2Limits.MaxRequestHeaderFieldSize); _serverSettings.MaxHeaderListSize = (uint)httpLimits.MaxRequestHeadersTotalSize; + _serverSettings.InitialWindowSize = (uint)http2Limits.InitialStreamWindowSize; + var connectionWindow = (uint)http2Limits.InitialConnectionWindowSize; + _inputFlowControl = new InputFlowControl(connectionWindow, connectionWindow / 2); } public string ConnectionId => _context.ConnectionId; @@ -183,6 +186,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 if (_state != Http2ConnectionState.Closed) { await _frameWriter.WriteSettingsAsync(_serverSettings.GetNonProtocolDefaults()); + // Inform the client that the connection window is larger than the default. It can't be lowered here, + // It can only be lowered by not issuing window updates after data is received. + var connectionWindow = _context.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var diff = connectionWindow - (int)Http2PeerSettings.DefaultInitialWindowSize; + if (diff > 0) + { + await _frameWriter.WriteWindowUpdateAsync(0, diff); + } } while (_state != Http2ConnectionState.Closed) @@ -541,6 +552,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 RemoteEndPoint = _context.RemoteEndPoint, StreamLifetimeHandler = this, ClientPeerSettings = _clientSettings, + ServerPeerSettings = _serverSettings, FrameWriter = _frameWriter, ConnectionInputFlowControl = _inputFlowControl, ConnectionOutputFlowControl = _outputFlowControl, diff --git a/src/Kestrel.Core/Internal/Http2/Http2Stream.cs b/src/Kestrel.Core/Internal/Http2/Http2Stream.cs index 8bcdd5fcdc..dca23f399f 100644 --- a/src/Kestrel.Core/Internal/Http2/Http2Stream.cs +++ b/src/Kestrel.Core/Internal/Http2/Http2Stream.cs @@ -36,13 +36,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 _context.StreamId, _context.FrameWriter, context.ConnectionInputFlowControl, - Http2PeerSettings.DefaultInitialWindowSize, - Http2PeerSettings.DefaultInitialWindowSize / 2); + _context.ServerPeerSettings.InitialWindowSize, + _context.ServerPeerSettings.InitialWindowSize / 2); _outputFlowControl = new StreamOutputFlowControl(context.ConnectionOutputFlowControl, context.ClientPeerSettings.InitialWindowSize); _http2Output = new Http2OutputProducer(context.StreamId, context.FrameWriter, _outputFlowControl, context.TimeoutControl, context.MemoryPool); - RequestBodyPipe = CreateRequestBodyPipe(); + RequestBodyPipe = CreateRequestBodyPipe(_context.ServerPeerSettings.InitialWindowSize); Output = _http2Output; } @@ -446,14 +446,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 _inputFlowControl.Abort(); } - private Pipe CreateRequestBodyPipe() + private Pipe CreateRequestBodyPipe(uint windowSize) => new Pipe(new PipeOptions ( pool: _context.MemoryPool, readerScheduler: ServiceContext.Scheduler, writerScheduler: PipeScheduler.Inline, - pauseWriterThreshold: Http2PeerSettings.DefaultInitialWindowSize, - resumeWriterThreshold: Http2PeerSettings.DefaultInitialWindowSize, + // Never pause within the window range. Flow control will prevent more data from being added. + // See the assert in OnDataAsync. + pauseWriterThreshold: windowSize + 1, + resumeWriterThreshold: windowSize + 1, useSynchronizationContext: false, minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize )); diff --git a/src/Kestrel.Core/Internal/Http2/Http2StreamContext.cs b/src/Kestrel.Core/Internal/Http2/Http2StreamContext.cs index 698b29f749..71db771a2b 100644 --- a/src/Kestrel.Core/Internal/Http2/Http2StreamContext.cs +++ b/src/Kestrel.Core/Internal/Http2/Http2StreamContext.cs @@ -21,6 +21,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 public IPEndPoint LocalEndPoint { get; set; } public IHttp2StreamLifetimeHandler StreamLifetimeHandler { get; set; } public Http2PeerSettings ClientPeerSettings { get; set; } + public Http2PeerSettings ServerPeerSettings { get; set; } public Http2FrameWriter FrameWriter { get; set; } public InputFlowControl ConnectionInputFlowControl { get; set; } public OutputFlowControl ConnectionOutputFlowControl { get; set; } diff --git a/test/Kestrel.InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/test/Kestrel.InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index a098609f91..ef2be68ae4 100644 --- a/test/Kestrel.InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/test/Kestrel.InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -102,7 +102,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests _connectionContext.ServiceContext.ServerOptions.Limits.Http2.MaxFrameSize = length; _connection = new Http2Connection(_connectionContext); - await InitializeConnectionAsync(_echoApplication, expectedSettingsCount: 3); + await InitializeConnectionAsync(_echoApplication, expectedSettingsCount: 4); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); await SendDataAsync(1, new byte[length], endStream: true); @@ -182,31 +182,44 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests } [Fact] - public async Task DATA_Received_GreaterThanDefaultInitialWindowSize_ReadByStream() + public async Task DATA_Received_GreaterThanInitialWindowSize_ReadByStream() { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); + var initialStreamWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialStreamWindowSize; + var framesInStreamWindow = initialStreamWindowSize / Http2PeerSettings.DefaultMaxFrameSize; + var initialConnectionWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var framesInConnectionWindow = initialConnectionWindowSize / Http2PeerSettings.DefaultMaxFrameSize; - // Double the client stream windows to 128KiB so no stream WINDOW_UPDATEs need to be sent. - _clientSettings.InitialWindowSize = Http2PeerSettings.DefaultInitialWindowSize * 2; + // Grow the client stream windows so no stream WINDOW_UPDATEs need to be sent. + _clientSettings.InitialWindowSize = int.MaxValue; await InitializeConnectionAsync(_echoApplication); - // Double the client connection window to 128KiB. - await SendWindowUpdateAsync(0, (int)Http2PeerSettings.DefaultInitialWindowSize); + // Grow the client connection windows so no connection WINDOW_UPDATEs need to be sent. + await SendWindowUpdateAsync(0, int.MaxValue - (int)Http2PeerSettings.DefaultInitialWindowSize); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); + + // Rounds down so we don't go over the half window size and trigger an update + for (var i = 0; i < framesInStreamWindow / 2; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } await ExpectAsync(Http2FrameType.HEADERS, withLength: 37, withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, withStreamId: 1); - var dataFrame1 = await ExpectAsync(Http2FrameType.DATA, - withLength: _maxData.Length, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 1); + var dataFrames = new List(); + + for (var i = 0; i < framesInStreamWindow / 2; i++) + { + var dataFrame1 = await ExpectAsync(Http2FrameType.DATA, + withLength: _maxData.Length, + withFlags: (byte)Http2DataFrameFlags.NONE, + withStreamId: 1); + dataFrames.Add(dataFrame1); + } // Writing over half the initial window size induces both a connection-level and stream-level window update. await SendDataAsync(1, _maxData, endStream: false); @@ -215,34 +228,43 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests withLength: 4, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); - var connectionWindowUpdateFrame1 = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, - withLength: 4, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 0); var dataFrame2 = await ExpectAsync(Http2FrameType.DATA, withLength: _maxData.Length, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); + dataFrames.Add(dataFrame2); + // Write a few more frames to get close to the connection window threshold + var additionalFrames = (framesInConnectionWindow / 2) - (framesInStreamWindow / 2) - 1; + for (var i = 0; i < additionalFrames; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + + var dataFrame1 = await ExpectAsync(Http2FrameType.DATA, + withLength: _maxData.Length, + withFlags: (byte)Http2DataFrameFlags.NONE, + withStreamId: 1); + dataFrames.Add(dataFrame1); + } + + // Write one more to cross the connection window update threshold await SendDataAsync(1, _maxData, endStream: false); + var connectionWindowUpdateFrame1 = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, + withLength: 4, + withFlags: (byte)Http2DataFrameFlags.NONE, + withStreamId: 0); + var dataFrame3 = await ExpectAsync(Http2FrameType.DATA, withLength: _maxData.Length, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); + dataFrames.Add(dataFrame3); - await SendDataAsync(1, _maxData, endStream: true); + // End + await SendDataAsync(1, new Memory(), endStream: true); - var connectionWindowUpdateFrame2 = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, - withLength: 4, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 0); - - var dataFrame4 = await ExpectAsync(Http2FrameType.DATA, - withLength: _maxData.Length, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 1); await ExpectAsync(Http2FrameType.DATA, withLength: 0, withFlags: (byte)Http2DataFrameFlags.END_STREAM, @@ -250,13 +272,44 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame1.PayloadSequence.ToArray())); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame2.PayloadSequence.ToArray())); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame3.PayloadSequence.ToArray())); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame4.PayloadSequence.ToArray())); - Assert.Equal(_maxData.Length * 2, streamWindowUpdateFrame1.WindowUpdateSizeIncrement); - Assert.Equal(_maxData.Length * 2, connectionWindowUpdateFrame1.WindowUpdateSizeIncrement); - Assert.Equal(_maxData.Length * 2, connectionWindowUpdateFrame2.WindowUpdateSizeIncrement); + foreach (var frame in dataFrames) + { + Assert.True(_maxData.AsSpan().SequenceEqual(frame.PayloadSequence.ToArray())); + } + var updateSize = ((framesInStreamWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, streamWindowUpdateFrame1.WindowUpdateSizeIncrement); + updateSize = ((framesInConnectionWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, connectionWindowUpdateFrame1.WindowUpdateSizeIncrement); + } + + [Fact] + public async Task DATA_Received_RightAtWindowLimit_DoesNotPausePipe() + { + var initialStreamWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialStreamWindowSize; + var framesInStreamWindow = initialStreamWindowSize / Http2PeerSettings.DefaultMaxFrameSize; + var initialConnectionWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var framesInConnectionWindow = initialConnectionWindowSize / Http2PeerSettings.DefaultMaxFrameSize; + + await InitializeConnectionAsync(_waitForAbortApplication); + + await StartStreamAsync(1, _browserRequestHeaders, endStream: false); + + // Rounds down so we don't go over the limit + for (var i = 0; i < framesInStreamWindow; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } + + var remainder = initialStreamWindowSize % (int)Http2PeerSettings.DefaultMaxFrameSize; + + // Write just to the limit. + // This should not produce a async task from the request body pipe. See the Debug.Assert in Http2Stream.OnDataAsync + await SendDataAsync(1, new Memory(_maxData, 0, remainder), endStream: false); + + // End + await SendDataAsync(1, new Memory(), endStream: true); + + await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); } [Fact] @@ -358,61 +411,77 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests } [Fact] - public async Task DATA_Received_Multiplexed_GreaterThanDefaultInitialWindowSize_ReadByStream() + public async Task DATA_Received_Multiplexed_GreaterThanInitialWindowSize_ReadByStream() { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); + var initialStreamWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialStreamWindowSize; + var initialConnectionWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var framesInStreamWindow = initialStreamWindowSize / Http2PeerSettings.DefaultMaxFrameSize; + var framesInConnectionWindow = initialConnectionWindowSize / Http2PeerSettings.DefaultMaxFrameSize; - // Double the client stream windows to 128KiB so no stream WINDOW_UPDATEs need to be sent. - _clientSettings.InitialWindowSize = Http2PeerSettings.DefaultInitialWindowSize * 2; + // Grow the client stream windows so no stream WINDOW_UPDATEs need to be sent. + _clientSettings.InitialWindowSize = int.MaxValue; await InitializeConnectionAsync(_echoApplication); - // Double the client connection window to 128KiB. - await SendWindowUpdateAsync(0, (int)Http2PeerSettings.DefaultInitialWindowSize); + // Grow the client connection windows so no connection WINDOW_UPDATEs need to be sent. + await SendWindowUpdateAsync(0, int.MaxValue - (int)Http2PeerSettings.DefaultInitialWindowSize); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); + + // Rounds down so we don't go over the half window size and trigger an update + for (var i = 0; i < framesInStreamWindow / 2; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } await ExpectAsync(Http2FrameType.HEADERS, withLength: 37, withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, withStreamId: 1); - var dataFrame1 = await ExpectAsync(Http2FrameType.DATA, - withLength: _maxData.Length, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 1); + var dataFrames = new List(); - // Writing over half the initial window size induces both a connection-level and stream-level window update. + for (var i = 0; i < framesInStreamWindow / 2; i++) + { + var dataFrame1 = await ExpectAsync(Http2FrameType.DATA, + withLength: _maxData.Length, + withFlags: (byte)Http2DataFrameFlags.NONE, + withStreamId: 1); + dataFrames.Add(dataFrame1); + } + + // Writing over half the initial window size induces a stream-level window update. await SendDataAsync(1, _maxData, endStream: false); var streamWindowUpdateFrame = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, withLength: 4, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); - var connectionWindowUpdateFrame1 = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, - withLength: 4, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 0); var dataFrame2 = await ExpectAsync(Http2FrameType.DATA, withLength: _maxData.Length, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); + dataFrames.Add(dataFrame2); - await SendDataAsync(1, _maxData, endStream: false); + // No update expected for these + var additionalFrames = (framesInConnectionWindow / 2) - (framesInStreamWindow / 2) - 1; + for (var i = 0; i < additionalFrames; i++) + { + await SendDataAsync(1, _maxData, endStream: false); - var dataFrame3 = await ExpectAsync(Http2FrameType.DATA, - withLength: _maxData.Length, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 1); + var dataFrame3 = await ExpectAsync(Http2FrameType.DATA, + withLength: _maxData.Length, + withFlags: (byte)Http2DataFrameFlags.NONE, + withStreamId: 1); + dataFrames.Add(dataFrame3); + } // Uploading data to a new stream induces a second connection-level but not stream-level window update. await StartStreamAsync(3, _browserRequestHeaders, endStream: false); await SendDataAsync(3, _maxData, endStream: true); - var connectionWindowUpdateFrame2 = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, + var connectionWindowUpdateFrame = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, withLength: 4, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 0); @@ -426,17 +495,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests withLength: _maxData.Length, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 3); + dataFrames.Add(dataFrame4); await ExpectAsync(Http2FrameType.DATA, withLength: 0, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); + // Would trigger a stream window update, except it's the last frame. await SendDataAsync(1, _maxData, endStream: true); var dataFrame5 = await ExpectAsync(Http2FrameType.DATA, withLength: _maxData.Length, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); + dataFrames.Add(dataFrame5); await ExpectAsync(Http2FrameType.DATA, withLength: 0, withFlags: (byte)Http2DataFrameFlags.END_STREAM, @@ -444,16 +516,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await StopConnectionAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame1.PayloadSequence.ToArray())); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame2.PayloadSequence.ToArray())); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame3.PayloadSequence.ToArray())); - Assert.Equal(_maxData.Length * 2, streamWindowUpdateFrame.WindowUpdateSizeIncrement); - Assert.Equal(_maxData.Length * 2, connectionWindowUpdateFrame1.WindowUpdateSizeIncrement); - - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame4.PayloadSequence.ToArray())); - Assert.Equal(_maxData.Length * 2, connectionWindowUpdateFrame2.WindowUpdateSizeIncrement); - - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame5.PayloadSequence.ToArray())); + foreach (var frame in dataFrames) + { + Assert.True(_maxData.AsSpan().SequenceEqual(frame.PayloadSequence.ToArray())); + } + var updateSize = ((framesInStreamWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, streamWindowUpdateFrame.WindowUpdateSizeIncrement); + updateSize = ((framesInConnectionWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); } [Fact] @@ -557,38 +627,59 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests [InlineData(255)] public async Task DATA_Received_WithPadding_CountsTowardsInputFlowControl(byte padLength) { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); - + var initialWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialStreamWindowSize; + var framesInWindow = initialWindowSize / Http2PeerSettings.DefaultMaxFrameSize; var maxDataMinusPadding = _maxData.AsMemory(0, _maxData.Length - padLength - 1); + // Grow the client stream windows so no stream WINDOW_UPDATEs need to be sent. + _clientSettings.InitialWindowSize = int.MaxValue; + await InitializeConnectionAsync(_echoApplication); + // Grow the client connection windows so no connection WINDOW_UPDATEs need to be sent. + await SendWindowUpdateAsync(0, int.MaxValue - (int)Http2PeerSettings.DefaultInitialWindowSize); + await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataWithPaddingAsync(1, maxDataMinusPadding, padLength, endStream: false); + var dataSent = 0; + // Rounds down so we don't go over the half window size and trigger an update + for (var i = 0; i < framesInWindow / 2; i++) + { + await SendDataWithPaddingAsync(1, maxDataMinusPadding, padLength, endStream: false); + dataSent += maxDataMinusPadding.Length; + } await ExpectAsync(Http2FrameType.HEADERS, withLength: 37, withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, withStreamId: 1); - var dataFrame1 = await ExpectAsync(Http2FrameType.DATA, - withLength: maxDataMinusPadding.Length, - withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 1); + // The frames come back in various sizes depending on the pipe buffers, and without the padding we sent. + while (dataSent > 0) + { + var frame = await ReceiveFrameAsync(); + Assert.Equal(Http2FrameType.DATA, frame.Type); + Assert.True(dataSent >= frame.PayloadLength); + Assert.Equal(Http2DataFrameFlags.NONE, frame.DataFlags); + Assert.Equal(1, frame.StreamId); - // Writing over half the initial window size induces both a connection-level and stream-level window update. - await SendDataAsync(1, _maxData, endStream: true); + dataSent -= frame.PayloadLength; + } + + // Writing over half the initial window size induces a stream-level window update. + await SendDataAsync(1, _maxData, endStream: false); var connectionWindowUpdateFrame = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, withLength: 4, withFlags: (byte)Http2DataFrameFlags.NONE, - withStreamId: 0); + withStreamId: 1); - var dataFrame2 = await ExpectAsync(Http2FrameType.DATA, + var dataFrame3 = await ExpectAsync(Http2FrameType.DATA, withLength: _maxData.Length, withFlags: (byte)Http2DataFrameFlags.NONE, withStreamId: 1); + + await SendDataAsync(1, new Memory(), endStream: true); + await ExpectAsync(Http2FrameType.DATA, withLength: 0, withFlags: (byte)Http2DataFrameFlags.END_STREAM, @@ -596,22 +687,25 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); - Assert.True(maxDataMinusPadding.Span.SequenceEqual(dataFrame1.PayloadSequence.ToArray())); - Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame2.PayloadSequence.ToArray())); + Assert.True(_maxData.AsSpan().SequenceEqual(dataFrame3.PayloadSequence.ToArray())); - Assert.Equal(_maxData.Length * 2, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); + var updateSize = ((framesInWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); } [Fact] public async Task DATA_Received_ButNotConsumedByApp_CountsTowardsInputFlowControl() { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); + var initialConnectionWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var framesConnectionInWindow = initialConnectionWindowSize / Http2PeerSettings.DefaultMaxFrameSize; await InitializeConnectionAsync(_noopApplication); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); + for (var i = 0; i < framesConnectionInWindow / 2; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } await ExpectAsync(Http2FrameType.HEADERS, withLength: 55, @@ -622,7 +716,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 1); - // Writing over half the initial window size induces both a connection-level window update. + // Writing over half the initial window size induces a connection-level window update. + // But no stream window update since this is the last frame. await SendDataAsync(1, _maxData, endStream: true); var connectionWindowUpdateFrame = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, @@ -632,7 +727,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); - Assert.Equal(_maxData.Length * 2, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); + var updateSize = ((framesConnectionInWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); } [Fact] @@ -822,17 +918,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests [Fact] public async Task DATA_Received_NoStreamWindowSpace_ConnectionError() { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); + var initialWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialStreamWindowSize; + var framesInWindow = (initialWindowSize / Http2PeerSettings.DefaultMaxFrameSize) + 1; // Round up to overflow the window await InitializeConnectionAsync(_waitForAbortApplication); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); + for (var i = 0; i < framesInWindow; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } await WaitForConnectionErrorAsync( ignoreNonGoAwayFrames: false, @@ -844,17 +940,23 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests [Fact] public async Task DATA_Received_NoConnectionWindowSpace_ConnectionError() { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); + var initialWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var framesInWindow = initialWindowSize / Http2PeerSettings.DefaultMaxFrameSize; await InitializeConnectionAsync(_waitForAbortApplication); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); + for (var i = 0; i < framesInWindow / 2; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } await StartStreamAsync(3, _browserRequestHeaders, endStream: false); - await SendDataAsync(3, _maxData, endStream: false); + for (var i = 0; i < framesInWindow / 2; i++) + { + await SendDataAsync(3, _maxData, endStream: false); + } + // One extra to overflow the connection window await SendDataAsync(3, _maxData, endStream: false); await WaitForConnectionErrorAsync( @@ -1956,14 +2058,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests [Fact] public async Task RST_STREAM_Received_ReturnsSpaceToConnectionInputFlowControlWindow() { - // _maxData should be 1/4th of the default initial window size + 1. - Assert.Equal(Http2PeerSettings.DefaultInitialWindowSize + 1, (uint)_maxData.Length * 4); + var initialConnectionWindowSize = _connectionContext.ServiceContext.ServerOptions.Limits.Http2.InitialConnectionWindowSize; + var framesInConnectionWindow = initialConnectionWindowSize / Http2PeerSettings.DefaultMaxFrameSize; await InitializeConnectionAsync(_waitForAbortApplication); await StartStreamAsync(1, _browserRequestHeaders, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); - await SendDataAsync(1, _maxData, endStream: false); + + // Rounds down so we don't go over the half window size and trigger an update + for (var i = 0; i < framesInConnectionWindow / 2; i++) + { + await SendDataAsync(1, _maxData, endStream: false); + } + + // Go over the threshold and trigger an update await SendDataAsync(1, _maxData, endStream: false); await SendRstStreamAsync(1); @@ -1977,7 +2085,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); Assert.Contains(1, _abortedStreamIds); - Assert.Equal(_maxData.Length * 3, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); + var updateSize = ((framesInConnectionWindow / 2) + 1) * _maxData.Length; + Assert.Equal(updateSize, connectionWindowUpdateFrame.WindowUpdateSizeIncrement); } [Fact] @@ -2065,22 +2174,33 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await SendSettingsAsync(); var frame = await ExpectAsync(Http2FrameType.SETTINGS, - withLength: Http2FrameReader.SettingSize * 2, + withLength: Http2FrameReader.SettingSize * 3, withFlags: 0, withStreamId: 0); // Only non protocol defaults are sent var settings = Http2FrameReader.ReadSettings(frame.PayloadSequence); - Assert.Equal(2, settings.Count); + Assert.Equal(3, settings.Count); var setting = settings[0]; Assert.Equal(Http2SettingsParameter.SETTINGS_MAX_CONCURRENT_STREAMS, setting.Parameter); Assert.Equal(100u, setting.Value); setting = settings[1]; + Assert.Equal(Http2SettingsParameter.SETTINGS_INITIAL_WINDOW_SIZE, setting.Parameter); + Assert.Equal(96 * 1024u, setting.Value); + + setting = settings[2]; Assert.Equal(Http2SettingsParameter.SETTINGS_MAX_HEADER_LIST_SIZE, setting.Parameter); Assert.Equal(32 * 1024u, setting.Value); + var update = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, + withLength: 4, + withFlags: (byte)Http2SettingsFrameFlags.NONE, + withStreamId: 0); + + Assert.Equal(1024 * 128 - (int)Http2PeerSettings.DefaultInitialWindowSize, update.WindowUpdateSizeIncrement); + await ExpectAsync(Http2FrameType.SETTINGS, withLength: 0, withFlags: (byte)Http2SettingsFrameFlags.ACK, @@ -2094,6 +2214,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests { _connection.ServerSettings.MaxConcurrentStreams = 1; _connection.ServerSettings.MaxHeaderListSize = 4 * 1024; + _connection.ServerSettings.InitialWindowSize = 1024 * 1024 * 10; _connectionTask = _connection.ProcessRequestsAsync(new DummyApplication(_noopApplication)); @@ -2101,22 +2222,33 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await SendSettingsAsync(); var frame = await ExpectAsync(Http2FrameType.SETTINGS, - withLength: Http2FrameReader.SettingSize * 2, + withLength: Http2FrameReader.SettingSize * 3, withFlags: 0, withStreamId: 0); // Only non protocol defaults are sent var settings = Http2FrameReader.ReadSettings(frame.PayloadSequence); - Assert.Equal(2, settings.Count); + Assert.Equal(3, settings.Count); var setting = settings[0]; Assert.Equal(Http2SettingsParameter.SETTINGS_MAX_CONCURRENT_STREAMS, setting.Parameter); Assert.Equal(1u, setting.Value); setting = settings[1]; + Assert.Equal(Http2SettingsParameter.SETTINGS_INITIAL_WINDOW_SIZE, setting.Parameter); + Assert.Equal(1024 * 1024 * 10u, setting.Value); + + setting = settings[2]; Assert.Equal(Http2SettingsParameter.SETTINGS_MAX_HEADER_LIST_SIZE, setting.Parameter); Assert.Equal(4 * 1024u, setting.Value); + var update = await ExpectAsync(Http2FrameType.WINDOW_UPDATE, + withLength: 4, + withFlags: (byte)Http2SettingsFrameFlags.NONE, + withStreamId: 0); + + Assert.Equal(1024 * 128u - Http2PeerSettings.DefaultInitialWindowSize, (uint)update.WindowUpdateSizeIncrement); + await ExpectAsync(Http2FrameType.SETTINGS, withLength: 0, withFlags: (byte)Http2SettingsFrameFlags.ACK, @@ -2277,7 +2409,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests context.Response.Headers["A"] = new string('a', headerValueLength); context.Response.Headers["B"] = new string('b', headerValueLength); return context.Response.Body.WriteAsync(new byte[payloadLength], 0, payloadLength); - }, expectedSettingsCount: 3); + }, expectedSettingsCount: 4); // Update client settings _clientSettings.MaxFrameSize = (uint)payloadLength; @@ -2323,7 +2455,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests await InitializeConnectionAsync(context => { return context.Response.Body.WriteAsync(new byte[clientMaxFrame], 0, clientMaxFrame); - }, expectedSettingsCount: 3); + }, expectedSettingsCount: 4); // Start request await StartStreamAsync(1, _browserRequestHeaders, endStream: true); diff --git a/test/Kestrel.InMemory.FunctionalTests/Http2/Http2TestBase.cs b/test/Kestrel.InMemory.FunctionalTests/Http2/Http2TestBase.cs index 4023b5080f..d14744906a 100644 --- a/test/Kestrel.InMemory.FunctionalTests/Http2/Http2TestBase.cs +++ b/test/Kestrel.InMemory.FunctionalTests/Http2/Http2TestBase.cs @@ -310,7 +310,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests _decodedHeaders[name.GetAsciiStringNonNullCharacters()] = value.GetAsciiOrUTF8StringNonNullCharacters(); } - protected async Task InitializeConnectionAsync(RequestDelegate application, int expectedSettingsCount = 2) + protected async Task InitializeConnectionAsync(RequestDelegate application, int expectedSettingsCount = 3) { _connectionTask = _connection.ProcessRequestsAsync(new DummyApplication(application)); @@ -322,6 +322,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests withFlags: 0, withStreamId: 0); + await ExpectAsync(Http2FrameType.WINDOW_UPDATE, + withLength: 4, + withFlags: 0, + withStreamId: 0); + await ExpectAsync(Http2FrameType.SETTINGS, withLength: 0, withFlags: (byte)Http2SettingsFrameFlags.ACK, diff --git a/test/Kestrel.InMemory.FunctionalTests/HttpProtocolSelectionTests.cs b/test/Kestrel.InMemory.FunctionalTests/HttpProtocolSelectionTests.cs index 7b2028ba69..22994a3130 100644 --- a/test/Kestrel.InMemory.FunctionalTests/HttpProtocolSelectionTests.cs +++ b/test/Kestrel.InMemory.FunctionalTests/HttpProtocolSelectionTests.cs @@ -40,9 +40,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests // Expect a SETTINGS frame (type 0x4) with default settings var expected = new byte[] { - 0x00, 0x00, 0x0C, // Payload Length (6 * settings count) + 0x00, 0x00, 0x12, // Payload Length (6 * settings count) 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // SETTINGS frame (type 0x04) 0x00, 0x03, 0x00, 0x00, 0x00, 0x64, // Connection limit + 0x00, 0x04, 0x00, 0x01, 0x80, 0x00, // Initial window size 0x00, 0x06, 0x00, 0x00, 0x80, 0x00 // Header size limit }; var testContext = new TestServiceContext(LoggerFactory);