Expose HttpResponse PipeWriter in Kestrel (#7110)

src/Http/Http.Abstractions/src/Extensions/HttpResponseWritingExtensions.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.IO.Pipelines;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -60,8 +61,73 @@ namespace Microsoft.AspNetCore.Http
                 throw new ArgumentNullException(nameof(encoding));
             }
 
-            byte[] data = encoding.GetBytes(text);
-            return response.Body.WriteAsync(data, 0, data.Length, cancellationToken);
+            // Need to call StartAsync before GetMemory/GetSpan
+            if (!response.HasStarted)
+            {
+                var startAsyncTask = response.StartAsync(cancellationToken);
+                if (!startAsyncTask.IsCompletedSuccessfully)
+                {
+                    return StartAndWriteAsyncAwaited(response, text, encoding, cancellationToken, startAsyncTask);
+                }
+            }
+
+            Write(response, text, encoding);
+
+            var flushAsyncTask = response.BodyPipe.FlushAsync(cancellationToken);
+            if (flushAsyncTask.IsCompletedSuccessfully)
+            {
+                // Most implementations of ValueTask reset state in GetResult, so call it before returning a completed task.
+                flushAsyncTask.GetAwaiter().GetResult();
+                return Task.CompletedTask;
+            }
+
+            return flushAsyncTask.AsTask();
+        }
+
+        private static async Task StartAndWriteAsyncAwaited(this HttpResponse response, string text, Encoding encoding, CancellationToken cancellationToken, Task startAsyncTask)
+        {
+            await startAsyncTask;
+            Write(response, text, encoding);
+            await response.BodyPipe.FlushAsync(cancellationToken);
+        }
+
+        private static void Write(this HttpResponse response, string text, Encoding encoding)
+        {
+            var pipeWriter = response.BodyPipe;
+            var encodedLength = encoding.GetByteCount(text);
+            var destination = pipeWriter.GetSpan(encodedLength);
+
+            if (encodedLength <= destination.Length)
+            {
+                // Just call Encoding.GetBytes if everything will fit into a single segment.
+                var bytesWritten = encoding.GetBytes(text, destination);
+                pipeWriter.Advance(bytesWritten);
+            }
+            else
+            {
+                WriteMutliSegmentEncoded(pipeWriter, text, encoding, destination, encodedLength);
+            }
+        }
+
+        private static void WriteMutliSegmentEncoded(PipeWriter writer, string text, Encoding encoding, Span<byte> destination, int encodedLength)
+        {
+            var encoder = encoding.GetEncoder();
+            var source = text.AsSpan();
+            var completed = false;
+            var totalBytesUsed = 0;
+
+            // This may be a bug, but encoder.Convert returns completed = true for UTF7 too early.
+            // Therefore, we check encodedLength - totalBytesUsed too.
+            while (!completed || encodedLength - totalBytesUsed != 0)
+            {
+                encoder.Convert(source, destination, flush: source.Length == 0, out var charsUsed, out var bytesUsed, out completed);
+                totalBytesUsed += bytesUsed;
+
+                writer.Advance(bytesUsed);
+                source = source.Slice(charsUsed);
+
+                destination = writer.GetSpan(encodedLength - totalBytesUsed);
+            }
         }
     }
-}
+}

src/Http/Http.Abstractions/test/HttpResponseWritingExtensionsTests.cs

@@ -1,7 +1,11 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
+using System.IO.Pipelines;
+using System.IO.Pipelines.Tests;
+using System.Text;
 using System.Threading.Tasks;
 using Xunit;
 
@@ -28,6 +32,65 @@ namespace Microsoft.AspNetCore.Http
             Assert.Equal(22, context.Response.Body.Length);
         }
 
+        [Theory]
+        [MemberData(nameof(Encodings))]
+        public async Task WritingTextThatRequiresMultipleSegmentsWorks(Encoding encoding)
+        {
+            // Need to change the StreamPipeWriter with a capped MemoryPool
+            var memoryPool = new TestMemoryPool(maxBufferSize: 16);
+            var outputStream = new MemoryStream();
+            var streamPipeWriter = new StreamPipeWriter(outputStream, minimumSegmentSize: 0, memoryPool);
+
+            HttpContext context = new DefaultHttpContext();
+            context.Response.BodyPipe = streamPipeWriter;
+
+            var inputString = "昨日すき焼きを食べました";
+            var expected = encoding.GetBytes(inputString);
+            await context.Response.WriteAsync(inputString, encoding);
+
+            outputStream.Position = 0;
+            var actual = new byte[expected.Length];
+            var length = outputStream.Read(actual);
+
+            var res1 = encoding.GetString(actual);
+            var res2 = encoding.GetString(expected);
+            Assert.Equal(expected.Length, length);
+            Assert.Equal(expected, actual);
+            streamPipeWriter.Complete();
+        }
+
+        [Theory]
+        [MemberData(nameof(Encodings))]
+        public async Task WritingTextWithPassedInEncodingWorks(Encoding encoding)
+        {
+            HttpContext context = CreateRequest();
+
+            var inputString = "昨日すき焼きを食べました";
+            var expected = encoding.GetBytes(inputString);
+            await context.Response.WriteAsync(inputString, encoding);
+
+            context.Response.Body.Position = 0;
+            var actual = new byte[expected.Length * 2];
+            var length = context.Response.Body.Read(actual);
+
+            var actualShortened = new byte[length];
+            Array.Copy(actual, actualShortened, length);
+
+            Assert.Equal(expected.Length, length);
+            Assert.Equal(expected, actualShortened);
+        }
+
+        public static TheoryData<Encoding> Encodings =>
+            new TheoryData<Encoding>
+            {
+                        { Encoding.ASCII },
+                        { Encoding.BigEndianUnicode },
+                        { Encoding.Unicode },
+                        { Encoding.UTF32 },
+                        { Encoding.UTF7 },
+                        { Encoding.UTF8 }
+            };
+
         private HttpContext CreateRequest()
         {
             HttpContext context = new DefaultHttpContext();

src/Http/Http.Abstractions/test/Microsoft.AspNetCore.Http.Abstractions.Tests.csproj

@@ -4,6 +4,10 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Http\test\Microsoft.AspNetCore.Http.Tests.csproj" />
+  </ItemGroup>
+
   <ItemGroup>
     <Reference Include="Microsoft.AspNetCore.Http" />
   </ItemGroup>

src/Http/Http.Features/src/FeatureReferences.cs

@@ -95,4 +95,4 @@ namespace Microsoft.AspNetCore.Http.Features
         public TFeature Fetch<TFeature>(ref TFeature cached, Func<IFeatureCollection, TFeature> factory)
             where TFeature : class => Fetch(ref cached, Collection, factory);
     }
-}
+}

src/Http/Http/src/Internal/DefaultHttpResponse.cs

@@ -151,7 +151,7 @@ namespace Microsoft.AspNetCore.Http.Internal
                 return HttpResponseFeature.Body.FlushAsync(cancellationToken);
             }
 
-            return HttpResponseStartFeature.StartAsync();
+            return HttpResponseStartFeature.StartAsync(cancellationToken);
         }
 
         struct FeatureInterfaces

src/Http/Http/src/ReadOnlyPipeStream.cs

@@ -13,7 +13,6 @@ namespace System.IO.Pipelines
     /// </summary>
     public class ReadOnlyPipeStream : Stream
     {
-        private readonly PipeReader _pipeReader;
         private bool _allowSynchronousIO = true;
 
         /// <summary>
@@ -33,7 +32,7 @@ namespace System.IO.Pipelines
         public ReadOnlyPipeStream(PipeReader pipeReader, bool allowSynchronousIO)
         {
             _allowSynchronousIO = allowSynchronousIO;
-            _pipeReader = pipeReader;
+            InnerPipeReader = pipeReader;
         }
 
         /// <inheritdoc />
@@ -62,6 +61,8 @@ namespace System.IO.Pipelines
             set => throw new NotSupportedException();
         }
 
+        public PipeReader InnerPipeReader { get; }
+
         /// <inheritdoc />
         public override void Write(byte[] buffer, int offset, int count)
             => throw new NotSupportedException();
@@ -160,7 +161,7 @@ namespace System.IO.Pipelines
         {
             while (true)
             {
-                var result = await _pipeReader.ReadAsync(cancellationToken);
+                var result = await InnerPipeReader.ReadAsync(cancellationToken);
                 var readableBuffer = result.Buffer;
                 var readableBufferLength = readableBuffer.Length;
 
@@ -186,7 +187,7 @@ namespace System.IO.Pipelines
                 }
                 finally
                 {
-                    _pipeReader.AdvanceTo(consumed);
+                    InnerPipeReader.AdvanceTo(consumed);
                 }
             }
         }
@@ -211,7 +212,7 @@ namespace System.IO.Pipelines
         {
             while (true)
             {
-                var result = await _pipeReader.ReadAsync(cancellationToken);
+                var result = await InnerPipeReader.ReadAsync(cancellationToken);
                 var readableBuffer = result.Buffer;
                 var readableBufferLength = readableBuffer.Length;
 
@@ -232,7 +233,7 @@ namespace System.IO.Pipelines
                 }
                 finally
                 {
-                    _pipeReader.AdvanceTo(readableBuffer.End);
+                    InnerPipeReader.AdvanceTo(readableBuffer.End);
                 }
             }
         }

src/Http/Http/src/StreamPipeReader.cs

@@ -17,7 +17,6 @@ namespace System.IO.Pipelines
     {
         private readonly int _minimumSegmentSize;
         private readonly int _minimumReadThreshold;
-        private readonly Stream _readingStream;
         private readonly MemoryPool<byte> _pool;
 
         private CancellationTokenSource _internalTokenSource;
@@ -42,7 +41,6 @@ namespace System.IO.Pipelines
         {
         }
 
-
         /// <summary>
         /// Creates a new StreamPipeReader.
         /// </summary>
@@ -50,7 +48,7 @@ namespace System.IO.Pipelines
         /// <param name="options">The options to use.</param>
         public StreamPipeReader(Stream readingStream, StreamPipeReaderOptions options)
         {
-            _readingStream = readingStream ?? throw new ArgumentNullException(nameof(readingStream));
+            InnerStream = readingStream ?? throw new ArgumentNullException(nameof(readingStream));
 
             if (options == null)
             {
@@ -70,7 +68,7 @@ namespace System.IO.Pipelines
         /// <summary>
         /// Gets the inner stream that is being read from.
         /// </summary>
-        public Stream InnerStream => _readingStream;
+        public Stream InnerStream { get; }
 
         /// <inheritdoc />
         public override void AdvanceTo(SequencePosition consumed)
@@ -235,7 +233,7 @@ namespace System.IO.Pipelines
                 {
                     AllocateReadTail();
 #if NETCOREAPP3_0
-                    var length = await _readingStream.ReadAsync(_readTail.AvailableMemory.Slice(_readTail.End), tokenSource.Token);
+                    var length = await InnerStream.ReadAsync(_readTail.AvailableMemory.Slice(_readTail.End), tokenSource.Token);
 #elif NETSTANDARD2_0
                     if (!MemoryMarshal.TryGetArray<byte>(_readTail.AvailableMemory.Slice(_readTail.End), out var arraySegment))
                     {

src/Http/Http/src/StreamPipeWriter.cs

@@ -14,7 +14,6 @@ namespace System.IO.Pipelines
     public class StreamPipeWriter : PipeWriter, IDisposable
     {
         private readonly int _minimumSegmentSize;
-        private readonly Stream _writingStream;
         private int _bytesWritten;
 
         private List<CompletedBuffer> _completedSegments;
@@ -53,14 +52,14 @@ namespace System.IO.Pipelines
         public StreamPipeWriter(Stream writingStream, int minimumSegmentSize, MemoryPool<byte> pool = null)
         {
             _minimumSegmentSize = minimumSegmentSize;
-            _writingStream = writingStream;
+            InnerStream = writingStream;
             _pool = pool ?? MemoryPool<byte>.Shared;
         }
 
         /// <summary>
         /// Gets the inner stream that is being written to.
         /// </summary>
-        public Stream InnerStream => _writingStream;
+        public Stream InnerStream { get; }
 
         /// <inheritdoc />
         public override void Advance(int count)
@@ -180,10 +179,10 @@ namespace System.IO.Pipelines
                         {
                             var segment = _completedSegments[0];
 #if NETCOREAPP3_0
-                            await _writingStream.WriteAsync(segment.Buffer.Slice(0, segment.Length), localToken);
+                            await InnerStream.WriteAsync(segment.Buffer.Slice(0, segment.Length), localToken);
 #elif NETSTANDARD2_0
                             MemoryMarshal.TryGetArray<byte>(segment.Buffer, out var arraySegment);
-                            await _writingStream.WriteAsync(arraySegment.Array, 0, segment.Length, localToken);
+                            await InnerStream.WriteAsync(arraySegment.Array, 0, segment.Length, localToken);
 #else
 #error Target frameworks need to be updated.
 #endif
@@ -196,10 +195,10 @@ namespace System.IO.Pipelines
                     if (!_currentSegment.IsEmpty)
                     {
 #if NETCOREAPP3_0
-                        await _writingStream.WriteAsync(_currentSegment.Slice(0, _position), localToken);
+                        await InnerStream.WriteAsync(_currentSegment.Slice(0, _position), localToken);
 #elif NETSTANDARD2_0
                         MemoryMarshal.TryGetArray<byte>(_currentSegment, out var arraySegment);
-                        await _writingStream.WriteAsync(arraySegment.Array, 0, _position, localToken);
+                        await InnerStream.WriteAsync(arraySegment.Array, 0, _position, localToken);
 #else
 #error Target frameworks need to be updated.
 #endif
@@ -207,7 +206,7 @@ namespace System.IO.Pipelines
                         _position = 0;
                     }
 
-                    await _writingStream.FlushAsync(localToken);
+                    await InnerStream.FlushAsync(localToken);
 
                     return new FlushResult(isCanceled: false, _isCompleted);
                 }

src/Http/Http/src/WriteOnlyPipeStream.cs

@@ -11,7 +11,6 @@ namespace System.IO.Pipelines
     /// </summary>
     public class WriteOnlyPipeStream : Stream
     {
-        private PipeWriter _pipeWriter;
         private bool _allowSynchronousIO = true;
 
         /// <summary>
@@ -30,7 +29,7 @@ namespace System.IO.Pipelines
         /// <param name="allowSynchronousIO">Whether synchronous IO is allowed.</param>
         public WriteOnlyPipeStream(PipeWriter pipeWriter, bool allowSynchronousIO)
         {
-            _pipeWriter = pipeWriter;
+            InnerPipeWriter = pipeWriter;
             _allowSynchronousIO = allowSynchronousIO;
         }
 
@@ -60,6 +59,8 @@ namespace System.IO.Pipelines
             set => throw new NotSupportedException();
         }
 
+        public PipeWriter InnerPipeWriter { get; }
+
         /// <inheritdoc />
         public override int Read(byte[] buffer, int offset, int count)
             => throw new NotSupportedException();
@@ -82,7 +83,7 @@ namespace System.IO.Pipelines
         /// <inheritdoc />
         public override async Task FlushAsync(CancellationToken cancellationToken)
         {
-            await _pipeWriter.FlushAsync(cancellationToken);
+            await InnerPipeWriter.FlushAsync(cancellationToken);
         }
 
         /// <inheritdoc />
@@ -150,13 +151,27 @@ namespace System.IO.Pipelines
         /// <inheritdoc />
         public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
         {
-            return WriteAsync(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken).AsTask();
+            return WriteAsyncInternal(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken);
         }
 
         /// <inheritdoc />
-        public override async ValueTask WriteAsync(ReadOnlyMemory<byte> source, CancellationToken cancellationToken = default)
+        public override ValueTask WriteAsync(ReadOnlyMemory<byte> source, CancellationToken cancellationToken = default)
         {
-            await _pipeWriter.WriteAsync(source, cancellationToken);
+            return new ValueTask(WriteAsyncInternal(source, cancellationToken));
+        }
+
+        private Task WriteAsyncInternal(ReadOnlyMemory<byte> source, CancellationToken cancellationToken = default)
+        {
+            var task = InnerPipeWriter.WriteAsync(source, cancellationToken);
+
+            if (task.IsCompletedSuccessfully)
+            {
+                // Most ValueTask implementations reset in GetResult, so call it before returning completed task
+                task.GetAwaiter().GetResult();
+                return Task.CompletedTask;
+            }
+
+            return task.AsTask();
         }
     }
 }

src/Http/Http/test/Internal/DefaultHttpResponseTests.cs

@@ -116,6 +116,22 @@ namespace Microsoft.AspNetCore.Http.Internal
             mock.Verify(m => m.StartAsync(default), Times.Once());
         }
 
+        [Fact]
+        public async Task ResponseStart_CallsFeatureIfSetWithProvidedCancellationToken()
+        {
+            var features = new FeatureCollection();
+
+            var mock = new Mock<IHttpResponseStartFeature>();
+            var ct = new CancellationToken();
+            mock.Setup(o => o.StartAsync(It.Is<CancellationToken>((localCt) => localCt.Equals(ct)))).Returns(Task.CompletedTask);
+            features.Set(mock.Object);
+
+            var context = new DefaultHttpContext(features);
+            await context.Response.StartAsync(ct);
+
+            mock.Verify(m => m.StartAsync(default), Times.Once());
+        }
+
         [Fact]
         public async Task ResponseStart_CallsResponseBodyFlushIfNotSet()
         {

src/Http/Http/test/ReadOnlyPipeStreamTests.cs

@@ -150,6 +150,13 @@ namespace System.IO.Pipelines.Tests
             Assert.Throws<InvalidOperationException>(() => readOnlyPipeStream.Read(new byte[0], 0, 0));
         }
 
+        [Fact]
+        public void InnerPipeReaderReturnsPipeReader()
+        {
+            var readOnlyPipeStream = new ReadOnlyPipeStream(Reader, allowSynchronousIO: false);
+            Assert.Equal(Reader, readOnlyPipeStream.InnerPipeReader);
+        }
+
         private async Task<Mock<PipeReader>> SetupMockPipeReader()
         {
             await WriteByteArrayToPipeAsync(new byte[1]);

src/Http/Http/test/StreamPipeReaderTests.cs

@@ -611,6 +611,12 @@ namespace System.IO.Pipelines.Tests
             Assert.True(readResult.IsCompleted);
         }
 
+        [Fact]
+        public void InnerStreamReturnsStream()
+        {
+            Assert.Equal(Stream, ((StreamPipeReader)Reader).InnerStream);
+        }
+
         private async Task<string> ReadFromPipeAsString()
         {
             var readResult = await Reader.ReadAsync();

src/Http/Http/test/StreamPipeWriterTests.cs

@@ -25,27 +25,22 @@ namespace System.IO.Pipelines.Tests
         }
 
         [Theory]
-        [InlineData(100, 1000)]
-        [InlineData(100, 8000)]
-        [InlineData(100, 10000)]
-        [InlineData(8000, 100)]
-        [InlineData(8000, 8000)]
-        public async Task CanAdvanceWithPartialConsumptionOfFirstSegment(int firstWriteLength, int secondWriteLength)
+        [InlineData(100)]
+        [InlineData(4000)]
+        public async Task CanAdvanceWithPartialConsumptionOfFirstSegment(int firstWriteLength)
         {
             Writer = new StreamPipeWriter(Stream, MinimumSegmentSize, new TestMemoryPool(maxBufferSize: 20000));
             await Writer.WriteAsync(Encoding.ASCII.GetBytes("a"));
 
-            var expectedLength = firstWriteLength + secondWriteLength + 1;
-
             var memory = Writer.GetMemory(firstWriteLength);
             Writer.Advance(firstWriteLength);
 
-            memory = Writer.GetMemory(secondWriteLength);
-            Writer.Advance(secondWriteLength);
+            memory = Writer.GetMemory();
+            Writer.Advance(memory.Length);
 
             await Writer.FlushAsync();
 
-            Assert.Equal(expectedLength, Read().Length);
+            Assert.Equal(firstWriteLength + memory.Length + 1, Read().Length);
         }
 
         [Fact]
@@ -408,6 +403,12 @@ namespace System.IO.Pipelines.Tests
             Assert.Equal(expectedString, ReadAsString());
         }
 
+        [Fact]
+        public void InnerStreamReturnsStream()
+        {
+            Assert.Equal(Stream, ((StreamPipeWriter)Writer).InnerStream);
+        }
+
         private void WriteStringToStream(string input)
         {
             var buffer = Encoding.ASCII.GetBytes(input);

src/Http/Http/test/WriteOnlyPipeStreamTests.cs

@@ -195,5 +195,12 @@ namespace System.IO.Pipelines.Tests
             Assert.Throws<InvalidOperationException>(() => writeOnlyPipeStream.Write(new byte[0], 0, 0));
             Assert.Throws<InvalidOperationException>(() => writeOnlyPipeStream.Flush());
         }
+
+        [Fact]
+        public void InnerPipeWriterReturnsPipeWriter()
+        {
+            var writeOnlyPipeStream = new WriteOnlyPipeStream(Writer, allowSynchronousIO: false);
+            Assert.Equal(Writer, writeOnlyPipeStream.InnerPipeWriter);
+        }
     }
 }

src/Middleware/ResponseCompression/src/BodyWrapperStream.cs

@@ -15,13 +15,14 @@ namespace Microsoft.AspNetCore.ResponseCompression
     /// <summary>
     /// Stream wrapper that create specific compression stream only if necessary.
     /// </summary>
-    internal class BodyWrapperStream : Stream, IHttpBufferingFeature, IHttpSendFileFeature
+    internal class BodyWrapperStream : Stream, IHttpBufferingFeature, IHttpSendFileFeature, IHttpResponseStartFeature
     {
         private readonly HttpContext _context;
         private readonly Stream _bodyOriginalStream;
         private readonly IResponseCompressionProvider _provider;
         private readonly IHttpBufferingFeature _innerBufferFeature;
         private readonly IHttpSendFileFeature _innerSendFileFeature;
+        private readonly IHttpResponseStartFeature _innerStartFeature;
 
         private ICompressionProvider _compressionProvider = null;
         private bool _compressionChecked = false;
@@ -30,13 +31,14 @@ namespace Microsoft.AspNetCore.ResponseCompression
         private bool _autoFlush = false;
 
         internal BodyWrapperStream(HttpContext context, Stream bodyOriginalStream, IResponseCompressionProvider provider,
-            IHttpBufferingFeature innerBufferFeature, IHttpSendFileFeature innerSendFileFeature)
+            IHttpBufferingFeature innerBufferFeature, IHttpSendFileFeature innerSendFileFeature, IHttpResponseStartFeature innerStartFeature)
         {
             _context = context;
             _bodyOriginalStream = bodyOriginalStream;
             _provider = provider;
             _innerBufferFeature = innerBufferFeature;
             _innerSendFileFeature = innerSendFileFeature;
+            _innerStartFeature = innerStartFeature;
         }
 
         internal ValueTask FinishCompressionAsync()
@@ -318,5 +320,17 @@ namespace Microsoft.AspNetCore.ResponseCompression
                 }
             }
         }
+
+        public Task StartAsync(CancellationToken token = default)
+        {
+            OnWrite();
+
+            if (_innerStartFeature != null)
+            {
+                return _innerStartFeature.StartAsync(token);
+            }
+
+            return Task.CompletedTask;
+        }
     }
 }

src/Middleware/ResponseCompression/src/ResponseCompressionMiddleware.cs

@@ -5,7 +5,6 @@ using System;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
-using Microsoft.Extensions.Options;
 
 namespace Microsoft.AspNetCore.ResponseCompression
 {
@@ -55,9 +54,10 @@ namespace Microsoft.AspNetCore.ResponseCompression
             var bodyStream = context.Response.Body;
             var originalBufferFeature = context.Features.Get<IHttpBufferingFeature>();
             var originalSendFileFeature = context.Features.Get<IHttpSendFileFeature>();
+            var originalStartFeature = context.Features.Get<IHttpResponseStartFeature>();
 
             var bodyWrapperStream = new BodyWrapperStream(context, bodyStream, _provider,
-                originalBufferFeature, originalSendFileFeature);
+                originalBufferFeature, originalSendFileFeature, originalStartFeature);
             context.Response.Body = bodyWrapperStream;
             context.Features.Set<IHttpBufferingFeature>(bodyWrapperStream);
             if (originalSendFileFeature != null)
@@ -65,6 +65,11 @@ namespace Microsoft.AspNetCore.ResponseCompression
                 context.Features.Set<IHttpSendFileFeature>(bodyWrapperStream);
             }
 
+            if (originalStartFeature != null)
+            {
+                context.Features.Set<IHttpResponseStartFeature>(bodyWrapperStream);
+            }
+
             try
             {
                 await _next(context);
@@ -78,6 +83,11 @@ namespace Microsoft.AspNetCore.ResponseCompression
                 {
                     context.Features.Set(originalSendFileFeature);
                 }
+
+                if (originalStartFeature != null)
+                {
+                    context.Features.Set(originalStartFeature);
+                }
             }
         }
     }

src/Middleware/ResponseCompression/test/BodyWrapperStreamTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -26,7 +26,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var httpContext = new DefaultHttpContext();
             httpContext.Response.Headers[HeaderNames.Vary] = providedVaryHeader;
-            var stream = new BodyWrapperStream(httpContext, new MemoryStream(), new MockResponseCompressionProvider(flushable: true), null, null);
+            var stream = new BodyWrapperStream(httpContext, new MemoryStream(), new MockResponseCompressionProvider(flushable: true), null, null, null);
 
             stream.Write(new byte[] { }, 0, 0);
 
@@ -51,7 +51,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
                     written = new ArraySegment<byte>(b, 0, c).ToArray();
                 });
 
-            var stream = new BodyWrapperStream(new DefaultHttpContext(), mock.Object, new MockResponseCompressionProvider(flushable), null, null);
+            var stream = new BodyWrapperStream(new DefaultHttpContext(), mock.Object, new MockResponseCompressionProvider(flushable), null, null, null);
 
             stream.DisableResponseBuffering();
             stream.Write(buffer, 0, buffer.Length);
@@ -67,7 +67,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
             var buffer = new byte[] { 1 };
 
             var memoryStream = new MemoryStream();
-            var stream = new BodyWrapperStream(new DefaultHttpContext(), memoryStream, new MockResponseCompressionProvider(flushable), null, null);
+            var stream = new BodyWrapperStream(new DefaultHttpContext(), memoryStream, new MockResponseCompressionProvider(flushable), null, null, null);
 
             stream.DisableResponseBuffering();
             await stream.WriteAsync(buffer, 0, buffer.Length);
@@ -80,7 +80,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var memoryStream = new MemoryStream();
 
-            var stream = new BodyWrapperStream(new DefaultHttpContext(), memoryStream, new MockResponseCompressionProvider(true), null, null);
+            var stream = new BodyWrapperStream(new DefaultHttpContext(), memoryStream, new MockResponseCompressionProvider(true), null, null, null);
 
             stream.DisableResponseBuffering();
 
@@ -99,7 +99,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
 
             var memoryStream = new MemoryStream();
 
-            var stream = new BodyWrapperStream(new DefaultHttpContext(), memoryStream, new MockResponseCompressionProvider(flushable), null, null);
+            var stream = new BodyWrapperStream(new DefaultHttpContext(), memoryStream, new MockResponseCompressionProvider(flushable), null, null, null);
 
             stream.DisableResponseBuffering();
             stream.BeginWrite(buffer, 0, buffer.Length, (o) => {}, null);

src/Middleware/ResponseCompression/test/ResponseCompressionMiddlewareTest.cs

@@ -37,8 +37,8 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             get
             {
-                yield return new EncodingTestData("gzip", expectedBodyLength: 24);
-                yield return new EncodingTestData("br", expectedBodyLength: 20);
+                yield return new EncodingTestData("gzip", expectedBodyLength: 30);
+                yield return new EncodingTestData("br", expectedBodyLength: 21);
             }
         }
 
@@ -64,7 +64,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var (response, logMessages) = await InvokeMiddleware(100, requestAcceptEncodings: new[] { "gzip", "deflate" }, responseType: TextPlain);
 
-            CheckResponseCompressed(response, expectedBodyLength: 24, expectedEncoding: "gzip");
+            CheckResponseCompressed(response, expectedBodyLength: 30, expectedEncoding: "gzip");
             AssertCompressedWithLog(logMessages, "gzip");
         }
 
@@ -73,7 +73,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var (response, logMessages) = await InvokeMiddleware(100, requestAcceptEncodings: new[] { "br" }, responseType: TextPlain);
 
-            CheckResponseCompressed(response, expectedBodyLength: 20, expectedEncoding: "br");
+            CheckResponseCompressed(response, expectedBodyLength: 21, expectedEncoding: "br");
             AssertCompressedWithLog(logMessages, "br");
         }
 
@@ -84,7 +84,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var (response, logMessages) = await InvokeMiddleware(100, new[] { encoding1, encoding2 }, responseType: TextPlain);
 
-            CheckResponseCompressed(response, expectedBodyLength: 20, expectedEncoding: "br");
+            CheckResponseCompressed(response, expectedBodyLength: 21, expectedEncoding: "br");
             AssertCompressedWithLog(logMessages, "br");
         }
 
@@ -101,7 +101,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
 
             var (response, logMessages) = await InvokeMiddleware(100, new[] { encoding1, encoding2 }, responseType: TextPlain, configure: Configure);
 
-            CheckResponseCompressed(response, expectedBodyLength: 24, expectedEncoding: "gzip");
+            CheckResponseCompressed(response, expectedBodyLength: 30, expectedEncoding: "gzip");
             AssertCompressedWithLog(logMessages, "gzip");
         }
 
@@ -126,7 +126,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var (response, logMessages) = await InvokeMiddleware(uncompressedBodyLength: 100, requestAcceptEncodings: new[] { "gzip" }, contentType);
 
-            CheckResponseCompressed(response, expectedBodyLength: 24, expectedEncoding: "gzip");
+            CheckResponseCompressed(response, expectedBodyLength: 30, expectedEncoding: "gzip");
             AssertCompressedWithLog(logMessages, "gzip");
         }
 
@@ -158,7 +158,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
 
             var response = await client.SendAsync(request);
 
-            CheckResponseCompressed(response, expectedBodyLength: 123, expectedEncoding: "gzip");
+            CheckResponseCompressed(response, expectedBodyLength: 133, expectedEncoding: "gzip");
         }
 
         [Theory]
@@ -251,7 +251,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
 
             if (compress)
             {
-                CheckResponseCompressed(response, expectedBodyLength: 24, expectedEncoding: "gzip");
+                CheckResponseCompressed(response, expectedBodyLength: 30, expectedEncoding: "gzip");
                 AssertCompressedWithLog(logMessages, "gzip");
             }
             else
@@ -272,7 +272,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
                     options.ExcludedMimeTypes = new[] { "text/*" };
                 });
 
-            CheckResponseCompressed(response, expectedBodyLength: 24, expectedEncoding: "gzip");
+            CheckResponseCompressed(response, expectedBodyLength: 30, expectedEncoding: "gzip");
             AssertCompressedWithLog(logMessages, "gzip");
         }
 
@@ -317,7 +317,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         {
             var (response, logMessages) = await InvokeMiddleware(100, requestAcceptEncodings: new[] { "*" }, responseType: TextPlain);
 
-            CheckResponseCompressed(response, expectedBodyLength: 20, expectedEncoding: "br");
+            CheckResponseCompressed(response, expectedBodyLength: 21, expectedEncoding: "br");
             AssertCompressedWithLog(logMessages, "br");
         }
 
@@ -334,9 +334,9 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
         }
 
         [Theory]
-        [InlineData(new[] { "identity;q=0.5", "gzip;q=1" }, 24)]
-        [InlineData(new[] { "identity;q=0", "gzip;q=0.8" }, 24)]
-        [InlineData(new[] { "identity;q=0.5", "gzip" }, 24)]
+        [InlineData(new[] { "identity;q=0.5", "gzip;q=1" }, 30)]
+        [InlineData(new[] { "identity;q=0", "gzip;q=0.8" }, 30)]
+        [InlineData(new[] { "identity;q=0.5", "gzip" }, 30)]
         public async Task Request_AcceptWithHigherCompressionQuality_Compressed(string[] acceptEncodings, int expectedBodyLength)
         {
             var (response, logMessages) = await InvokeMiddleware(100, requestAcceptEncodings: acceptEncodings, responseType: TextPlain);
@@ -404,7 +404,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
 
         [Theory]
         [InlineData(false, 100)]
-        [InlineData(true, 24)]
+        [InlineData(true, 30)]
         public async Task Request_Https_CompressedIfEnabled(bool enableHttps, int expectedLength)
         {
             var sink = new TestSink(
@@ -908,7 +908,7 @@ namespace Microsoft.AspNetCore.ResponseCompression.Tests
 
             var response = await client.SendAsync(request);
 
-            CheckResponseCompressed(response, expectedBodyLength: 40, expectedEncoding: "gzip");
+            CheckResponseCompressed(response, expectedBodyLength: 46, expectedEncoding: "gzip");
 
             Assert.False(fakeSendFile.Invoked);
         }

src/Middleware/WebSockets/samples/EchoApp/Program.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;

src/Servers/Kestrel/Core/src/CoreStrings.resx

@@ -596,4 +596,7 @@ For more information on configuring HTTPS see https://go.microsoft.com/fwlink/?l
   <data name="ServerShutdownDuringConnectionInitialization" xml:space="preserve">
     <value>Server shutdown started during connection initialization.</value>
   </data>
+  <data name="StartAsyncBeforeGetMemory" xml:space="preserve">
+    <value>Cannot call GetMemory() until response has started. Call HttpResponse.StartAsync() before calling GetMemory().</value>
+  </data>
 </root>

src/Servers/Kestrel/Core/src/Internal/Http/ChunkWriter.cs

@@ -44,13 +44,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             return count;
         }
 
-        internal static void WriteBeginChunkBytes(this ref BufferWriter<PipeWriter> start, int dataCount)
+        internal static int WriteBeginChunkBytes(this ref BufferWriter<PipeWriter> start, int dataCount)
         {
             // 10 bytes is max length + \r\n
             start.Ensure(10);
 
             var count = BeginChunkBytes(dataCount, start.Span);
             start.Advance(count);
+            return count;
         }
 
         internal static void WriteEndChunkBytes(this ref BufferWriter<PipeWriter> start)

src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs

@@ -52,7 +52,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 _context.ConnectionContext,
                 _context.ServiceContext.Log,
                 _context.TimeoutControl,
-                this);
+                this,
+                _context.MemoryPool);
 
             Input = _context.Transport.Input;
             Output = _http1Output;

src/Servers/Kestrel/Core/src/Internal/Http/Http1OutputProducer.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Buffers;
+using System.Diagnostics;
 using System.IO.Pipelines;
 using System.Text;
 using System.Threading;
@@ -25,15 +26,30 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         private readonly IKestrelTrace _log;
         private readonly IHttpMinResponseDataRateFeature _minResponseDataRateFeature;
         private readonly TimingPipeFlusher _flusher;
+        private readonly MemoryPool<byte> _memoryPool;
 
-        // This locks access to to all of the below fields
+        // This locks access to all of the below fields
         private readonly object _contextLock = new object();
 
-        private bool _completed = false;
+        private bool _pipeWriterCompleted;
+        private bool _completed;
         private bool _aborted;
         private long _unflushedBytes;
-
+        private bool _autoChunk;
         private readonly PipeWriter _pipeWriter;
+        private const int MemorySizeThreshold = 1024;
+        private const int BeginChunkLengthMax = 5;
+        private const int EndChunkLength = 2;
+
+        // Chunked responses need to be treated uniquely when using GetMemory + Advance.
+        // We need to know the size of the data written to the chunk before calling Advance on the
+        // PipeWriter, meaning we internally track how far we have advanced through a current chunk (_advancedBytesForChunk).
+        // Once write or flush is called, we modify the _currentChunkMemory to prepend the size of data written
+        // and append the end terminator.
+        private int _advancedBytesForChunk;
+        private Memory<byte> _currentChunkMemory;
+        private bool _currentChunkMemoryUpdated;
+        private IMemoryOwner<byte> _fakeMemoryOwner;
 
         public Http1OutputProducer(
             PipeWriter pipeWriter,
@@ -41,7 +57,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             ConnectionContext connectionContext,
             IKestrelTrace log,
             ITimeoutControl timeoutControl,
-            IHttpMinResponseDataRateFeature minResponseDataRateFeature)
+            IHttpMinResponseDataRateFeature minResponseDataRateFeature,
+            MemoryPool<byte> memoryPool)
         {
             _pipeWriter = pipeWriter;
             _connectionId = connectionId;
@@ -49,6 +66,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             _log = log;
             _minResponseDataRateFeature = minResponseDataRateFeature;
             _flusher = new TimingPipeFlusher(pipeWriter, timeoutControl, log);
+            _memoryPool = memoryPool;
         }
 
         public Task WriteDataAsync(ReadOnlySpan<byte> buffer, CancellationToken cancellationToken = default)
@@ -58,27 +76,114 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 return Task.FromCanceled(cancellationToken);
             }
 
+            return WriteAsync(buffer, cancellationToken).AsTask();
+        }
+
+        public ValueTask<FlushResult> WriteDataToPipeAsync(ReadOnlySpan<byte> buffer, CancellationToken cancellationToken = default)
+        {
+            if (cancellationToken.IsCancellationRequested)
+            {
+                return new ValueTask<FlushResult>(Task.FromCanceled<FlushResult>(cancellationToken));
+            }
+
             return WriteAsync(buffer, cancellationToken);
         }
 
-        public Task WriteStreamSuffixAsync()
+        public ValueTask<FlushResult> WriteStreamSuffixAsync()
         {
             return WriteAsync(_endChunkedResponseBytes.Span);
         }
 
-        public Task FlushAsync(CancellationToken cancellationToken = default)
+        public ValueTask<FlushResult> FlushAsync(CancellationToken cancellationToken = default)
         {
             return WriteAsync(Constants.EmptyData, cancellationToken);
         }
 
-        public Task WriteChunkAsync(ReadOnlySpan<byte> buffer, CancellationToken cancellationToken)
+        public Memory<byte> GetMemory(int sizeHint = 0)
         {
             lock (_contextLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return GetFakeMemory(sizeHint);
                 }
+                else if (_autoChunk)
+                {
+                    return GetChunkedMemory(sizeHint);
+                }
+                else
+                {
+                    return _pipeWriter.GetMemory(sizeHint);
+                }
+            }
+        }
+
+        public Span<byte> GetSpan(int sizeHint = 0)
+        {
+            lock (_contextLock)
+            {
+                if (_completed)
+                {
+                    return GetFakeMemory(sizeHint).Span;
+                }
+                else if (_autoChunk)
+                {
+                    return GetChunkedMemory(sizeHint).Span;
+                }
+                else
+                {
+                    return _pipeWriter.GetMemory(sizeHint).Span;
+                }
+            }
+        }
+
+        public void Advance(int bytes)
+        {
+            lock (_contextLock)
+            {
+                if (_completed)
+                {
+                    return;
+                }
+
+                if (_autoChunk)
+                {
+                    if (bytes < 0)
+                    {
+                        throw new ArgumentOutOfRangeException(nameof(bytes));
+                    }
+
+                    if (bytes + _advancedBytesForChunk > _currentChunkMemory.Length - BeginChunkLengthMax - EndChunkLength)
+                    {
+                        throw new InvalidOperationException("Can't advance past buffer size.");
+                    }
+                    _advancedBytesForChunk += bytes;
+                }
+                else
+                {
+                    _pipeWriter.Advance(bytes);
+                }
+            }
+        }
+
+        public void CancelPendingFlush()
+        {
+            _pipeWriter.CancelPendingFlush();
+        }
+
+        // This method is for chunked http responses that directly call response.WriteAsync
+        public ValueTask<FlushResult> WriteChunkAsync(ReadOnlySpan<byte> buffer, CancellationToken cancellationToken)
+        {
+            lock (_contextLock)
+            {
+                if (_pipeWriterCompleted)
+                {
+                    return default;
+                }
+
+                // Make sure any memory used with GetMemory/Advance is written before the chunk
+                // passed in.
+                WriteCurrentMemoryToPipeWriter();
 
                 if (buffer.Length > 0)
                 {
@@ -96,11 +201,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             return FlushAsync(cancellationToken);
         }
 
-        public void WriteResponseHeaders(int statusCode, string reasonPhrase, HttpResponseHeaders responseHeaders)
+        public void WriteResponseHeaders(int statusCode, string reasonPhrase, HttpResponseHeaders responseHeaders, bool autoChunk)
         {
             lock (_contextLock)
             {
-                if (_completed)
+                if (_pipeWriterCompleted)
                 {
                     return;
                 }
@@ -117,6 +222,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 writer.Commit();
 
                 _unflushedBytes += writer.BytesCommitted;
+                _autoChunk = autoChunk;
             }
         }
 
@@ -124,12 +230,22 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         {
             lock (_contextLock)
             {
-                if (_completed)
+                if (_fakeMemoryOwner != null)
                 {
-                    return;
+                    _fakeMemoryOwner.Dispose();
+                    _fakeMemoryOwner = null;
                 }
 
+                CompletePipe();
+            }
+        }
+
+        private void CompletePipe()
+        {
+            if (!_pipeWriterCompleted)
+            {
                 _log.ConnectionDisconnect(_connectionId);
+                _pipeWriterCompleted = true;
                 _completed = true;
                 _pipeWriter.Complete();
             }
@@ -139,7 +255,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         {
             // Abort can be called after Dispose if there's a flush timeout.
             // It's important to still call _lifetimeFeature.Abort() in this case.
-
             lock (_contextLock)
             {
                 if (_aborted)
@@ -149,24 +264,40 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
                 _aborted = true;
                 _connectionContext.Abort(error);
-                Dispose();
+
+                CompletePipe();
             }
         }
 
-        public Task Write100ContinueAsync()
+        public void Complete()
+        {
+            lock (_contextLock)
+            {
+                _completed = true;
+            }
+        }
+
+        public ValueTask<FlushResult> Write100ContinueAsync()
         {
             return WriteAsync(_continueBytes.Span);
         }
 
-        private Task WriteAsync(
+        private ValueTask<FlushResult> WriteAsync(
             ReadOnlySpan<byte> buffer,
             CancellationToken cancellationToken = default)
         {
             lock (_contextLock)
             {
-                if (_completed)
+                if (_pipeWriterCompleted)
                 {
-                    return Task.CompletedTask;
+                    return default;
+                }
+
+                if (_autoChunk)
+                {
+                    // If there is data that was chunked before writing (ex someone did GetMemory->Advance->WriteAsync)
+                    // make sure to write whatever was advanced first
+                    WriteCurrentMemoryToPipeWriter();
                 }
 
                 var writer = new BufferWriter<PipeWriter>(_pipeWriter);
@@ -188,5 +319,77 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                     cancellationToken);
             }
         }
+
+        // These methods are for chunked http responses that use GetMemory/Advance
+        private Memory<byte> GetChunkedMemory(int sizeHint)
+        {
+            // The max size of a chunk will be the size of memory returned from the PipeWriter (today 4096)
+            // minus 5 for the max chunked prefix size and minus 2 for the chunked ending, leaving a total of
+            // 4089.
+
+            if (!_currentChunkMemoryUpdated)
+            {
+                // First time calling GetMemory
+                _currentChunkMemory = _pipeWriter.GetMemory(sizeHint);
+                _currentChunkMemoryUpdated = true;
+            }
+
+            var memoryMaxLength = _currentChunkMemory.Length - BeginChunkLengthMax - EndChunkLength;
+            if (_advancedBytesForChunk >= memoryMaxLength - Math.Min(MemorySizeThreshold, sizeHint))
+            {
+                // Chunk is completely written, commit it to the pipe so GetMemory will return a new chunk of memory.
+                WriteCurrentMemoryToPipeWriter();
+                _currentChunkMemory = _pipeWriter.GetMemory(sizeHint);
+                _currentChunkMemoryUpdated = true;
+            }
+
+            var actualMemory = _currentChunkMemory.Slice(
+                BeginChunkLengthMax + _advancedBytesForChunk,
+                memoryMaxLength - _advancedBytesForChunk);
+
+            Debug.Assert(actualMemory.Length <= 4089);
+
+            return actualMemory;
+        }
+
+        private void WriteCurrentMemoryToPipeWriter()
+        {
+            var writer = new BufferWriter<PipeWriter>(_pipeWriter);
+
+            Debug.Assert(_advancedBytesForChunk <= _currentChunkMemory.Length);
+
+            if (_advancedBytesForChunk > 0)
+            {
+                var bytesWritten = writer.WriteBeginChunkBytes(_advancedBytesForChunk);
+
+                Debug.Assert(bytesWritten <= BeginChunkLengthMax);
+
+                if (bytesWritten < BeginChunkLengthMax)
+                {
+                    // If the current chunk of memory isn't completely utilized, we need to copy the contents forwards.
+                    // This occurs if someone uses less than 255 bytes of the current Memory segment.
+                    // Therefore, we need to copy it forwards by either 1 or 2 bytes (depending on number of bytes)
+                    _currentChunkMemory.Slice(BeginChunkLengthMax, _advancedBytesForChunk).CopyTo(_currentChunkMemory.Slice(bytesWritten));
+                }
+
+                writer.Advance(_advancedBytesForChunk);
+                writer.WriteEndChunkBytes();
+                writer.Commit();
+                _advancedBytesForChunk = 0;
+                _unflushedBytes += writer.BytesCommitted;
+            }
+
+            // If there is an empty write, we still need to update the current chunk
+            _currentChunkMemoryUpdated = false;
+        }
+
+        private Memory<byte> GetFakeMemory(int sizeHint)
+        {
+            if (_fakeMemoryOwner == null)
+            {
+                _fakeMemoryOwner = _memoryPool.Rent(sizeHint);
+            }
+            return _fakeMemoryOwner.Memory;
+        }
     }
 }

src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs

@@ -2,7 +2,10 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
+using System.IO.Pipelines;
 using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
@@ -10,6 +13,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
+using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 {
@@ -21,7 +25,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                                         IHttpRequestIdentifierFeature,
                                         IHttpBodyControlFeature,
                                         IHttpMaxRequestBodySizeFeature,
-                                        IHttpResponseStartFeature
+                                        IHttpResponseStartFeature,
+                                        IResponseBodyPipeFeature
     {
         // NOTE: When feature interfaces are added to or removed from this HttpProtocol class implementation,
         // then the list of `implementedFeatures` in the generated code project MUST also be updated.
@@ -111,12 +116,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             set => ResponseHeaders = value;
         }
 
-        Stream IHttpResponseFeature.Body
-        {
-            get => ResponseBody;
-            set => ResponseBody = value;
-        }
-
         CancellationToken IHttpRequestLifetimeFeature.RequestAborted
         {
             get => RequestAborted;
@@ -193,6 +192,40 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             }
         }
 
+        PipeWriter IResponseBodyPipeFeature.ResponseBodyPipe
+        {
+            get
+            {
+                return ResponsePipeWriter;
+            }
+            set
+            {
+                ResponsePipeWriter = value;
+                ResponseBody = new WriteOnlyPipeStream(ResponsePipeWriter);
+            }
+        }
+
+        Stream IHttpResponseFeature.Body
+        {
+            get
+            {
+                return ResponseBody;
+            }
+            set
+            {
+                ResponseBody = value;
+                var responsePipeWriter = new StreamPipeWriter(ResponseBody, minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize, _context.MemoryPool);
+                ResponsePipeWriter = responsePipeWriter;
+
+                // The StreamPipeWrapper needs to be disposed as it hold onto blocks of memory
+                if (_wrapperObjectsToDispose == null)
+                {
+                    _wrapperObjectsToDispose = new List<IDisposable>();
+                }
+                _wrapperObjectsToDispose.Add(responsePipeWriter);
+            }
+        }
+
         protected void ResetHttp1Features()
         {
             _currentIHttpMinRequestBodyDataRateFeature = this;
@@ -250,7 +283,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             ApplicationAbort();
         }
 
-        protected abstract void ApplicationAbort();
 
         Task IHttpResponseStartFeature.StartAsync(CancellationToken cancellationToken)
         {
@@ -259,6 +291,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 return Task.CompletedTask;
             }
 
+            cancellationToken.ThrowIfCancellationRequested();
+
             return InitializeResponseAsync(0);
         }
     }

src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.Generated.cs

@@ -15,6 +15,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
     {
         private static readonly Type IHttpRequestFeatureType = typeof(IHttpRequestFeature);
         private static readonly Type IHttpResponseFeatureType = typeof(IHttpResponseFeature);
+        private static readonly Type IResponseBodyPipeFeatureType = typeof(IResponseBodyPipeFeature);
         private static readonly Type IHttpRequestIdentifierFeatureType = typeof(IHttpRequestIdentifierFeature);
         private static readonly Type IServiceProvidersFeatureType = typeof(IServiceProvidersFeature);
         private static readonly Type IHttpRequestLifetimeFeatureType = typeof(IHttpRequestLifetimeFeature);
@@ -39,6 +40,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
         private object _currentIHttpRequestFeature;
         private object _currentIHttpResponseFeature;
+        private object _currentIResponseBodyPipeFeature;
         private object _currentIHttpRequestIdentifierFeature;
         private object _currentIServiceProvidersFeature;
         private object _currentIHttpRequestLifetimeFeature;
@@ -69,6 +71,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         {
             _currentIHttpRequestFeature = this;
             _currentIHttpResponseFeature = this;
+            _currentIResponseBodyPipeFeature = this;
             _currentIHttpUpgradeFeature = this;
             _currentIHttpRequestIdentifierFeature = this;
             _currentIHttpRequestLifetimeFeature = this;
@@ -153,6 +156,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 {
                     feature = _currentIHttpResponseFeature;
                 }
+                else if (key == IResponseBodyPipeFeatureType)
+                {
+                    feature = _currentIResponseBodyPipeFeature;
+                }
                 else if (key == IHttpRequestIdentifierFeatureType)
                 {
                     feature = _currentIHttpRequestIdentifierFeature;
@@ -257,6 +264,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 {
                     _currentIHttpResponseFeature = value;
                 }
+                else if (key == IResponseBodyPipeFeatureType)
+                {
+                    _currentIResponseBodyPipeFeature = value;
+                }
                 else if (key == IHttpRequestIdentifierFeatureType)
                 {
                     _currentIHttpRequestIdentifierFeature = value;
@@ -359,6 +370,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 feature = (TFeature)_currentIHttpResponseFeature;
             }
+            else if (typeof(TFeature) == typeof(IResponseBodyPipeFeature))
+            {
+                feature = (TFeature)_currentIResponseBodyPipeFeature;
+            }
             else if (typeof(TFeature) == typeof(IHttpRequestIdentifierFeature))
             {
                 feature = (TFeature)_currentIHttpRequestIdentifierFeature;
@@ -467,6 +482,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 _currentIHttpResponseFeature = feature;
             }
+            else if (typeof(TFeature) == typeof(IResponseBodyPipeFeature))
+            {
+                _currentIResponseBodyPipeFeature = feature;
+            }
             else if (typeof(TFeature) == typeof(IHttpRequestIdentifierFeature))
             {
                 _currentIHttpRequestIdentifierFeature = feature;
@@ -567,6 +586,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 yield return new KeyValuePair<Type, object>(IHttpResponseFeatureType, _currentIHttpResponseFeature);
             }
+            if (_currentIResponseBodyPipeFeature != null)
+            {
+                yield return new KeyValuePair<Type, object>(IResponseBodyPipeFeatureType, _currentIResponseBodyPipeFeature);
+            }
             if (_currentIHttpRequestIdentifierFeature != null)
             {
                 yield return new KeyValuePair<Type, object>(IHttpRequestIdentifierFeatureType, _currentIHttpRequestIdentifierFeature);

src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Buffers;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -31,7 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: " + Constants.ServerName);
 
         protected Streams _streams;
-
+        private HttpResponsePipeWriter _originalPipeWriter;
         private Stack<KeyValuePair<Func<object, Task>, object>> _onStarting;
         private Stack<KeyValuePair<Func<object, Task>, object>> _onCompleted;
 
@@ -64,6 +63,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         protected string _methodText = null;
         private string _scheme = null;
 
+        private List<IDisposable> _wrapperObjectsToDispose;
+
         public HttpProtocol(HttpConnectionContext context)
         {
             _context = context;
@@ -227,6 +228,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
         public IHeaderDictionary ResponseHeaders { get; set; }
         public Stream ResponseBody { get; set; }
+        public PipeWriter ResponsePipeWriter { get; set; }
 
         public CancellationToken RequestAborted
         {
@@ -295,10 +297,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         {
             if (_streams == null)
             {
-                _streams = new Streams(bodyControl: this, httpResponseControl: this);
+                var pipeWriter = new HttpResponsePipeWriter(this);
+                _streams = new Streams(bodyControl: this, pipeWriter);
+                _originalPipeWriter = pipeWriter;
             }
 
             (RequestBody, ResponseBody) = _streams.Start(messageBody);
+            ResponsePipeWriter = _originalPipeWriter;
         }
 
         public void StopStreams() => _streams.Stop();
@@ -371,6 +376,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 }
             }
 
+            if (_wrapperObjectsToDispose != null)
+            {
+                foreach (var disposable in _wrapperObjectsToDispose)
+                {
+                    disposable.Dispose();
+                }
+            }
+
             _requestHeadersParsed = 0;
 
             _responseBytesWritten = 0;
@@ -382,6 +395,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
         protected abstract void OnReset();
 
+        protected abstract void ApplicationAbort();
+
         protected virtual void OnRequestProcessingEnding()
         {
         }
@@ -780,21 +795,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
         }
 
-        public Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken))
-        {
-            if (!HasResponseStarted)
-            {
-                var initializeTask = InitializeResponseAsync(0);
-                // If return is Task.CompletedTask no awaiting is required
-                if (!ReferenceEquals(initializeTask, Task.CompletedTask))
-                {
-                    return InitializeAndFlushAsyncAwaited(initializeTask, cancellationToken);
-                }
-            }
-
-            return FlushAsyncInternal(cancellationToken);
-        }
-
         [MethodImpl(MethodImplOptions.NoInlining)]
         private async Task InitializeAndFlushAsyncAwaited(Task initializeTask, CancellationToken cancellationToken)
         {
@@ -803,85 +803,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         }
 
         [MethodImpl(MethodImplOptions.NoInlining)]
-        private Task FlushAsyncInternal(CancellationToken cancellationToken)
+        private ValueTask<FlushResult> FlushAsyncInternal(CancellationToken cancellationToken)
         {
             _requestProcessingStatus = RequestProcessingStatus.HeadersFlushed;
             return Output.FlushAsync(cancellationToken);
         }
 
-        public Task WriteAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken = default(CancellationToken))
-        {
-            // For the first write, ensure headers are flushed if WriteDataAsync isn't called.
-            var firstWrite = !HasResponseStarted;
-
-            if (firstWrite)
-            {
-                var initializeTask = InitializeResponseAsync(data.Length);
-                // If return is Task.CompletedTask no awaiting is required
-                if (!ReferenceEquals(initializeTask, Task.CompletedTask))
-                {
-                    return WriteAsyncAwaited(initializeTask, data, cancellationToken);
-                }
-            }
-            else
-            {
-                VerifyAndUpdateWrite(data.Length);
-            }
-
-            if (_canWriteResponseBody)
-            {
-                if (_autoChunk)
-                {
-                    if (data.Length == 0)
-                    {
-                        return !firstWrite ? Task.CompletedTask : FlushAsyncInternal(cancellationToken);
-                    }
-                    return WriteChunkedAsync(data.Span, cancellationToken);
-                }
-                else
-                {
-                    CheckLastWrite();
-                    return WriteDataAsync(data, cancellationToken: cancellationToken);
-                }
-            }
-            else
-            {
-                HandleNonBodyResponseWrite();
-                return !firstWrite ? Task.CompletedTask : FlushAsyncInternal(cancellationToken);
-            }
-        }
-
-        public async Task WriteAsyncAwaited(Task initializeTask, ReadOnlyMemory<byte> data, CancellationToken cancellationToken)
-        {
-            await initializeTask;
-
-            // WriteAsyncAwaited is only called for the first write to the body.
-            // Ensure headers are flushed if Write(Chunked)Async isn't called.
-            if (_canWriteResponseBody)
-            {
-                if (_autoChunk)
-                {
-                    if (data.Length == 0)
-                    {
-                        await FlushAsyncInternal(cancellationToken);
-                        return;
-                    }
-
-                    await WriteChunkedAsync(data.Span, cancellationToken);
-                }
-                else
-                {
-                    CheckLastWrite();
-                    await WriteDataAsync(data, cancellationToken);
-                }
-            }
-            else
-            {
-                HandleNonBodyResponseWrite();
-                await FlushAsyncInternal(cancellationToken);
-            }
-        }
-
         private Task WriteDataAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken)
         {
             _requestProcessingStatus = RequestProcessingStatus.HeadersFlushed;
@@ -958,12 +885,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             }
         }
 
-        private Task WriteChunkedAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken)
-        {
-            _requestProcessingStatus = RequestProcessingStatus.HeadersFlushed;
-            return Output.WriteChunkAsync(data, cancellationToken);
-        }
-
         public void ProduceContinue()
         {
             if (HasResponseStarted)
@@ -1090,7 +1011,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
             if (!HasFlushedHeaders)
             {
-                return FlushAsyncInternal(default);
+                return FlushAsyncInternal(default).AsTask();
             }
 
             return Task.CompletedTask;
@@ -1208,7 +1129,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 responseHeaders.SetRawDate(dateHeaderValues.String, dateHeaderValues.Bytes);
             }
 
-            Output.WriteResponseHeaders(StatusCode, ReasonPhrase, responseHeaders);
+            Output.WriteResponseHeaders(StatusCode, ReasonPhrase, responseHeaders, _autoChunk);
         }
 
         private bool CanWriteResponseBody()
@@ -1345,5 +1266,172 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
             Log.ApplicationError(ConnectionId, TraceIdentifier, ex);
         }
+
+        public void Advance(int bytes)
+        {
+            if (_canWriteResponseBody)
+            {
+                VerifyAndUpdateWrite(bytes);
+                Output.Advance(bytes);
+            }
+            else
+            {
+                HandleNonBodyResponseWrite();
+
+                // For HEAD requests, we still use the number of bytes written for logging
+                // how many bytes were written. 
+                VerifyAndUpdateWrite(bytes);
+            }
+        }
+
+        public Memory<byte> GetMemory(int sizeHint = 0)
+        {
+            ThrowIfResponseNotStarted();
+
+            return Output.GetMemory(sizeHint);
+        }
+
+        public Span<byte> GetSpan(int sizeHint = 0)
+        {
+            ThrowIfResponseNotStarted();
+
+            return Output.GetSpan(sizeHint);
+        }
+
+        [StackTraceHidden]
+        private void ThrowIfResponseNotStarted()
+        {
+            if (!HasResponseStarted)
+            {
+                throw new InvalidOperationException(CoreStrings.StartAsyncBeforeGetMemory);
+            }
+        }
+
+        public ValueTask<FlushResult> FlushPipeAsync(CancellationToken cancellationToken)
+        {
+            if (!HasResponseStarted)
+            {
+                var initializeTask = InitializeResponseAsync(0);
+                // If return is Task.CompletedTask no awaiting is required
+                if (!ReferenceEquals(initializeTask, Task.CompletedTask))
+                {
+                    return FlushAsyncAwaited(initializeTask, cancellationToken);
+                }
+            }
+
+            return Output.FlushAsync(cancellationToken);
+        }
+
+        public void CancelPendingFlush()
+        {
+            Output.CancelPendingFlush();
+        }
+
+        public void Complete(Exception ex)
+        {
+            if (ex != null)
+            {
+                var wrappedException = new ConnectionAbortedException("The BodyPipe was completed with an exception.", ex);
+                ReportApplicationError(wrappedException);
+
+                if (HasResponseStarted)
+                {
+                    ApplicationAbort();
+                }
+            }
+            Output.Complete();
+        }
+
+        public ValueTask<FlushResult> WritePipeAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken)
+        {
+            // For the first write, ensure headers are flushed if WriteDataAsync isn't called.
+            var firstWrite = !HasResponseStarted;
+
+            if (firstWrite)
+            {
+                var initializeTask = InitializeResponseAsync(data.Length);
+                // If return is Task.CompletedTask no awaiting is required
+                if (!ReferenceEquals(initializeTask, Task.CompletedTask))
+                {
+                    return WriteAsyncAwaited(initializeTask, data, cancellationToken);
+                }
+            }
+            else
+            {
+                VerifyAndUpdateWrite(data.Length);
+            }
+
+            if (_canWriteResponseBody)
+            {
+                if (_autoChunk)
+                {
+                    if (data.Length == 0)
+                    {
+                        return !firstWrite ? default : FlushAsyncInternal(cancellationToken);
+                    }
+
+                    _requestProcessingStatus = RequestProcessingStatus.HeadersFlushed;
+                    return Output.WriteChunkAsync(data.Span, cancellationToken);
+                }
+                else
+                {
+                    CheckLastWrite();
+                    return Output.WriteDataToPipeAsync(data.Span, cancellationToken: cancellationToken);
+                }
+            }
+            else
+            {
+                HandleNonBodyResponseWrite();
+                return !firstWrite ? default : FlushAsyncInternal(cancellationToken);
+            }
+        }
+
+        public Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken))
+        {
+            return FlushPipeAsync(cancellationToken).AsTask();
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private async ValueTask<FlushResult> FlushAsyncAwaited(Task initializeTask, CancellationToken cancellationToken)
+        {
+            await initializeTask;
+            return await Output.FlushAsync(cancellationToken);
+        }
+
+        public Task WriteAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken = default(CancellationToken))
+        {
+            return WritePipeAsync(data, cancellationToken).AsTask();
+        }
+
+        public async ValueTask<FlushResult> WriteAsyncAwaited(Task initializeTask, ReadOnlyMemory<byte> data, CancellationToken cancellationToken)
+        {
+            await initializeTask;
+
+            // WriteAsyncAwaited is only called for the first write to the body.
+            // Ensure headers are flushed if Write(Chunked)Async isn't called.
+            if (_canWriteResponseBody)
+            {
+                if (_autoChunk)
+                {
+                    if (data.Length == 0)
+                    {
+                        return await FlushAsyncInternal(cancellationToken);
+                    }
+
+                    _requestProcessingStatus = RequestProcessingStatus.HeadersFlushed;
+                    return await Output.WriteChunkAsync(data.Span, cancellationToken);
+                }
+                else
+                {
+                    CheckLastWrite();
+                    return await Output.WriteDataToPipeAsync(data.Span, cancellationToken: cancellationToken);
+                }
+            }
+            else
+            {
+                HandleNonBodyResponseWrite();
+                return await FlushAsyncInternal(cancellationToken);
+            }
+        }
     }
 }

src/Servers/Kestrel/Core/src/Internal/Http/HttpResponsePipeWriter.cs

@@ -0,0 +1,114 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO.Pipelines;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
+{
+    public class HttpResponsePipeWriter : PipeWriter
+    {
+        private HttpStreamState _state;
+        private readonly IHttpResponseControl _pipeControl;
+
+        public HttpResponsePipeWriter(IHttpResponseControl pipeControl)
+        {
+            _pipeControl = pipeControl;
+            _state = HttpStreamState.Closed;
+        }
+
+        public override void Advance(int bytes)
+        {
+            ValidateState();
+            _pipeControl.Advance(bytes);
+        }
+
+        public override void CancelPendingFlush()
+        {
+            ValidateState();
+            _pipeControl.CancelPendingFlush();
+        }
+
+        public override void Complete(Exception exception = null)
+        {
+            _pipeControl.Complete(exception);
+        }
+
+        public override ValueTask<FlushResult> FlushAsync(CancellationToken cancellationToken = default)
+        {
+            ValidateState(cancellationToken);
+            return _pipeControl.FlushPipeAsync(cancellationToken);
+        }
+
+        public override Memory<byte> GetMemory(int sizeHint = 0)
+        {
+            ValidateState();
+            return _pipeControl.GetMemory(sizeHint);
+        }
+
+        public override Span<byte> GetSpan(int sizeHint = 0)
+        {
+            ValidateState();
+            return _pipeControl.GetSpan(sizeHint);
+        }
+
+        public override void OnReaderCompleted(Action<Exception, object> callback, object state)
+        {
+            throw new NotSupportedException();
+        }
+
+        public override ValueTask<FlushResult> WriteAsync(ReadOnlyMemory<byte> source, CancellationToken cancellationToken = default)
+        {
+            ValidateState(cancellationToken);
+            return _pipeControl.WritePipeAsync(source, cancellationToken);
+        }
+
+        public void StartAcceptingWrites()
+        {
+            // Only start if not aborted
+            if (_state == HttpStreamState.Closed)
+            {
+                _state = HttpStreamState.Open;
+            }
+        }
+
+        public void StopAcceptingWrites()
+        {
+            // Can't use dispose (or close) as can be disposed too early by user code
+            // As exampled in EngineTests.ZeroContentLengthNotSetAutomaticallyForCertainStatusCodes
+            _state = HttpStreamState.Closed;
+        }
+
+        public void Abort()
+        {
+            // We don't want to throw an ODE until the app func actually completes.
+            if (_state != HttpStreamState.Closed)
+            {
+                _state = HttpStreamState.Aborted;
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void ValidateState(CancellationToken cancellationToken = default)
+        {
+            var state = _state;
+            if (state == HttpStreamState.Open || state == HttpStreamState.Aborted)
+            {
+                // Aborted state only throws on write if cancellationToken requests it
+                cancellationToken.ThrowIfCancellationRequested();
+            }
+            else
+            {
+                ThrowObjectDisposedException();
+            }
+
+            void ThrowObjectDisposedException()
+            {
+                throw new ObjectDisposedException(nameof(HttpResponseStream), CoreStrings.WritingToResponseBodyAfterResponseCompleted);
+            }
+        }
+    }
+}

src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseStream.cs

@@ -2,59 +2,21 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.IO;
-using System.Runtime.CompilerServices;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
+using System.IO.Pipelines;
 using Microsoft.AspNetCore.Http.Features;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 {
-    internal class HttpResponseStream : WriteOnlyStream
+    internal class HttpResponseStream : WriteOnlyPipeStream
     {
+        private readonly HttpResponsePipeWriter _pipeWriter;
         private readonly IHttpBodyControlFeature _bodyControl;
-        private readonly IHttpResponseControl _httpResponseControl;
-        private HttpStreamState _state;
 
-        public HttpResponseStream(IHttpBodyControlFeature bodyControl, IHttpResponseControl httpResponseControl)
+        public HttpResponseStream(IHttpBodyControlFeature bodyControl, HttpResponsePipeWriter pipeWriter)
+            : base(pipeWriter)
         {
             _bodyControl = bodyControl;
-            _httpResponseControl = httpResponseControl;
-            _state = HttpStreamState.Closed;
-        }
-
-        public override bool CanSeek => false;
-
-        public override long Length
-            => throw new NotSupportedException();
-
-        public override long Position
-        {
-            get => throw new NotSupportedException();
-            set => throw new NotSupportedException();
-        }
-
-        public override void Flush()
-        {
-            FlushAsync(default).GetAwaiter().GetResult();
-        }
-
-        public override Task FlushAsync(CancellationToken cancellationToken)
-        {
-            ValidateState(cancellationToken);
-
-            return _httpResponseControl.FlushAsync(cancellationToken);
-        }
-
-        public override long Seek(long offset, SeekOrigin origin)
-        {
-            throw new NotSupportedException();
-        }
-
-        public override void SetLength(long value)
-        {
-            throw new NotSupportedException();
+            _pipeWriter = pipeWriter;
         }
 
         public override void Write(byte[] buffer, int offset, int count)
@@ -64,104 +26,32 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 throw new InvalidOperationException(CoreStrings.SynchronousWritesDisallowed);
             }
 
-            WriteAsync(buffer, offset, count, default).GetAwaiter().GetResult();
+            base.Write(buffer, offset, count);
         }
 
-        public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object state)
+        public override void Flush()
         {
-            var task = WriteAsync(buffer, offset, count, default, state);
-            if (callback != null)
+            if (!_bodyControl.AllowSynchronousIO)
             {
-                task.ContinueWith(t => callback.Invoke(t));
+                throw new InvalidOperationException(CoreStrings.SynchronousWritesDisallowed);
             }
-            return task;
-        }
 
-        public override void EndWrite(IAsyncResult asyncResult)
-        {
-            ((Task<object>)asyncResult).GetAwaiter().GetResult();
-        }
-
-        private Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken, object state)
-        {
-            var tcs = new TaskCompletionSource<object>(state);
-            var task = WriteAsync(buffer, offset, count, cancellationToken);
-            task.ContinueWith((task2, state2) =>
-            {
-                var tcs2 = (TaskCompletionSource<object>)state2;
-                if (task2.IsCanceled)
-                {
-                    tcs2.SetCanceled();
-                }
-                else if (task2.IsFaulted)
-                {
-                    tcs2.SetException(task2.Exception);
-                }
-                else
-                {
-                    tcs2.SetResult(null);
-                }
-            }, tcs, cancellationToken);
-            return tcs.Task;
-        }
-
-        public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
-        {
-            ValidateState(cancellationToken);
-
-            return _httpResponseControl.WriteAsync(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken);
-        }
-
-        public override ValueTask WriteAsync(ReadOnlyMemory<byte> source, CancellationToken cancellationToken = default)
-        {
-            ValidateState(cancellationToken);
-
-            return new ValueTask(_httpResponseControl.WriteAsync(source, cancellationToken));
+            base.Flush();
         }
 
         public void StartAcceptingWrites()
         {
-            // Only start if not aborted
-            if (_state == HttpStreamState.Closed)
-            {
-                _state = HttpStreamState.Open;
-            }
+            _pipeWriter.StartAcceptingWrites();
         }
 
         public void StopAcceptingWrites()
         {
-            // Can't use dispose (or close) as can be disposed too early by user code
-            // As exampled in EngineTests.ZeroContentLengthNotSetAutomaticallyForCertainStatusCodes
-            _state = HttpStreamState.Closed;
+            _pipeWriter.StopAcceptingWrites();
         }
 
         public void Abort()
         {
-            // We don't want to throw an ODE until the app func actually completes.
-            if (_state != HttpStreamState.Closed)
-            {
-                _state = HttpStreamState.Aborted;
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private void ValidateState(CancellationToken cancellationToken)
-        {
-            var state = _state;
-            if (state == HttpStreamState.Open || state == HttpStreamState.Aborted)
-            {
-                // Aborted state only throws on write if cancellationToken requests it
-                cancellationToken.ThrowIfCancellationRequested();
-            }
-            else
-            {
-                ThrowObjectDisposedException();
-            }
-
-            void ThrowObjectDisposedException()
-            {
-                throw new ObjectDisposedException(nameof(HttpResponseStream), CoreStrings.WritingToResponseBodyAfterResponseCompleted);
-            }
+            _pipeWriter.Abort();
         }
     }
 }

src/Servers/Kestrel/Core/src/Internal/Http/IHttpOutputAborter.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using Microsoft.AspNetCore.Connections;

src/Servers/Kestrel/Core/src/Internal/Http/IHttpOutputProducer.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.IO.Pipelines;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -9,12 +10,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 {
     public interface IHttpOutputProducer
     {
-        Task WriteChunkAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken);
-        Task FlushAsync(CancellationToken cancellationToken);
-        Task Write100ContinueAsync();
-        void WriteResponseHeaders(int statusCode, string ReasonPhrase, HttpResponseHeaders responseHeaders);
+        ValueTask<FlushResult> WriteChunkAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken);
+        ValueTask<FlushResult> FlushAsync(CancellationToken cancellationToken);
+        ValueTask<FlushResult> Write100ContinueAsync();
+        void WriteResponseHeaders(int statusCode, string ReasonPhrase, HttpResponseHeaders responseHeaders, bool autoChunk);
         // This takes ReadOnlySpan instead of ReadOnlyMemory because it always synchronously copies data before flushing.
+        ValueTask<FlushResult> WriteDataToPipeAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken);
         Task WriteDataAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken);
-        Task WriteStreamSuffixAsync();
+        ValueTask<FlushResult> WriteStreamSuffixAsync();
+        void Advance(int bytes);
+        Span<byte> GetSpan(int sizeHint = 0);
+        Memory<byte> GetMemory(int sizeHint = 0);
+        void CancelPendingFlush();
+        void Complete();
     }
 }

src/Servers/Kestrel/Core/src/Internal/Http/IHttpResponseControl.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.IO.Pipelines;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -10,7 +11,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
     public interface IHttpResponseControl
     {
         void ProduceContinue();
-        Task WriteAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken);
-        Task FlushAsync(CancellationToken cancellationToken);
+        Memory<byte> GetMemory(int sizeHint = 0);
+        Span<byte> GetSpan(int sizeHint = 0);
+        void Advance(int bytes);
+        ValueTask<FlushResult> FlushPipeAsync(CancellationToken cancellationToken);
+        ValueTask<FlushResult> WritePipeAsync(ReadOnlyMemory<byte> source, CancellationToken cancellationToken);
+        void CancelPendingFlush();
+        void Complete(Exception exception = null);
     }
 }

src/Servers/Kestrel/Core/src/Internal/Http/IHttpResponsePipeWriterControl.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO.Pipelines;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
+{
+    public interface IHttpResponsePipeWriterControl
+    {
+        void ProduceContinue();
+        Memory<byte> GetMemory(int sizeHint = 0);
+        Span<byte> GetSpan(int sizeHint = 0);
+        void Advance(int bytes);
+        ValueTask<FlushResult> FlushPipeAsync(CancellationToken cancellationToken);
+        ValueTask<FlushResult> WritePipeAsync(ReadOnlyMemory<byte> source, CancellationToken cancellationToken);
+        void CancelPendingFlush();
+    }
+}

src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs

@@ -705,7 +705,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                     }
                 }
 
-                return ackTask;
+                return ackTask.AsTask();
             }
             catch (Http2SettingsParameterOutOfRangeException ex)
             {
@@ -738,7 +738,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                 return Task.CompletedTask;
             }
 
-            return _frameWriter.WritePingAsync(Http2PingFrameFlags.ACK, payload);
+            return _frameWriter.WritePingAsync(Http2PingFrameFlags.ACK, payload).AsTask();
         }
 
         private Task ProcessGoAwayFrameAsync()

src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs

@@ -109,13 +109,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        public Task FlushAsync(IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
+        public ValueTask<FlushResult> FlushAsync(IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
                 
                 var bytesWritten = _unflushedBytes;
@@ -125,13 +125,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        public Task Write100ContinueAsync(int streamId)
+        public ValueTask<FlushResult> Write100ContinueAsync(int streamId)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PrepareHeaders(Http2HeadersFrameFlags.END_HEADERS, streamId);
@@ -181,13 +181,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        public Task WriteResponseTrailers(int streamId, HttpResponseTrailers headers)
+        public ValueTask<FlushResult> WriteResponseTrailers(int streamId, HttpResponseTrailers headers)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 try
@@ -236,7 +236,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        public Task WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence<byte> data, bool endStream)
+        public ValueTask<FlushResult> WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence<byte> data, bool endStream)
         {
             // The Length property of a ReadOnlySequence can be expensive, so we cache the value.
             var dataLength = data.Length;
@@ -245,7 +245,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             {
                 if (_completed || flowControl.IsAborted)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 // Zero-length data frames are allowed to be sent immediately even if there is no space available in the flow control window.
@@ -312,12 +312,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             // Plus padding
         }
 
-        private async Task WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence<byte> data, long dataLength, bool endStream)
+        private async ValueTask<FlushResult> WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence<byte> data, long dataLength, bool endStream)
         {
+            FlushResult flushResult = default;
+
             while (dataLength > 0)
             {
                 OutputFlowControlAwaitable availabilityAwaitable;
-                var writeTask = Task.CompletedTask;
+                var writeTask = default(ValueTask<FlushResult>);
 
                 lock (_writeLock)
                 {
@@ -373,7 +375,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                     await availabilityAwaitable;
                 }
 
-                await writeTask;
+                flushResult = await writeTask;
 
                 if (_minResponseDataRate != null)
                 {
@@ -383,6 +385,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
 
             // Ensure that the application continuation isn't executed inline by ProcessWindowUpdateFrameAsync.
             await ThreadPoolAwaitable.Instance;
+
+            return flushResult;
         }
 
         /* https://tools.ietf.org/html/rfc7540#section-6.9
@@ -390,13 +394,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             |R|              Window Size Increment (31)                     |
             +-+-------------------------------------------------------------+
         */
-        public Task WriteWindowUpdateAsync(int streamId, int sizeIncrement)
+        public ValueTask<FlushResult> WriteWindowUpdateAsync(int streamId, int sizeIncrement)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PrepareWindowUpdate(streamId, sizeIncrement);
@@ -413,13 +417,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             |                        Error Code (32)                        |
             +---------------------------------------------------------------+
         */
-        public Task WriteRstStreamAsync(int streamId, Http2ErrorCode errorCode)
+        public ValueTask<FlushResult> WriteRstStreamAsync(int streamId, Http2ErrorCode errorCode)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PrepareRstStream(streamId, errorCode);
@@ -440,13 +444,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             |                        Value (32)                             |
             +---------------------------------------------------------------+
         */
-        public Task WriteSettingsAsync(IList<Http2PeerSetting> settings)
+        public ValueTask<FlushResult> WriteSettingsAsync(IList<Http2PeerSetting> settings)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PrepareSettings(Http2SettingsFrameFlags.NONE);
@@ -473,13 +477,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
         }
 
         // No payload
-        public Task WriteSettingsAckAsync()
+        public ValueTask<FlushResult> WriteSettingsAckAsync()
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PrepareSettings(Http2SettingsFrameFlags.ACK);
@@ -495,13 +499,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             |                                                               |
             +---------------------------------------------------------------+
         */
-        public Task WritePingAsync(Http2PingFrameFlags flags, ReadOnlySequence<byte> payload)
+        public ValueTask<FlushResult> WritePingAsync(Http2PingFrameFlags flags, ReadOnlySequence<byte> payload)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PreparePing(Http2PingFrameFlags.ACK);
@@ -525,13 +529,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             |                  Additional Debug Data (*)                    | (not implemented)
             +---------------------------------------------------------------+
         */
-        public Task WriteGoAwayAsync(int lastStreamId, Http2ErrorCode errorCode)
+        public ValueTask<FlushResult> WriteGoAwayAsync(int lastStreamId, Http2ErrorCode errorCode)
         {
             lock (_writeLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _outgoingFrame.PrepareGoAway(lastStreamId, errorCode);
@@ -583,7 +587,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             output.Advance(Http2FrameReader.HeaderLength);
         }
 
-        private Task TimeFlushUnsynchronizedAsync()
+        private ValueTask<FlushResult> TimeFlushUnsynchronizedAsync()
         {
             var bytesWritten = _unflushedBytes;
             _unflushedBytes = 0;

src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs

@@ -27,7 +27,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
         private readonly Http2Stream _stream;
         private readonly object _dataWriterLock = new object();
         private readonly Pipe _dataPipe;
-        private readonly Task _dataWriteProcessingTask;
+        private readonly ValueTask<FlushResult> _dataWriteProcessingTask;
         private bool _startedWritingDataFrames;
         private bool _completed;
         private bool _disposed;
@@ -88,18 +88,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             throw new NotImplementedException();
         }
 
-        public Task FlushAsync(CancellationToken cancellationToken)
+        public ValueTask<FlushResult> FlushAsync(CancellationToken cancellationToken)
         {
             if (cancellationToken.IsCancellationRequested)
             {
-                return Task.FromCanceled(cancellationToken);
+                return new ValueTask<FlushResult>(Task.FromCanceled<FlushResult>(cancellationToken));
             }
 
             lock (_dataWriterLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 if (_startedWritingDataFrames)
@@ -117,20 +117,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        public Task Write100ContinueAsync()
+        public ValueTask<FlushResult> Write100ContinueAsync()
         {
             lock (_dataWriterLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 return _frameWriter.Write100ContinueAsync(_streamId);
             }
         }
 
-        public void WriteResponseHeaders(int statusCode, string ReasonPhrase, HttpResponseHeaders responseHeaders)
+        public void WriteResponseHeaders(int statusCode, string ReasonPhrase, HttpResponseHeaders responseHeaders, bool autoChunk)
         {
             lock (_dataWriterLock)
             {
@@ -164,17 +164,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                 _startedWritingDataFrames = true;
 
                 _dataPipe.Writer.Write(data);
-                return _flusher.FlushAsync(this, cancellationToken);
+                return _flusher.FlushAsync(this, cancellationToken).AsTask();
             }
         }
 
-        public Task WriteStreamSuffixAsync()
+        public ValueTask<FlushResult> WriteStreamSuffixAsync()
         {
             lock (_dataWriterLock)
             {
                 if (_completed)
                 {
-                    return Task.CompletedTask;
+                    return default;
                 }
 
                 _completed = true;
@@ -184,7 +184,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        public Task WriteRstStreamAsync(Http2ErrorCode error)
+        public ValueTask<FlushResult> WriteRstStreamAsync(Http2ErrorCode error)
         {
             lock (_dataWriterLock)
             {
@@ -196,8 +196,76 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
         }
 
-        private async Task ProcessDataWrites()
+        public void Advance(int bytes)
         {
+            lock (_dataWriterLock)
+            {
+                _startedWritingDataFrames = true;
+
+                _dataPipe.Writer.Advance(bytes);
+            }
+        }
+
+        public Span<byte> GetSpan(int sizeHint = 0)
+        {
+            lock (_dataWriterLock)
+            {
+                return _dataPipe.Writer.GetSpan(sizeHint);
+            }
+        }
+
+        public Memory<byte> GetMemory(int sizeHint = 0)
+        {
+            lock (_dataWriterLock)
+            {
+                return _dataPipe.Writer.GetMemory(sizeHint);
+            }
+        }
+
+        public void CancelPendingFlush()
+        {
+            lock (_dataWriterLock)
+            {
+                _dataPipe.Writer.CancelPendingFlush();
+            }
+        }
+
+        public ValueTask<FlushResult> WriteDataToPipeAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken)
+        {
+            if (cancellationToken.IsCancellationRequested)
+            {
+                return new ValueTask<FlushResult>(Task.FromCanceled<FlushResult>(cancellationToken));
+            }
+
+            lock (_dataWriterLock)
+            {
+                // This length check is important because we don't want to set _startedWritingDataFrames unless a data
+                // frame will actually be written causing the headers to be flushed.
+                if (_completed || data.Length == 0)
+                {
+                    return default;
+                }
+
+                _startedWritingDataFrames = true;
+
+                _dataPipe.Writer.Write(data);
+                return _flusher.FlushAsync(this, cancellationToken);
+            }
+        }
+
+        ValueTask<FlushResult> IHttpOutputProducer.WriteChunkAsync(ReadOnlySpan<byte> data, CancellationToken cancellationToken)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Complete()
+        {
+            // This will noop for now. See: https://github.com/aspnet/AspNetCore/issues/7370
+        }
+
+        private async ValueTask<FlushResult> ProcessDataWrites()
+        {
+            FlushResult flushResult = default;
             try
             {
                 ReadResult readResult;
@@ -210,14 +278,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                     {
                         if (readResult.Buffer.Length > 0)
                         {
-                            await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: false);
+                            flushResult = await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: false);
                         }
 
-                        await _frameWriter.WriteResponseTrailers(_streamId, _stream.Trailers);
+                        flushResult = await _frameWriter.WriteResponseTrailers(_streamId, _stream.Trailers);
                     }
                     else
                     {
-                        await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: readResult.IsCompleted);
+                        flushResult = await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: readResult.IsCompleted);
                     }
 
                     _dataPipe.Reader.AdvanceTo(readResult.Buffer.End);
@@ -233,6 +301,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             }
 
             _dataPipe.Reader.Complete();
+
+            return flushResult;
         }
 
         private static Pipe CreateDataPipe(MemoryPool<byte> pool)

src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -18,11 +18,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
         private readonly HttpRequestStream _emptyRequest;
         private readonly Stream _upgradeStream;
 
-        public Streams(IHttpBodyControlFeature bodyControl, IHttpResponseControl httpResponseControl)
+        public Streams(IHttpBodyControlFeature bodyControl, HttpResponsePipeWriter writer)
         {
             _request = new HttpRequestStream(bodyControl);
             _emptyRequest = new HttpRequestStream(bodyControl);
-            _response = new HttpResponseStream(bodyControl, httpResponseControl);
+            _response = new HttpResponseStream(bodyControl, writer);
             _upgradeableResponse = new WrappingStream(_response);
             _upgradeStream = new HttpUpgradeStream(_request, _response);
         }

src/Servers/Kestrel/Core/src/Internal/Infrastructure/TimingPipeFlusher.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
         private readonly IKestrelTrace _log;
 
         private readonly object _flushLock = new object();
-        private Task _lastFlushTask = Task.CompletedTask;
+        private Task<FlushResult> _lastFlushTask = null;
 
         public TimingPipeFlusher(
             PipeWriter writer,
@@ -34,22 +34,22 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
             _log = log;
         }
 
-        public Task FlushAsync()
+        public ValueTask<FlushResult> FlushAsync()
         {
             return FlushAsync(outputAborter: null, cancellationToken: default);
         }
 
-        public Task FlushAsync(IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
+        public ValueTask<FlushResult> FlushAsync(IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
         {
             return FlushAsync(minRate: null, count: 0, outputAborter: outputAborter, cancellationToken: cancellationToken);
         }
 
-        public Task FlushAsync(MinDataRate minRate, long count)
+        public ValueTask<FlushResult> FlushAsync(MinDataRate minRate, long count)
         {
             return FlushAsync(minRate, count, outputAborter: null, cancellationToken: default);
         }
 
-        public Task FlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
+        public ValueTask<FlushResult> FlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
         {
             var flushValueTask = _writer.FlushAsync(cancellationToken);
 
@@ -60,7 +60,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
 
             if (flushValueTask.IsCompletedSuccessfully)
             {
-                return Task.CompletedTask;
+                return new ValueTask<FlushResult>(flushValueTask.Result);
             }
 
             // https://github.com/dotnet/corefxlab/issues/1334
@@ -70,7 +70,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
             // we find previous flush Task which still accounts for any newly committed bytes and await that.
             lock (_flushLock)
             {
-                if (_lastFlushTask.IsCompleted)
+                if (_lastFlushTask == null || _lastFlushTask.IsCompleted)
                 {
                     _lastFlushTask = flushValueTask.AsTask();
                 }
@@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
             }
         }
 
-        private async Task TimeFlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
+        private async ValueTask<FlushResult> TimeFlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
         {
             if (minRate != null)
             {
@@ -88,7 +88,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
 
             try
             {
-                await _lastFlushTask;
+                return await _lastFlushTask;
             }
             catch (OperationCanceledException ex) when (outputAborter != null)
             {
@@ -99,13 +99,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
                 // A canceled token is the only reason flush should ever throw.
                 _log.LogError(0, ex, $"Unexpected exception in {nameof(TimingPipeFlusher)}.{nameof(TimeFlushAsync)}.");
             }
-
-            if (minRate != null)
+            finally
             {
-                _timeoutControl.StopTimingWrite();
+                if (minRate != null)
+                {
+                    _timeoutControl.StopTimingWrite();
+                }
+
+                cancellationToken.ThrowIfCancellationRequested();
             }
 
-            cancellationToken.ThrowIfCancellationRequested();
+            return default;
         }
     }
 }

src/Servers/Kestrel/Core/src/Properties/CoreStrings.Designer.cs

@@ -2239,6 +2239,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
         /// </summary>
         internal static string FormatServerShutdownDuringConnectionInitialization()
             => GetString("ServerShutdownDuringConnectionInitialization");
+        /// <summary>
+        /// Cannot call GetMemory() until response has started. Call HttpResponse.StartAsync() before calling GetMemory().
+        /// </summary>
+        internal static string StartAsyncBeforeGetMemory
+        {
+            get => GetString("StartAsyncBeforeGetMemory");
+        }
+
+        /// <summary>
+        /// Cannot call GetMemory() until response has started. Call HttpResponse.StartAsync() before calling GetMemory().
+        /// </summary>
+        internal static string FormatStartAsyncBeforeGetMemory()
+            => GetString("StartAsyncBeforeGetMemory");
 
         private static string GetString(string name, params string[] formatterNames)
         {

src/Servers/Kestrel/Core/test/HttpProtocolFeatureCollectionTests.cs

@@ -117,6 +117,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         {
             _collection[typeof(IHttpRequestFeature)] = CreateHttp1Connection();
             _collection[typeof(IHttpResponseFeature)] = CreateHttp1Connection();
+            _collection[typeof(IResponseBodyPipeFeature)] = CreateHttp1Connection();
             _collection[typeof(IHttpRequestIdentifierFeature)] = CreateHttp1Connection();
             _collection[typeof(IHttpRequestLifetimeFeature)] = CreateHttp1Connection();
             _collection[typeof(IHttpConnectionFeature)] = CreateHttp1Connection();
@@ -136,6 +137,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         {
             _collection.Set<IHttpRequestFeature>(CreateHttp1Connection());
             _collection.Set<IHttpResponseFeature>(CreateHttp1Connection());
+            _collection.Set<IResponseBodyPipeFeature>(CreateHttp1Connection());
             _collection.Set<IHttpRequestIdentifierFeature>(CreateHttp1Connection());
             _collection.Set<IHttpRequestLifetimeFeature>(CreateHttp1Connection());
             _collection.Set<IHttpConnectionFeature>(CreateHttp1Connection());
@@ -173,6 +175,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         {
             Assert.Same(_collection.Get<IHttpRequestFeature>(), _collection[typeof(IHttpRequestFeature)]);
             Assert.Same(_collection.Get<IHttpResponseFeature>(), _collection[typeof(IHttpResponseFeature)]);
+            Assert.Same(_collection.Get<IResponseBodyPipeFeature>(), _collection[typeof(IResponseBodyPipeFeature)]);
             Assert.Same(_collection.Get<IHttpRequestIdentifierFeature>(), _collection[typeof(IHttpRequestIdentifierFeature)]);
             Assert.Same(_collection.Get<IHttpRequestLifetimeFeature>(), _collection[typeof(IHttpRequestLifetimeFeature)]);
             Assert.Same(_collection.Get<IHttpConnectionFeature>(), _collection[typeof(IHttpConnectionFeature)]);

src/Servers/Kestrel/Core/test/HttpResponsePipeWriterTests.cs

@@ -0,0 +1,84 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
+using Moq;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
+{
+    public class HttpResponsePipeWriterTests
+    {
+        [Fact]
+        public void OnReaderCompletedThrowsNotSupported()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            Assert.Throws<NotSupportedException>(() => pipeWriter.OnReaderCompleted((a, b) => { }, null));
+        }
+
+        [Fact]
+        public void AdvanceAfterStopAcceptingWritesThrowsObjectDisposedException()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            pipeWriter.StartAcceptingWrites();
+            pipeWriter.StopAcceptingWrites();
+            var ex = Assert.Throws<ObjectDisposedException>(() => { pipeWriter.Advance(1); });
+            Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message);
+        }
+
+        [Fact]
+        public void GetMemoryAfterStopAcceptingWritesThrowsObjectDisposedException()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            pipeWriter.StartAcceptingWrites();
+            pipeWriter.StopAcceptingWrites();
+            var ex = Assert.Throws<ObjectDisposedException>(() => { pipeWriter.GetMemory(); });
+            Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message);
+        }
+
+        [Fact]
+        public void GetSpanAfterStopAcceptingWritesThrowsObjectDisposedException()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            pipeWriter.StartAcceptingWrites();
+            pipeWriter.StopAcceptingWrites();
+            var ex = Assert.Throws<ObjectDisposedException>(() => { pipeWriter.GetSpan(); });
+            Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message);
+        }
+
+        [Fact]
+        public void FlushAsyncAfterStopAcceptingWritesThrowsObjectDisposedException()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            pipeWriter.StartAcceptingWrites();
+            pipeWriter.StopAcceptingWrites();
+            var ex = Assert.Throws<ObjectDisposedException>(() => { pipeWriter.FlushAsync(); });
+            Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message);
+        }
+
+        [Fact]
+        public void WriteAsyncAfterStopAcceptingWritesThrowsObjectDisposedException()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            pipeWriter.StartAcceptingWrites();
+            pipeWriter.StopAcceptingWrites();
+            var ex = Assert.Throws<ObjectDisposedException>(() => { pipeWriter.WriteAsync(new Memory<byte>()); });
+            Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message);
+        }
+
+        [Fact]
+        public void CompleteCallsStopAcceptingWrites()
+        {
+            var pipeWriter = CreateHttpResponsePipeWriter();
+            pipeWriter.Complete();
+            var ex = Assert.Throws<ObjectDisposedException>(() => { pipeWriter.WriteAsync(new Memory<byte>()); });
+            Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message);
+        }
+
+        private static HttpResponsePipeWriter CreateHttpResponsePipeWriter()
+        {
+            return new HttpResponsePipeWriter(Mock.Of<IHttpResponseControl>());
+        }
+    }
+}

src/Servers/Kestrel/Core/test/HttpResponseStreamTests.cs

@@ -1,13 +1,13 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
 using System.IO;
+using System.IO.Pipelines;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
-using Microsoft.AspNetCore.Server.Kestrel.Core.Tests.TestHelpers;
 using Moq;
 using Xunit;
 
@@ -18,77 +18,77 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         [Fact]
         public void CanReadReturnsFalse()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.False(stream.CanRead);
         }
 
         [Fact]
         public void CanSeekReturnsFalse()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.False(stream.CanSeek);
         }
 
         [Fact]
         public void CanWriteReturnsTrue()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.True(stream.CanWrite);
         }
 
         [Fact]
         public void ReadThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.Read(new byte[1], 0, 1));
         }
 
         [Fact]
         public void ReadByteThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.ReadByte());
         }
 
         [Fact]
         public async Task ReadAsyncThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             await Assert.ThrowsAsync<NotSupportedException>(() => stream.ReadAsync(new byte[1], 0, 1));
         }
 
         [Fact]
         public void BeginReadThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.BeginRead(new byte[1], 0, 1, null, null));
         }
 
         [Fact]
         public void SeekThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.Seek(0, SeekOrigin.Begin));
         }
 
         [Fact]
         public void LengthThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.Length);
         }
 
         [Fact]
         public void SetLengthThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.SetLength(0));
         }
 
         [Fact]
         public void PositionThrows()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), new MockHttpResponseControl());
+            var stream = CreateHttpResponseStream();
             Assert.Throws<NotSupportedException>(() => stream.Position);
             Assert.Throws<NotSupportedException>(() => stream.Position = 0);
         }
@@ -96,7 +96,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         [Fact]
         public void StopAcceptingWritesCausesWriteToThrowObjectDisposedException()
         {
-            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), Mock.Of<IHttpResponseControl>());
+            var pipeWriter = new HttpResponsePipeWriter(Mock.Of<IHttpResponseControl>());
+            var stream = new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), pipeWriter);
             stream.StartAcceptingWrites();
             stream.StopAcceptingWrites();
             var ex = Assert.Throws<ObjectDisposedException>(() => { stream.WriteAsync(new byte[1], 0, 1); });
@@ -110,9 +111,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             var mockBodyControl = new Mock<IHttpBodyControlFeature>();
             mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(() => allowSynchronousIO);
             var mockHttpResponseControl = new Mock<IHttpResponseControl>();
-            mockHttpResponseControl.Setup(m => m.WriteAsync(It.IsAny<ReadOnlyMemory<byte>>(), CancellationToken.None)).Returns(Task.CompletedTask);
+            mockHttpResponseControl.Setup(m => m.WritePipeAsync(It.IsAny<ReadOnlyMemory<byte>>(), CancellationToken.None)).Returns(new ValueTask<FlushResult>(new FlushResult()));
 
-            var stream = new HttpResponseStream(mockBodyControl.Object, mockHttpResponseControl.Object);
+            var pipeWriter = new HttpResponsePipeWriter(mockHttpResponseControl.Object);
+            var stream = new HttpResponseStream(mockBodyControl.Object, pipeWriter);
             stream.StartAcceptingWrites();
 
             // WriteAsync doesn't throw.
@@ -125,5 +127,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             // If IHttpBodyControlFeature.AllowSynchronousIO is true, Write no longer throws.
             stream.Write(new byte[1], 0, 1);
         }
+
+        private static HttpResponseStream CreateHttpResponseStream()
+        {
+            var pipeWriter = new HttpResponsePipeWriter(Mock.Of<IHttpResponseControl>());
+            return new HttpResponseStream(Mock.Of<IHttpBodyControlFeature>(), pipeWriter);
+        }
     }
 }

src/Servers/Kestrel/Core/test/Microsoft.AspNetCore.Server.Kestrel.Core.Tests.csproj

@@ -13,6 +13,10 @@
     <Content Include="$(KestrelSharedSourceRoot)test\TestCertificates\*.pfx" LinkBase="shared\TestCertificates" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Compile Remove="C:\Users\jukotali\code\aspnetcore\src\Servers\Kestrel\shared\test\HttpResponseWritingExtensions.cs" />
+  </ItemGroup>
+
   <ItemGroup>
     <Reference Include="Microsoft.AspNetCore.Http" />
     <Reference Include="Microsoft.AspNetCore.Server.Kestrel.Core" />

src/Servers/Kestrel/Core/test/OutputProducerTests.cs

@@ -89,14 +89,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
                 connectionContext,
                 serviceContext.Log,
                 Mock.Of<ITimeoutControl>(),
-                Mock.Of<IHttpMinResponseDataRateFeature>());
+                Mock.Of<IHttpMinResponseDataRateFeature>(),
+                _memoryPool);
 
             return socketOutput;
         }
 
         private class TestHttpOutputProducer : Http1OutputProducer
         {
-            public TestHttpOutputProducer(Pipe pipe, string connectionId, ConnectionContext connectionContext, IKestrelTrace log, ITimeoutControl timeoutControl, IHttpMinResponseDataRateFeature minResponseDataRateFeature) : base(pipe.Writer, connectionId, connectionContext, log, timeoutControl, minResponseDataRateFeature)
+            public TestHttpOutputProducer(Pipe pipe, string connectionId, ConnectionContext connectionContext, IKestrelTrace log, ITimeoutControl timeoutControl, IHttpMinResponseDataRateFeature minResponseDataRateFeature, MemoryPool<byte> memoryPool) : base(pipe.Writer, connectionId, connectionContext, log, timeoutControl, minResponseDataRateFeature, memoryPool)
             {
                 Pipe = pipe;
             }

src/Servers/Kestrel/Core/test/StreamsTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         [Fact]
         public async Task StreamsThrowAfterAbort()
         {
-            var streams = new Streams(Mock.Of<IHttpBodyControlFeature>(), Mock.Of<IHttpResponseControl>());
+            var streams = new Streams(Mock.Of<IHttpBodyControlFeature>(), new HttpResponsePipeWriter(Mock.Of<IHttpResponseControl>()));
             var (request, response) = streams.Start(new MockMessageBody());
 
             var ex = new Exception("My error");
@@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         [Fact]
         public async Task StreamsThrowOnAbortAfterUpgrade()
         {
-            var streams = new Streams(Mock.Of<IHttpBodyControlFeature>(), Mock.Of<IHttpResponseControl>());
+            var streams = new Streams(Mock.Of<IHttpBodyControlFeature>(), new HttpResponsePipeWriter(Mock.Of<IHttpResponseControl>()));
             var (request, response) = streams.Start(new MockMessageBody(upgradeable: true));
 
             var upgrade = streams.Upgrade();
@@ -52,7 +52,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         [Fact]
         public async Task StreamsThrowOnUpgradeAfterAbort()
         {
-            var streams = new Streams(Mock.Of<IHttpBodyControlFeature>(), Mock.Of<IHttpResponseControl>());
+            var streams = new Streams(Mock.Of<IHttpBodyControlFeature>(), new HttpResponsePipeWriter(Mock.Of<IHttpResponseControl>()));
 
             var (request, response) = streams.Start(new MockMessageBody(upgradeable: true));
             var ex = new Exception("My error");

src/Servers/Kestrel/Core/test/TestHelpers/MockHttpResponseControl.cs

@@ -1,27 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
-
-namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests.TestHelpers
-{
-    public class MockHttpResponseControl : IHttpResponseControl
-    {
-        public Task FlushAsync(CancellationToken cancellationToken)
-        {
-            return Task.CompletedTask;
-        }
-
-        public void ProduceContinue()
-        {
-        }
-
-        public Task WriteAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken)
-        {
-            return Task.CompletedTask;
-        }
-    }
-}

src/Servers/Kestrel/perf/Kestrel.Performance/InMemoryTransportBenchmark.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using BenchmarkDotNet.Attributes;

src/Servers/Kestrel/shared/test/TestApp.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
@@ -38,5 +39,39 @@ namespace Microsoft.AspNetCore.Testing
         {
             return Task.CompletedTask;
         }
+
+        public static async Task EchoAppPipeWriter(HttpContext httpContext)
+        {
+            var request = httpContext.Request;
+            var response = httpContext.Response;
+            var buffer = new byte[httpContext.Request.ContentLength ?? 0];
+
+            if (buffer.Length > 0)
+            {
+                await request.Body.ReadUntilEndAsync(buffer).DefaultTimeout();
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory(buffer.Length);
+                buffer.CopyTo(memory);
+                response.BodyPipe.Advance(buffer.Length);
+                await response.BodyPipe.FlushAsync();
+            }
+        }
+
+        public static async Task EchoAppPipeWriterChunked(HttpContext httpContext)
+        {
+            var request = httpContext.Request;
+            var response = httpContext.Response;
+            var data = new MemoryStream();
+            await request.Body.CopyToAsync(data);
+            var bytes = data.ToArray();
+
+            response.Headers["Content-Length"] = bytes.Length.ToString();
+            await response.StartAsync();
+
+            var memory = response.BodyPipe.GetMemory(bytes.Length);
+            bytes.CopyTo(memory);
+            response.BodyPipe.Advance(bytes.Length);
+            await response.BodyPipe.FlushAsync();
+        }
     }
-}
+}

src/Servers/Kestrel/test/FunctionalTests/ResponseTests.cs

@@ -20,6 +20,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.AspNetCore.Server.Kestrel.Https;
 using Microsoft.AspNetCore.Server.Kestrel.Https.Internal;
+using Microsoft.AspNetCore.Server.Kestrel.Tests;
 using Microsoft.AspNetCore.Testing;
 using Microsoft.AspNetCore.Testing.xunit;
 using Microsoft.Extensions.Logging;
@@ -62,7 +63,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
 
                         for (int i = 0; i < 1024; i++)
                         {
-                            await context.Response.Body.WriteAsync(bytes, 0, bytes.Length);
+                            await context.Response.BodyPipe.WriteAsync(new Memory<byte>(bytes, 0, bytes.Length));
                         }
                     });
                 });
@@ -208,7 +209,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
 
                 try
                 {
-                    await response.WriteAsync(largeString, lifetime.RequestAborted);
+                    await response.WriteAsync(largeString, cancellationToken: lifetime.RequestAborted);
                 }
                 catch (Exception ex)
                 {
@@ -297,7 +298,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                 {
                     for (var i = 0; i < 1000; i++)
                     {
-                        await context.Response.Body.WriteAsync(scratchBuffer, 0, scratchBuffer.Length, context.RequestAborted);
+                        await context.Response.BodyPipe.WriteAsync(new Memory<byte>(scratchBuffer, 0, scratchBuffer.Length), context.RequestAborted);
                         await Task.Delay(10);
                     }
                 }
@@ -499,7 +500,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                 {
                     for (; i < chunks; i++)
                     {
-                        await context.Response.Body.WriteAsync(chunkData, 0, chunkData.Length, context.RequestAborted);
+                        await context.Response.BodyPipe.WriteAsync(new Memory<byte>(chunkData, 0, chunkData.Length), context.RequestAborted);
                         await Task.Yield();
                     }
 
@@ -606,7 +607,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                 {
                     for (var i = 0; i < chunks; i++)
                     {
-                        await context.Response.Body.WriteAsync(chunkData, 0, chunkData.Length, context.RequestAborted);
+                        await context.Response.BodyPipe.WriteAsync(new Memory<byte>(chunkData, 0, chunkData.Length), context.RequestAborted);
                     }
                 }
                 catch (OperationCanceledException)
@@ -770,7 +771,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
 
                 for (var i = 0; i < chunkCount; i++)
                 {
-                    await context.Response.Body.WriteAsync(chunkData, 0, chunkData.Length, context.RequestAborted);
+                    await context.Response.BodyPipe.WriteAsync(new Memory<byte>(chunkData, 0, chunkData.Length), context.RequestAborted);
                 }
 
                 appFuncCompleted.SetResult(null);
@@ -847,7 +848,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
                 context.Response.Headers[$"X-Custom-Header"] = headerStringValues;
                 context.Response.ContentLength = 0;
 
-                await context.Response.Body.FlushAsync();
+                await context.Response.BodyPipe.FlushAsync();
             }
 
             using (var server = new TestServer(App, testContext, listenOptions))

src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedResponseTests.cs

@@ -23,8 +23,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 var response = httpContext.Response;
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello "), 0, 6);
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("World!"), 0, 6);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello "), 0, 6));
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("World!"), 0, 6));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -162,9 +162,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 var response = httpContext.Response;
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello "), 0, 6);
-                await response.Body.WriteAsync(new byte[0], 0, 0);
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("World!"), 0, 6);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello "), 0, 6));
+                await response.BodyPipe.WriteAsync(new Memory<byte>(new byte[0], 0, 0));
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("World!"), 0, 6));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -244,7 +244,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 var response = httpContext.Response;
-                await response.Body.WriteAsync(new byte[0], 0, 0);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(new byte[0], 0, 0));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -275,7 +275,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 var response = httpContext.Response;
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World!"), 0, 12);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello World!"), 0, 12));
                 throw new Exception();
             }, testContext))
             {
@@ -309,7 +309,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 var response = httpContext.Response;
-                await response.Body.WriteAsync(new byte[0], 0, 0);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(new byte[0], 0, 0));
                 throw new Exception();
             }, testContext))
             {
@@ -344,12 +344,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 var response = httpContext.Response;
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello "), 0, 6);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello "), 0, 6));
 
                 // Don't complete response until client has received the first chunk.
                 await flushWh.Task.DefaultTimeout();
 
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("World!"), 0, 6);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("World!"), 0, 6));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -391,9 +391,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 var response = httpContext.Response;
                 response.Headers["Transfer-Encoding"] = "chunked";
 
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("6\r\nHello \r\n"), 0, 11);
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("6\r\nWorld!\r\n"), 0, 11);
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("0\r\n\r\n"), 0, 5);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("6\r\nHello \r\n"), 0, 11));
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("6\r\nWorld!\r\n"), 0, 11));
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("0\r\n\r\n"), 0, 5));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -420,6 +420,416 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 await server.StopAsync();
             }
         }
+
+        [Fact]
+        public async Task ChunksWithGetMemoryBeforeFirstFlushStillFlushes()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory();
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                memory = response.BodyPipe.GetMemory();
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                secondPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.BodyPipe.FlushAsync();
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "c",
+                        "Hello World!",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ChunksWithGetMemoryLargeWriteBeforeFirstFlush()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(5000); // This will return 4089
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes(new string('a', memory.Length));
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(memory.Length);
+
+                memory = response.BodyPipe.GetMemory();
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                secondPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.BodyPipe.FlushAsync();
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "ff9",
+                        new string('a', 4089),
+                        "6",
+                        "World!",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ChunksWithGetMemoryWithInitialFlushWorks()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.BodyPipe.FlushAsync();
+
+                var memory = response.BodyPipe.GetMemory(5000); // This will return 4089
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes(new string('a', memory.Length));
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(memory.Length);
+
+                memory = response.BodyPipe.GetMemory();
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                secondPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.BodyPipe.FlushAsync();
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "ff9",
+                        new string('a', 4089),
+                        "6",
+                        "World!",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ChunkGetMemoryMultipleAdvance()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(6);
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "c",
+                        "Hello World!",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ChunkGetSpanMultipleAdvance()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                // To avoid using span in an async method
+                void NonAsyncMethod()
+                {
+                    var span = response.BodyPipe.GetSpan(4096);
+                    var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                    fisrtPartOfResponse.CopyTo(span);
+                    response.BodyPipe.Advance(6);
+
+                    var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                    secondPartOfResponse.CopyTo(span.Slice(6));
+                    response.BodyPipe.Advance(6);
+                }
+
+                await response.StartAsync();
+
+                NonAsyncMethod();
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "c",
+                        "Hello World!",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ChunkGetMemoryAndWrite()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.WriteAsync("World!");
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "c",
+                        "Hello World!",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task GetMemoryWithSizeHint()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(0);
+
+                // Headers are already written to memory, sliced appropriately
+                Assert.Equal(4005, memory.Length);
+
+                memory = response.BodyPipe.GetMemory(1000000);
+                Assert.Equal(4005, memory.Length);
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Theory]
+        [InlineData(15)]
+        [InlineData(255)]
+        public async Task ChunkGetMemoryWithSmallerSizesWork(int writeSize)
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes(new string('a', writeSize));
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(writeSize);
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        writeSize.ToString("X").ToLower(),
+                        new string('a', writeSize),
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ChunkedWithBothPipeAndStreamWorks()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+                var secondPartOfResponse = Encoding.ASCII.GetBytes(" world");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(6);
+
+                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("hello, world"));
+                await response.BodyPipe.WriteAsync(Encoding.ASCII.GetBytes("hello, world"));
+                await response.WriteAsync("hello, world");
+
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "c",
+                        "hello, world",
+                        "c",
+                        "hello, world",
+                        "c",
+                        "hello, world",
+                        "c",
+                        "hello, world",
+                        "0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
     }
 }
 

src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionAdapterTests.cs

@@ -4,6 +4,7 @@
 using System;
 using System.IO;
 using System.Net;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
@@ -18,8 +19,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 {
     public class ConnectionAdapterTests : TestApplicationErrorLoggerLoggedTest
     {
-        [Fact]
-        public async Task CanReadAndWriteWithRewritingConnectionAdapter()
+
+        public static TheoryData<RequestDelegate> EchoAppRequestDelegates =>
+            new TheoryData<RequestDelegate>
+            {
+                { TestApp.EchoApp },
+                { TestApp.EchoAppPipeWriter }
+            };
+
+        [Theory]
+        [MemberData(nameof(EchoAppRequestDelegates))]
+        public async Task CanReadAndWriteWithRewritingConnectionAdapter(RequestDelegate requestDelegate)
         {
             var adapter = new RewritingConnectionAdapter();
             var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
@@ -31,7 +41,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
             var sendString = "POST / HTTP/1.0\r\nContent-Length: 12\r\n\r\nHello World?";
 
-            using (var server = new TestServer(TestApp.EchoApp, serviceContext, listenOptions))
+            using (var server = new TestServer(requestDelegate, serviceContext, listenOptions))
             {
                 using (var connection = server.CreateConnection())
                 {
@@ -50,8 +60,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             Assert.Equal(sendString.Length, adapter.BytesRead);
         }
 
-        [Fact]
-        public async Task CanReadAndWriteWithAsyncConnectionAdapter()
+        [Theory]
+        [MemberData(nameof(EchoAppRequestDelegates))]
+        public async Task CanReadAndWriteWithAsyncConnectionAdapter(RequestDelegate requestDelegate)
         {
             var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
             {
@@ -60,7 +71,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
             var serviceContext = new TestServiceContext(LoggerFactory);
 
-            using (var server = new TestServer(TestApp.EchoApp, serviceContext, listenOptions))
+            using (var server = new TestServer(requestDelegate, serviceContext, listenOptions))
             {
                 using (var connection = server.CreateConnection())
                 {
@@ -80,8 +91,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
-        [Fact]
-        public async Task ImmediateFinAfterOnConnectionAsyncClosesGracefully()
+        [Theory]
+        [MemberData(nameof(EchoAppRequestDelegates))]
+        public async Task ImmediateFinAfterOnConnectionAsyncClosesGracefully(RequestDelegate requestDelegate)
         {
             var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
             {
@@ -90,7 +102,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
             var serviceContext = new TestServiceContext(LoggerFactory);
 
-            using (var server = new TestServer(TestApp.EchoApp, serviceContext, listenOptions))
+            using (var server = new TestServer(requestDelegate, serviceContext, listenOptions))
             {
                 using (var connection = server.CreateConnection())
                 {
@@ -102,8 +114,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
-        [Fact]
-        public async Task ImmediateFinAfterThrowingClosesGracefully()
+        [Theory]
+        [MemberData(nameof(EchoAppRequestDelegates))]
+        public async Task ImmediateFinAfterThrowingClosesGracefully(RequestDelegate requestDelegate)
         {
             var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
             {
@@ -112,7 +125,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
             var serviceContext = new TestServiceContext(LoggerFactory);
 
-            using (var server = new TestServer(TestApp.EchoApp, serviceContext, listenOptions))
+            using (var server = new TestServer(requestDelegate, serviceContext, listenOptions))
             {
                 using (var connection = server.CreateConnection())
                 {
@@ -124,9 +137,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
-        [Fact]
+        [Theory]
         [CollectDump]
-        public async Task ImmediateShutdownAfterOnConnectionAsyncDoesNotCrash()
+        [MemberData(nameof(EchoAppRequestDelegates))]
+        public async Task ImmediateShutdownAfterOnConnectionAsyncDoesNotCrash(RequestDelegate requestDelegate)
         {
             var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
             {
@@ -136,7 +150,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             var serviceContext = new TestServiceContext(LoggerFactory);
 
             var stopTask = Task.CompletedTask;
-            using (var server = new TestServer(TestApp.EchoApp, serviceContext, listenOptions))
+            using (var server = new TestServer(requestDelegate, serviceContext, listenOptions))
             using (var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)))
             {
                 using (var connection = server.CreateConnection())
@@ -180,8 +194,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
-        [Fact]
-        public async Task ThrowingSynchronousConnectionAdapterDoesNotCrashServer()
+        [Theory]
+        [MemberData(nameof(EchoAppRequestDelegates))]
+        public async Task ThrowingSynchronousConnectionAdapterDoesNotCrashServer(RequestDelegate requestDelegate)
         {
             var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
             {
@@ -190,7 +205,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
             var serviceContext = new TestServiceContext(LoggerFactory);
 
-            using (var server = new TestServer(TestApp.EchoApp, serviceContext, listenOptions))
+            using (var server = new TestServer(requestDelegate, serviceContext, listenOptions))
             {
                 using (var connection = server.CreateConnection())
                 {
@@ -241,6 +256,40 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
+        [Fact]
+        public async Task CanFlushAsyncWithConnectionAdapterPipeWriter()
+        {
+            var listenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 0))
+            {
+                ConnectionAdapters = { new PassThroughConnectionAdapter() }
+            };
+
+            var serviceContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async context =>
+            {
+                await context.Response.BodyPipe.WriteAsync(Encoding.ASCII.GetBytes("Hello "));
+                await context.Response.BodyPipe.FlushAsync();
+                await context.Response.BodyPipe.WriteAsync(Encoding.ASCII.GetBytes("World!"));
+            }, serviceContext, listenOptions))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.0",
+                        "",
+                        "");
+                    await connection.ReceiveEnd(
+                        "HTTP/1.1 200 OK",
+                        "Connection: close",
+                        $"Date: {serviceContext.DateHeaderValue}",
+                        "",
+                        "Hello World!");
+                }
+                await server.StopAsync();
+            }
+        }
+
         private class RewritingConnectionAdapter : IConnectionAdapter
         {
             private RewritingStream _rewritingStream;

src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs

@@ -7,6 +7,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Runtime.ExceptionServices;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Connections;
@@ -228,7 +229,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
         }
 
         [Theory]
-        [InlineData("/","/")]
+        [InlineData("/", "/")]
         [InlineData("/a%5E", "/a^")]
         [InlineData("/a%E2%82%AC", "/a€")]
         [InlineData("/a%2Fb", "/a%2Fb")] // Forward slash, not decoded
@@ -910,7 +911,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             Assert.IsType<Http2StreamErrorException>(thrownEx.InnerException);
         }
 
-        [Fact]
+        [Fact] // TODO https://github.com/aspnet/AspNetCore/issues/7034
         public async Task ContentLength_Response_FirstWriteMoreBytesWritten_Throws_Sends500()
         {
             var headers = new[]
@@ -928,12 +929,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             await StartStreamAsync(1, headers, endStream: true);
 
             var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
-                withLength: 55,
+                withLength: 56,
                 withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
                 withStreamId: 1);
-            await ExpectAsync(Http2FrameType.DATA,
-                withLength: 0,
-                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+            await ExpectAsync(Http2FrameType.RST_STREAM,
+                withLength: 4,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
                 withStreamId: 1);
 
             Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Exception?.Message.Contains("Response Content-Length mismatch: too many bytes written (12 of 11).") ?? false);
@@ -944,8 +945,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
 
             Assert.Equal(3, _decodedHeaders.Count);
             Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
-            Assert.Equal("500", _decodedHeaders[HeaderNames.Status]);
-            Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("11", _decodedHeaders[HeaderNames.ContentLength]);
         }
 
         [Fact]
@@ -2468,5 +2469,719 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
             Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
         }
+
+        [Fact]
+        public async Task GetMemoryAdvance_Works()
+        {
+            var headers = new[]
+             {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory();
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                memory = response.BodyPipe.GetMemory();
+                var secondPartOfResponse = Encoding.ASCII.GetBytes(" world");
+                secondPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+
+            Assert.True(_helloWorldBytes.AsSpan().SequenceEqual(dataFrame.PayloadSequence.ToArray()));
+        }
+
+        [Fact]
+        public async Task WriteAsync_GetMemoryLargeWriteBeforeFirstFlush()
+        {
+            var headers = new[]
+             {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(5000);
+                Assert.Equal(4096, memory.Length);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes(new string('a', memory.Length));
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(memory.Length);
+
+                memory = response.BodyPipe.GetMemory();
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("aaaaaa");
+                secondPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.BodyPipe.FlushAsync();
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 4102,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal(Encoding.ASCII.GetBytes(new string('a', 4102)), dataFrame.PayloadSequence.ToArray());
+        }
+
+        [Fact]
+        public async Task WriteAsync_WithGetMemoryWithInitialFlushWorks()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.BodyPipe.FlushAsync();
+
+                var memory = response.BodyPipe.GetMemory(5000);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes(new string('a', memory.Length));
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(memory.Length);
+
+                memory = response.BodyPipe.GetMemory();
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("aaaaaa");
+                secondPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.BodyPipe.FlushAsync();
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 4102,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal(Encoding.ASCII.GetBytes(new string('a', 4102)), dataFrame.PayloadSequence.ToArray());
+        }
+
+        [Fact]
+        public async Task WriteAsync_GetMemoryMultipleAdvance()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.BodyPipe.FlushAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                var secondPartOfResponse = Encoding.ASCII.GetBytes(" world");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(6);
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.True(_helloWorldBytes.AsSpan().SequenceEqual(dataFrame.PayloadSequence.ToArray()));
+        }
+
+        [Fact]
+        public async Task WriteAsync_GetSpanMultipleAdvance()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.BodyPipe.FlushAsync();
+
+                void NonAsyncMethod()
+                {
+                    var span = response.BodyPipe.GetSpan();
+                    var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                    fisrtPartOfResponse.CopyTo(span);
+                    response.BodyPipe.Advance(6);
+
+                    var secondPartOfResponse = Encoding.ASCII.GetBytes(" world");
+                    secondPartOfResponse.CopyTo(span.Slice(6));
+                    response.BodyPipe.Advance(6);
+                }
+                NonAsyncMethod();
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.True(_helloWorldBytes.AsSpan().SequenceEqual(dataFrame.PayloadSequence.ToArray()));
+        }
+
+        [Fact]
+        public async Task WriteAsync_GetMemoryAndWrite()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                await response.WriteAsync(" world");
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.True(_helloWorldBytes.AsSpan().SequenceEqual(dataFrame.PayloadSequence.ToArray()));
+        }
+
+        [Fact]
+        public async Task WriteAsync_GetMemoryWithSizeHintAlwaysReturnsSameSize()
+        {
+            var headers = new[]
+{
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(0);
+                Assert.Equal(4096, memory.Length);
+
+                memory = response.BodyPipe.GetMemory(1000000);
+                Assert.Equal(4096, memory.Length);
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            var dataFrame = await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+        }
+
+        [Fact]
+        public async Task WriteAsync_BothPipeAndStreamWorks()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+                var secondPartOfResponse = Encoding.ASCII.GetBytes(" world");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(6);
+
+                await response.BodyPipe.FlushAsync();
+
+                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("hello, world"));
+                await response.BodyPipe.WriteAsync(Encoding.ASCII.GetBytes("hello, world"));
+                await response.WriteAsync("hello, world");
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+        }
+
+        [Fact]
+        public async Task ContentLengthWithGetSpanWorks()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+                response.ContentLength = 12;
+                await response.StartAsync();
+
+                void NonAsyncMethod()
+                {
+                    var span = response.BodyPipe.GetSpan(4096);
+                    var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                    fisrtPartOfResponse.CopyTo(span);
+                    response.BodyPipe.Advance(6);
+
+                    var secondPartOfResponse = Encoding.ASCII.GetBytes(" world");
+                    secondPartOfResponse.CopyTo(span.Slice(6));
+                    response.BodyPipe.Advance(6);
+                }
+
+                NonAsyncMethod();
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 56,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("12", _decodedHeaders[HeaderNames.ContentLength]);
+        }
+
+        [Fact]
+        public async Task ContentLengthWithGetMemoryWorks()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+                response.ContentLength = 12;
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(6);
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 56,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("12", _decodedHeaders[HeaderNames.ContentLength]);
+        }
+
+        [Fact]
+        public async Task ResponseBodyCanWrite()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                httpContext.Response.ContentLength = 12;
+                await httpContext.Response.Body.WriteAsync(Encoding.ASCII.GetBytes("hello, world"));
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 56,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("12", _decodedHeaders[HeaderNames.ContentLength]);
+        }
+
+        [Fact]
+        public async Task ResponseBodyAndResponsePipeWorks()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async httpContext =>
+            {
+                var response = httpContext.Response;
+                response.ContentLength = 54;
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+                var secondPartOfResponse = Encoding.ASCII.GetBytes(" world\r\n");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(8);
+                await response.BodyPipe.FlushAsync();
+                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("hello, world\r\n"));
+                await response.BodyPipe.WriteAsync(Encoding.ASCII.GetBytes("hello, world\r\n"));
+                await response.WriteAsync("hello, world");
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 56,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 14,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 14,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 14,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 12,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("54", _decodedHeaders[HeaderNames.ContentLength]);
+        }
+
+        [Fact]
+        public async Task ResponseBodyPipeCompleteWithoutExceptionDoesNotThrow()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async context =>
+            {
+                context.Response.BodyPipe.Complete();
+                await Task.CompletedTask;
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 55,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+        }
+
+        [Fact]
+        public async Task ResponseBodyPipeCompleteWithoutExceptionWritesDoNotThrow()
+        {
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async context =>
+            {
+                context.Response.BodyPipe.Complete();
+                await context.Response.WriteAsync("");
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            // Don't receive content length because we called WriteAsync which caused an invalid response
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+        }
+
+        [Fact]
+        public async Task ResponseBodyPipeCompleteWithExceptionThrows()
+        {
+            var expectedException = new Exception();
+            var headers = new[]
+            {
+                new KeyValuePair<string, string>(HeaderNames.Method, "GET"),
+                new KeyValuePair<string, string>(HeaderNames.Path, "/"),
+                new KeyValuePair<string, string>(HeaderNames.Scheme, "http"),
+            };
+            await InitializeConnectionAsync(async context =>
+            {
+                context.Response.BodyPipe.Complete(expectedException);
+                await Task.CompletedTask;
+            });
+
+            await StartStreamAsync(1, headers, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 55,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("500", _decodedHeaders[HeaderNames.Status]);
+
+            Assert.Contains(TestSink.Writes, w => w.EventId.Id == 13 && w.LogLevel == LogLevel.Error
+                       && w.Exception is ConnectionAbortedException && w.Exception.InnerException == expectedException);
+        }
     }
 }

src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TestBase.cs

@@ -177,7 +177,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             _mockConnectionContext.Setup(c => c.Abort(It.IsAny<ConnectionAbortedException>())).Callback<ConnectionAbortedException>(ex =>
             {
                 // Emulate transport abort so the _connectionTask completes.
-                _pair.Application.Output.Complete(ex);
+                Task.Run(() => _pair.Application.Output.Complete(ex));
             });
 
             _noopApplication = context => Task.CompletedTask;
@@ -380,7 +380,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
 
             _serviceContext = new TestServiceContext(LoggerFactory, _mockKestrelTrace.Object)
             {
-                Scheduler = PipeScheduler.Inline
+                Scheduler = PipeScheduler.Inline,
             };
         }
 

src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs

@@ -18,6 +18,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests.TestTransport;
 using Microsoft.AspNetCore.Testing;
 using Microsoft.Extensions.Logging.Testing;
+using Moq;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
@@ -58,6 +59,57 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
+        [Fact]
+        public async Task PipesAreNotPersistedBySettingStreamPipeWriterAcrossRequests()
+        {
+            var responseBodyPersisted = false;
+            PipeWriter bodyPipe = null;
+            using (var server = new TestServer(async context =>
+            {
+                if (context.Response.BodyPipe == bodyPipe)
+                {
+                    responseBodyPersisted = true;
+                }
+                bodyPipe = new StreamPipeWriter(new MemoryStream());
+                context.Response.BodyPipe = bodyPipe;
+
+                await context.Response.WriteAsync("hello, world");
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                Assert.Equal(string.Empty, await server.HttpClientSlim.GetStringAsync($"http://localhost:{server.Port}/"));
+                Assert.Equal(string.Empty, await server.HttpClientSlim.GetStringAsync($"http://localhost:{server.Port}/"));
+
+                Assert.False(responseBodyPersisted);
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task PipesAreNotPersistedAcrossRequests()
+        {
+            var responseBodyPersisted = false;
+            PipeWriter bodyPipe = null;
+            using (var server = new TestServer(async context =>
+            {
+                if (context.Response.BodyPipe == bodyPipe)
+                {
+                    responseBodyPersisted = true;
+                }
+                bodyPipe = context.Response.BodyPipe;
+
+                await context.Response.WriteAsync("hello, world");
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                Assert.Equal("hello, world", await server.HttpClientSlim.GetStringAsync($"http://localhost:{server.Port}/"));
+                Assert.Equal("hello, world", await server.HttpClientSlim.GetStringAsync($"http://localhost:{server.Port}/"));
+
+                Assert.False(responseBodyPersisted);
+
+                await server.StopAsync();
+            }
+        }
+
         [Fact]
         public async Task RequestBodyReadAsyncCanBeCancelled()
         {

src/Servers/Kestrel/test/InMemory.FunctionalTests/ResponseTests.cs

@@ -77,7 +77,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async context =>
             {
                 await context.Response.WriteAsync("hello, world");
-                await context.Response.Body.FlushAsync();
+                await context.Response.BodyPipe.FlushAsync();
                 ex = Assert.Throws<InvalidOperationException>(() => context.Response.OnStarting(_ => Task.CompletedTask, null));
             }, new TestServiceContext(LoggerFactory)))
             {
@@ -155,14 +155,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                     var data = new byte[1024 * 1024 * 10];
 
                     var timerTask = Task.Delay(TimeSpan.FromSeconds(1));
-                    var writeTask = context.Response.Body.WriteAsync(data, 0, data.Length, cts.Token).DefaultTimeout();
+                    var writeTask = context.Response.BodyPipe.WriteAsync(new Memory<byte>(data, 0, data.Length), cts.Token).AsTask().DefaultTimeout();
                     var completedTask = await Task.WhenAny(writeTask, timerTask);
 
                     while (completedTask == writeTask)
                     {
                         await writeTask;
                         timerTask = Task.Delay(TimeSpan.FromSeconds(1));
-                        writeTask = context.Response.Body.WriteAsync(data, 0, data.Length, cts.Token).DefaultTimeout();
+                        writeTask = context.Response.BodyPipe.WriteAsync(new Memory<byte>(data, 0, data.Length), cts.Token).AsTask().DefaultTimeout();
                         completedTask = await Task.WhenAny(writeTask, timerTask);
                     }
 
@@ -660,7 +660,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 await httpContext.Response.WriteAsync(response);
-                await httpContext.Response.Body.FlushAsync();
+                await httpContext.Response.BodyPipe.FlushAsync();
             }, new TestServiceContext(LoggerFactory, mockKestrelTrace.Object)))
             {
                 using (var connection = server.CreateConnection())
@@ -696,12 +696,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 ServerOptions = { AllowSynchronousIO = true }
             };
 
-            using (var server = new TestServer(httpContext =>
+            using (var server = new TestServer(async httpContext =>
             {
                 httpContext.Response.ContentLength = 11;
-                httpContext.Response.Body.Write(Encoding.ASCII.GetBytes("hello,"), 0, 6);
-                httpContext.Response.Body.Write(Encoding.ASCII.GetBytes(" world"), 0, 6);
-                return Task.CompletedTask;
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("hello,"), 0, 6));
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes(" world"), 0, 6));
             }, serviceContext))
             {
                 using (var connection = server.CreateConnection())
@@ -774,12 +773,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 ServerOptions = { AllowSynchronousIO = true }
             };
 
-            using (var server = new TestServer(httpContext =>
+            using (var server = new TestServer(async httpContext =>
             {
                 var response = Encoding.ASCII.GetBytes("hello, world");
                 httpContext.Response.ContentLength = 5;
-                httpContext.Response.Body.Write(response, 0, response.Length);
-                return Task.CompletedTask;
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, 0, response.Length));
             }, serviceContext))
             {
                 using (var connection = server.CreateConnection())
@@ -811,11 +809,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
         {
             var serviceContext = new TestServiceContext(LoggerFactory);
 
-            using (var server = new TestServer(httpContext =>
+            using (var server = new TestServer(async httpContext =>
             {
                 var response = Encoding.ASCII.GetBytes("hello, world");
                 httpContext.Response.ContentLength = 5;
-                return httpContext.Response.Body.WriteAsync(response, 0, response.Length);
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, 0, response.Length));
             }, serviceContext))
             {
                 using (var connection = server.CreateConnection())
@@ -986,8 +984,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
         }
 
         [Theory]
-        [InlineData(false)]
         [InlineData(true)]
+        [InlineData(false)]
         public async Task WhenAppSetsContentLengthToZeroAndDoesNotWriteNoErrorIsThrown(bool flushResponse)
         {
             var serviceContext = new TestServiceContext(LoggerFactory);
@@ -998,7 +996,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
                 if (flushResponse)
                 {
-                    await httpContext.Response.Body.FlushAsync();
+                    await httpContext.Response.BodyPipe.FlushAsync();
                 }
             }, serviceContext))
             {
@@ -1165,7 +1163,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 httpContext.Response.ContentLength = 12;
-                httpContext.Response.Body.Write(Encoding.ASCII.GetBytes("hello, world"), 0, 12);
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("hello, world"), 0, 12));
                 await flushed.Task;
             }, serviceContext))
             {
@@ -1408,7 +1406,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
         {
             var serviceContext = new TestServiceContext(LoggerFactory) { ServerOptions = { AllowSynchronousIO = true } };
 
-            using (var server = new TestServer(httpContext =>
+            using (var server = new TestServer(async httpContext =>
             {
                 httpContext.Response.OnStarting(() =>
                 {
@@ -1421,8 +1419,50 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 httpContext.Response.ContentLength = response.Length - 1;
 
                 // If OnStarting is not run before verifying writes, an error response will be sent.
-                httpContext.Response.Body.Write(response, 0, response.Length);
-                return Task.CompletedTask;
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, 0, response.Length));
+            }, serviceContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        $"Transfer-Encoding: chunked",
+                        "",
+                        "c",
+                        "hello, world",
+                        "0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task FirstWriteVerifiedAfterOnStartingWithResponseBody()
+        {
+            var serviceContext = new TestServiceContext(LoggerFactory) { ServerOptions = { AllowSynchronousIO = true } };
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.OnStarting(() =>
+                {
+                    // Change response to chunked
+                    httpContext.Response.ContentLength = null;
+                    return Task.CompletedTask;
+                });
+
+                var response = Encoding.ASCII.GetBytes("hello, world");
+                httpContext.Response.ContentLength = response.Length - 1;
+
+                // If OnStarting is not run before verifying writes, an error response will be sent.
+                await httpContext.Response.Body.WriteAsync(new Memory<byte>(response, 0, response.Length));
             }, serviceContext))
             {
                 using (var connection = server.CreateConnection())
@@ -1452,7 +1492,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
         {
             var serviceContext = new TestServiceContext(LoggerFactory) { ServerOptions = { AllowSynchronousIO = true } };
 
-            using (var server = new TestServer(httpContext =>
+            using (var server = new TestServer(async httpContext =>
             {
                 httpContext.Response.OnStarting(() =>
                 {
@@ -1465,9 +1505,54 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 httpContext.Response.ContentLength = response.Length - 1;
 
                 // If OnStarting is not run before verifying writes, an error response will be sent.
-                httpContext.Response.Body.Write(response, 0, response.Length / 2);
-                httpContext.Response.Body.Write(response, response.Length / 2, response.Length - response.Length / 2);
-                return Task.CompletedTask;
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, 0, response.Length / 2));
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, response.Length / 2, response.Length - response.Length / 2));
+            }, serviceContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        $"Transfer-Encoding: chunked",
+                        "",
+                        "6",
+                        "hello,",
+                        "6",
+                        " world",
+                        "0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task SubsequentWriteVerifiedAfterOnStartingWithResponseBody()
+        {
+            var serviceContext = new TestServiceContext(LoggerFactory) { ServerOptions = { AllowSynchronousIO = true } };
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.OnStarting(() =>
+                {
+                    // Change response to chunked
+                    httpContext.Response.ContentLength = null;
+                    return Task.CompletedTask;
+                });
+
+                var response = Encoding.ASCII.GetBytes("hello, world");
+                httpContext.Response.ContentLength = response.Length - 1;
+
+                // If OnStarting is not run before verifying writes, an error response will be sent.
+                await httpContext.Response.Body.WriteAsync(new Memory<byte>(response, 0, response.Length / 2));
+                await httpContext.Response.Body.WriteAsync(new Memory<byte>(response, response.Length / 2, response.Length - response.Length / 2));
             }, serviceContext))
             {
                 using (var connection = server.CreateConnection())
@@ -1510,7 +1595,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 httpContext.Response.ContentLength = response.Length - 1;
 
                 // If OnStarting is not run before verifying writes, an error response will be sent.
-                return httpContext.Response.Body.WriteAsync(response, 0, response.Length);
+                return httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, 0, response.Length)).AsTask();
             }, new TestServiceContext(LoggerFactory)))
             {
                 using (var connection = server.CreateConnection())
@@ -1551,8 +1636,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 httpContext.Response.ContentLength = response.Length - 1;
 
                 // If OnStarting is not run before verifying writes, an error response will be sent.
-                await httpContext.Response.Body.WriteAsync(response, 0, response.Length / 2);
-                await httpContext.Response.Body.WriteAsync(response, response.Length / 2, response.Length - response.Length / 2);
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, 0, response.Length / 2));
+                await httpContext.Response.BodyPipe.WriteAsync(new Memory<byte>(response, response.Length / 2, response.Length - response.Length / 2));
             }, new TestServiceContext(LoggerFactory)))
             {
                 using (var connection = server.CreateConnection())
@@ -2112,7 +2197,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                     throw onStartingException;
                 }, null);
 
-                var writeException = await Assert.ThrowsAsync<ObjectDisposedException>(async () => await response.Body.FlushAsync());
+                var writeException = await Assert.ThrowsAsync<ObjectDisposedException>(async () => await response.BodyPipe.FlushAsync());
                 Assert.Same(onStartingException, writeException.InnerException);
             }, testContext))
             {
@@ -2171,7 +2256,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
                 response.Headers["Content-Length"] = new[] { "11" };
 
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World"), 0, 11);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello World"), 0, 11));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -2218,7 +2303,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
                 response.Headers["Content-Length"] = new[] { "11" };
 
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World"), 0, 11);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello World"), 0, 11));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -2264,7 +2349,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
 
                 response.Headers["Content-Length"] = new[] { "11" };
 
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World"), 0, 11);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello World"), 0, 11));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -2307,7 +2392,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 }, null);
 
                 response.Headers["Content-Length"] = new[] { "11" };
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World"), 0, 11);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello World"), 0, 11));
                 throw new Exception();
             }, testContext))
             {
@@ -2349,7 +2434,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 }, null);
 
                 response.Headers["Content-Length"] = new[] { "11" };
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello"), 0, 5);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello"), 0, 5));
                 throw new Exception();
             }, testContext))
             {
@@ -2384,7 +2469,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             {
                 var response = httpContext.Response;
                 response.Headers["Content-Length"] = new[] { "11" };
-                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World"), 0, 11);
+                await response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello World"), 0, 11));
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -2829,7 +2914,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             using (var server = new TestServer(async httpContext =>
             {
                 await httpContext.Response.StartAsync();
-                await httpContext.Response.Body.FlushAsync();
+                await httpContext.Response.BodyPipe.FlushAsync();
                 Assert.True(httpContext.Response.HasStarted);
             }, testContext))
             {
@@ -2983,7 +3068,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                     var ioEx = Assert.Throws<InvalidOperationException>(() => context.Response.Body.Write(Encoding.ASCII.GetBytes("What!?"), 0, 6));
                     Assert.Equal(CoreStrings.SynchronousWritesDisallowed, ioEx.Message);
 
-                    await context.Response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello2"), 0, 6);
+                    await context.Response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello2"), 0, 6));
                 }
             }, new TestServiceContext(LoggerFactory)))
             {
@@ -3028,7 +3113,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
                 var ioEx = Assert.Throws<InvalidOperationException>(() => context.Response.Body.Write(Encoding.ASCII.GetBytes("What!?"), 0, 6));
                 Assert.Equal(CoreStrings.SynchronousWritesDisallowed, ioEx.Message);
 
-                return context.Response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello!"), 0, 6);
+                return context.Response.BodyPipe.WriteAsync(new Memory<byte>(Encoding.ASCII.GetBytes("Hello!"), 0, 6)).AsTask();
             }, testContext))
             {
                 using (var connection = server.CreateConnection())
@@ -3079,6 +3164,662 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
             }
         }
 
+        [Fact]
+        public async Task AdvanceNegativeValueThrowsArgumentOutOfRangeException()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                Assert.Throws<ArgumentOutOfRangeException>(() => response.BodyPipe.Advance(-1));
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task AdvanceWithTooLargeOfAValueThrowInvalidOperationException()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+
+                await response.StartAsync();
+
+                Assert.Throws<InvalidOperationException>(() => response.BodyPipe.Advance(1));
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task GetMemoryBeforeStartAsyncThrows()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(httpContext =>
+            {
+                Assert.Throws<InvalidOperationException>(() => httpContext.Response.BodyPipe.GetMemory());
+                return Task.CompletedTask;
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ContentLengthWithGetSpanWorks()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+                response.ContentLength = 12;
+                await response.StartAsync();
+
+                void NonAsyncMethod()
+                {
+                    var span = response.BodyPipe.GetSpan(4096);
+                    var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                    fisrtPartOfResponse.CopyTo(span);
+                    response.BodyPipe.Advance(6);
+
+                    var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                    secondPartOfResponse.CopyTo(span.Slice(6));
+                    response.BodyPipe.Advance(6);
+                }
+
+                NonAsyncMethod();
+
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Content-Length: 12",
+                        "",
+                        "Hello World!");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ContentLengthWithGetMemoryWorks()
+        {
+            var testContext = new TestServiceContext(LoggerFactory);
+
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+                response.ContentLength = 12;
+                await response.StartAsync();
+
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("Hello ");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+
+                var secondPartOfResponse = Encoding.ASCII.GetBytes("World!");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(6);
+            }, testContext))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host: ",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {testContext.DateHeaderValue}",
+                        "Content-Length: 12",
+                        "",
+                        "Hello World!");
+                }
+
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseBodyCanWrite()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.ContentLength = 12;
+                await httpContext.Response.Body.WriteAsync(Encoding.ASCII.GetBytes("hello, world"));
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 12",
+                        "",
+                        "hello, world");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseBodyAndResponsePipeWorks()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                var response = httpContext.Response;
+                response.ContentLength = 54;
+                await response.StartAsync();
+                var memory = response.BodyPipe.GetMemory(4096);
+                var fisrtPartOfResponse = Encoding.ASCII.GetBytes("hello,");
+                fisrtPartOfResponse.CopyTo(memory);
+                response.BodyPipe.Advance(6);
+                var secondPartOfResponse = Encoding.ASCII.GetBytes(" world\r\n");
+                secondPartOfResponse.CopyTo(memory.Slice(6));
+                response.BodyPipe.Advance(8);
+
+                await response.Body.WriteAsync(Encoding.ASCII.GetBytes("hello, world\r\n"));
+                await response.BodyPipe.WriteAsync(Encoding.ASCII.GetBytes("hello, world\r\n"));
+                await response.WriteAsync("hello, world");
+
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 54",
+                        "",
+                        "hello, world",
+                        "hello, world",
+                        "hello, world",
+                        "hello, world");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseBodyPipeCompleteWithoutExceptionDoesNotThrow()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.BodyPipe.Complete();
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseBodyPipeCompleteWithoutExceptionWritesDoNotThrow()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.BodyPipe.Complete();
+                await httpContext.Response.WriteAsync("test");
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponsePipeWriterCompleteWithException()
+        {
+            var expectedException = new Exception();
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.BodyPipe.Complete(expectedException);
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        $"HTTP/1.1 500 Internal Server Error",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                    Assert.Contains(TestSink.Writes, w => w.EventId.Id == 13 && w.LogLevel == LogLevel.Error
+                        && w.Exception is ConnectionAbortedException && w.Exception.InnerException == expectedException);
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseCompleteGetMemoryReturnsRentedMemory()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                await httpContext.Response.StartAsync();
+                httpContext.Response.BodyPipe.Complete();
+                var memory = httpContext.Response.BodyPipe.GetMemory(); // Shouldn't throw
+                Assert.Equal(4096, memory.Length);
+
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseCompleteGetMemoryAdvanceInLoopDoesNotThrow()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                await httpContext.Response.StartAsync();
+
+                httpContext.Response.BodyPipe.Complete();
+                for (var i = 0; i < 5; i++)
+                {
+                    var memory = httpContext.Response.BodyPipe.GetMemory(); // Shouldn't throw
+                    httpContext.Response.BodyPipe.Advance(memory.Length);
+                }
+
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Transfer-Encoding: chunked",
+                        "",
+                        "0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseSetBodyAndPipeBodyIsWrapped()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.Body = new MemoryStream();
+                httpContext.Response.BodyPipe = new Pipe().Writer;
+                Assert.IsType<WriteOnlyPipeStream>(httpContext.Response.Body);
+
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseSetBodyToSameValueTwiceGetPipeMultipleTimesDifferentObject()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                var memoryStream = new MemoryStream();
+                httpContext.Response.Body = memoryStream;
+                var bodyPipe1 = httpContext.Response.BodyPipe;
+
+                httpContext.Response.Body = memoryStream;
+                var bodyPipe2 = httpContext.Response.BodyPipe;
+
+                Assert.NotEqual(bodyPipe1, bodyPipe2);
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseSetPipeToSameValueTwiceGetBodyMultipleTimesDifferent()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                var pipeWriter = new Pipe().Writer;
+                httpContext.Response.BodyPipe = pipeWriter;
+                var body1 = httpContext.Response.Body;
+
+                httpContext.Response.BodyPipe = pipeWriter;
+                var body2 = httpContext.Response.Body;
+
+                Assert.NotEqual(body1, body2);
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseSetPipeAndBodyPipeIsWrapped()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                httpContext.Response.BodyPipe = new Pipe().Writer;
+                httpContext.Response.Body = new MemoryStream();
+                Assert.IsType<StreamPipeWriter>(httpContext.Response.BodyPipe);
+                Assert.IsType<MemoryStream>(httpContext.Response.Body);
+
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseWriteToBodyPipeAndStreamAllBlocksDisposed()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                for (var i = 0; i < 3; i++)
+                {
+                    httpContext.Response.BodyPipe = new Pipe().Writer;
+                    await httpContext.Response.Body.WriteAsync(new byte[1]);
+                    httpContext.Response.Body = new MemoryStream();
+                    await httpContext.Response.BodyPipe.WriteAsync(new byte[1]);
+                }
+
+                // TestMemoryPool will confirm that all rented blocks have been disposed, meaning dispose was called.
+
+                await Task.CompletedTask;
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponseStreamWrappingWorks()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                var oldBody = httpContext.Response.Body;
+                httpContext.Response.Body = new MemoryStream();
+
+                await httpContext.Response.BodyPipe.WriteAsync(new byte[1]);
+                await httpContext.Response.Body.WriteAsync(new byte[1]);
+
+                Assert.Equal(2, httpContext.Response.Body.Length);
+
+                httpContext.Response.Body = oldBody;
+
+                // Even though we are restoring the original response body, we will create a
+                // wrapper rather than restoring the original pipe.
+                Assert.IsType<StreamPipeWriter>(httpContext.Response.BodyPipe);
+
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task ResponsePipeWrappingWorks()
+        {
+            using (var server = new TestServer(async httpContext =>
+            {
+                var oldPipeWriter = httpContext.Response.BodyPipe;
+                var pipe = new Pipe();
+                httpContext.Response.BodyPipe = pipe.Writer;
+
+                await httpContext.Response.Body.WriteAsync(new byte[1]);
+                await httpContext.Response.BodyPipe.WriteAsync(new byte[1]);
+
+                var readResult = await pipe.Reader.ReadAsync();
+                Assert.Equal(2, readResult.Buffer.Length);
+
+                httpContext.Response.BodyPipe = oldPipeWriter;
+
+                // Even though we are restoring the original response body, we will create a
+                // wrapper rather than restoring the original pipe.
+                Assert.IsType<WriteOnlyPipeStream>(httpContext.Response.Body);
+            }, new TestServiceContext(LoggerFactory)))
+            {
+                using (var connection = server.CreateConnection())
+                {
+                    await connection.Send(
+                        "GET / HTTP/1.1",
+                        "Host:",
+                        "",
+                        "");
+                    await connection.Receive(
+                        "HTTP/1.1 200 OK",
+                        $"Date: {server.Context.DateHeaderValue}",
+                        "Content-Length: 0",
+                        "",
+                        "");
+                }
+                await server.StopAsync();
+            }
+        }
+
         private static async Task ResponseStatusCodeSetBeforeHttpContextDispose(
             ITestSink testSink,
             ILoggerFactory loggerFactory,

src/Servers/Kestrel/tools/CodeGenerator/HttpProtocolFeatureCollection.cs

@@ -13,6 +13,7 @@ namespace CodeGenerator
             {
                 "IHttpRequestFeature",
                 "IHttpResponseFeature",
+                "IResponseBodyPipeFeature",
                 "IHttpRequestIdentifierFeature",
                 "IServiceProvidersFeature",
                 "IHttpRequestLifetimeFeature",
@@ -60,6 +61,7 @@ namespace CodeGenerator
             {
                 "IHttpRequestFeature",
                 "IHttpResponseFeature",
+                "IResponseBodyPipeFeature",
                 "IHttpUpgradeFeature",
                 "IHttpRequestIdentifierFeature",
                 "IHttpRequestLifetimeFeature",
@@ -68,7 +70,7 @@ namespace CodeGenerator
                 "IHttpBodyControlFeature",
                 "IHttpResponseStartFeature"
             };
-
+            
             var usings = $@"
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Http.Features.Authentication;