HttpSys HTTP/2 Response Trailers #13893 (#16877)

src/Servers/HttpSys/src/FeatureContext.cs

@@ -35,7 +35,8 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         IHttpRequestIdentifierFeature,
         IHttpMaxRequestBodySizeFeature,
         IHttpBodyControlFeature,
-        IHttpSysRequestInfoFeature
+        IHttpSysRequestInfoFeature,
+        IHttpResponseTrailersFeature
     {
         private RequestContext _requestContext;
         private IFeatureCollection _features;
@@ -63,6 +64,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         private PipeWriter _pipeWriter;
         private bool _bodyCompleted;
         private IHeaderDictionary _responseHeaders;
+        private IHeaderDictionary _responseTrailers;
 
         private Fields _initializedFields;
 
@@ -346,6 +348,15 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             return Request.IsHttps ? this : null;
         }
 
+        internal IHttpResponseTrailersFeature GetResponseTrailersFeature()
+        {
+            if (Request.ProtocolVersion >= HttpVersion.Version20 && ComNetOS.SupportsTrailers)
+            {
+                return this;
+            }
+            return null;
+        }
+
         /* TODO: https://github.com/aspnet/HttpSysServer/issues/231
         byte[] ITlsTokenBindingFeature.GetProvidedTokenBindingId() => Request.GetProvidedTokenBindingId();
 
@@ -547,6 +558,12 @@ namespace Microsoft.AspNetCore.Server.HttpSys
 
         IReadOnlyDictionary<int, ReadOnlyMemory<byte>> IHttpSysRequestInfoFeature.RequestInfo => Request.RequestInfo;
 
+        IHeaderDictionary IHttpResponseTrailersFeature.Trailers
+        {
+            get => _responseTrailers ??= Response.Trailers;
+            set => _responseTrailers = value;
+        }
+
         internal async Task OnResponseStart()
         {
             if (_responseStarted)

src/Servers/HttpSys/src/MessagePump.cs

@@ -234,7 +234,9 @@ namespace Microsoft.AspNetCore.Server.HttpSys
                     {
                         // We haven't sent a response yet, try to send a 500 Internal Server Error
                         requestContext.Response.Headers.IsReadOnly = false;
+                        requestContext.Response.Trailers.IsReadOnly = false;
                         requestContext.Response.Headers.Clear();
+                        requestContext.Response.Trailers.Clear();
                         SetFatalResponse(requestContext, 500);
                     }
                 }

src/Servers/HttpSys/src/NativeInterop/ComNetOS.cs

@@ -11,11 +11,15 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         // Minimum support for Windows 7 is assumed.
         internal static readonly bool IsWin8orLater;
 
+        internal static readonly bool SupportsTrailers;
+
         static ComNetOS()
         {
             var win8Version = new Version(6, 2);
 
             IsWin8orLater = (Environment.OSVersion.Version >= win8Version);
+
+            SupportsTrailers = Environment.OSVersion.Version >= new Version(10, 0, 19505);
         }
     }
 }

src/Servers/HttpSys/src/RequestProcessing/Response.cs

@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Net;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -28,6 +29,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         private long _expectedBodyLength;
         private BoundaryType _boundaryType;
         private HttpApiTypes.HTTP_RESPONSE_V2 _nativeResponse;
+        private HeaderCollection _trailers;
 
         internal Response(RequestContext requestContext)
         {
@@ -142,6 +144,16 @@ namespace Microsoft.AspNetCore.Server.HttpSys
 
         public HeaderCollection Headers { get; }
 
+        public HeaderCollection Trailers => _trailers ??= new HeaderCollection(checkTrailers: true) { IsReadOnly = BodyIsFinished };
+
+        internal bool HasTrailers => _trailers?.Count > 0;
+
+        // Trailers are supported on this OS, it's HTTP/2, and the app added a Trailer response header to announce trailers were intended.
+        // Needed to delay the completion of Content-Length responses.
+        internal bool TrailersExpected => HasTrailers
+            || (ComNetOS.SupportsTrailers && Request.ProtocolVersion >= HttpVersion.Version20
+                    && Headers.ContainsKey(HttpKnownHeaderNames.Trailer));
+
         internal long ExpectedBodyLength
         {
             get { return _expectedBodyLength; }
@@ -168,6 +180,16 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             }
         }
 
+        // The response is being finished with or without trailers. Mark them as readonly to inform
+        // callers if they try to add them too late. E.g. after Content-Length or CompleteAsync().
+        internal void MakeTrailersReadOnly()
+        {
+            if (_trailers != null)
+            {
+                _trailers.IsReadOnly = true;
+            }
+        }
+
         internal void Abort()
         {
             // Update state for HasStarted. Do not attempt a graceful Dispose.
@@ -400,7 +422,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
                 _boundaryType = BoundaryType.ContentLength;
                 // ComputeLeftToWrite checks for HEAD requests when setting _leftToWrite
                 _expectedBodyLength = responseContentLength.Value;
-                if (_expectedBodyLength == writeCount && !isHeadRequest)
+                if (_expectedBodyLength == writeCount && !isHeadRequest && !TrailersExpected)
                 {
                     // A single write with the whole content-length. Http.Sys will set the content-length for us in this scenario.
                     // If we don't remove it then range requests served from cache will have two.
@@ -430,6 +452,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             else
             {
                 // v1.0 and the length cannot be determined, so we must close the connection after writing data
+                // Or v2.0 and chunking isn't required.
                 keepConnectionAlive = false;
                 _boundaryType = BoundaryType.Close;
             }
@@ -622,6 +645,73 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             }
         }
 
+        internal unsafe void SerializeTrailers(HttpApiTypes.HTTP_DATA_CHUNK[] dataChunks, int currentChunk, List<GCHandle> pins)
+        {
+            Debug.Assert(currentChunk == dataChunks.Length - 1);
+            Debug.Assert(HasTrailers);
+            MakeTrailersReadOnly();
+            var trailerCount = 0;
+
+            foreach (var trailerPair in Trailers)
+            {
+                trailerCount += trailerPair.Value.Count;
+            }
+
+            var pinnedHeaders = new List<GCHandle>();
+
+            var unknownHeaders = new HttpApiTypes.HTTP_UNKNOWN_HEADER[trailerCount];
+            var gcHandle = GCHandle.Alloc(unknownHeaders, GCHandleType.Pinned);
+            pinnedHeaders.Add(gcHandle);
+            dataChunks[currentChunk].DataChunkType = HttpApiTypes.HTTP_DATA_CHUNK_TYPE.HttpDataChunkTrailers;
+            dataChunks[currentChunk].trailers.trailerCount = (ushort)trailerCount;
+            dataChunks[currentChunk].trailers.pTrailers = gcHandle.AddrOfPinnedObject();
+
+            try
+            {
+                var unknownHeadersOffset = 0;
+
+                foreach (var headerPair in Trailers)
+                {
+                    if (headerPair.Value.Count == 0)
+                    {
+                        continue;
+                    }
+
+                    var headerName = headerPair.Key;
+                    var headerValues = headerPair.Value;
+
+                    for (int headerValueIndex = 0; headerValueIndex < headerValues.Count; headerValueIndex++)
+                    {
+                        // Add Name
+                        var bytes = HeaderEncoding.GetBytes(headerName);
+                        unknownHeaders[unknownHeadersOffset].NameLength = (ushort)bytes.Length;
+                        gcHandle = GCHandle.Alloc(bytes, GCHandleType.Pinned);
+                        pinnedHeaders.Add(gcHandle);
+                        unknownHeaders[unknownHeadersOffset].pName = (byte*)gcHandle.AddrOfPinnedObject();
+
+                        // Add Value
+                        var headerValue = headerValues[headerValueIndex] ?? string.Empty;
+                        bytes = HeaderEncoding.GetBytes(headerValue);
+                        unknownHeaders[unknownHeadersOffset].RawValueLength = (ushort)bytes.Length;
+                        gcHandle = GCHandle.Alloc(bytes, GCHandleType.Pinned);
+                        pinnedHeaders.Add(gcHandle);
+                        unknownHeaders[unknownHeadersOffset].pRawValue = (byte*)gcHandle.AddrOfPinnedObject();
+                        unknownHeadersOffset++;
+                    }
+                }
+
+                Debug.Assert(unknownHeadersOffset == trailerCount);
+            }
+            catch
+            {
+                FreePinnedHeaders(pinnedHeaders);
+                throw;
+            }
+
+            // Success, keep the pins.
+            pins.AddRange(pinnedHeaders);
+        }
+
         // Subset of ComputeHeaders
         internal void SendOpaqueUpgrade()
         {

src/Servers/HttpSys/src/RequestProcessing/ResponseBody.cs

@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
+using System.Net;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -187,26 +188,34 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         private List<GCHandle> PinDataBuffers(bool endOfRequest, ArraySegment<byte> data, out HttpApiTypes.HTTP_DATA_CHUNK[] dataChunks)
         {
             var pins = new List<GCHandle>();
+            var hasData = data.Count > 0;
             var chunked = _requestContext.Response.BoundaryType == BoundaryType.Chunked;
+            var addTrailers = endOfRequest && _requestContext.Response.HasTrailers;
+            Debug.Assert(!(addTrailers && chunked), "Trailers aren't currently supported for HTTP/1.1 chunking.");
 
             var currentChunk = 0;
             // Figure out how many data chunks
-            if (chunked && data.Count == 0 && endOfRequest)
+            if (chunked && !hasData && endOfRequest)
             {
                 dataChunks = new HttpApiTypes.HTTP_DATA_CHUNK[1];
                 SetDataChunk(dataChunks, ref currentChunk, pins, new ArraySegment<byte>(Helpers.ChunkTerminator));
                 return pins;
             }
-            else if (data.Count == 0)
+            else if (!hasData && !addTrailers)
             {
                 // No data
                 dataChunks = new HttpApiTypes.HTTP_DATA_CHUNK[0];
                 return pins;
             }
 
-            var chunkCount = 1;
+            var chunkCount = hasData ? 1 : 0;
-            if (chunked)
+            if (addTrailers)
             {
+                chunkCount++;
+            }
+            else if (chunked) // HTTP/1.1 chunking, not currently supported with trailers
+            {
+                Debug.Assert(hasData);
                 // Chunk framing
                 chunkCount += 2;
 
@@ -216,6 +225,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
                     chunkCount += 1;
                 }
             }
+
             dataChunks = new HttpApiTypes.HTTP_DATA_CHUNK[chunkCount];
 
             if (chunked)
@@ -224,7 +234,10 @@ namespace Microsoft.AspNetCore.Server.HttpSys
                 SetDataChunk(dataChunks, ref currentChunk, pins, chunkHeaderBuffer);
             }
 
-            SetDataChunk(dataChunks, ref currentChunk, pins, data);
+            if (hasData)
+            {
+                SetDataChunk(dataChunks, ref currentChunk, pins, data);
+            }
 
             if (chunked)
             {
@@ -236,6 +249,15 @@ namespace Microsoft.AspNetCore.Server.HttpSys
                 }
             }
 
+            if (addTrailers)
+            {
+                _requestContext.Response.SerializeTrailers(dataChunks, currentChunk, pins);
+            }
+            else if (endOfRequest)
+            {
+                _requestContext.Response.MakeTrailersReadOnly();
+            }
+
             return pins;
         }
 
@@ -433,7 +455,8 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             {
                 flags |= HttpApiTypes.HTTP_FLAGS.HTTP_SEND_RESPONSE_FLAG_DISCONNECT;
             }
-            else if (!endOfRequest && _leftToWrite != writeCount)
+            else if (!endOfRequest
+                && (_leftToWrite != writeCount || _requestContext.Response.TrailersExpected))
             {
                 flags |= HttpApiTypes.HTTP_FLAGS.HTTP_SEND_RESPONSE_FLAG_MORE_DATA;
             }
@@ -444,9 +467,10 @@ namespace Microsoft.AspNetCore.Server.HttpSys
                 // keep track of the data transferred
                 _leftToWrite -= writeCount;
             }
-            if (_leftToWrite == 0)
+            if (_leftToWrite == 0 && !_requestContext.Response.TrailersExpected)
             {
                 // in this case we already passed 0 as the flag, so we don't need to call HttpSendResponseEntityBody() when we Close()
+                _requestContext.Response.MakeTrailersReadOnly();
                 _disposed = true;
             }
             // else -1 unlimited

src/Servers/HttpSys/src/StandardFeatureCollection.cs

@@ -27,6 +27,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             { typeof(IHttpMaxRequestBodySizeFeature), _identityFunc },
             { typeof(IHttpBodyControlFeature), _identityFunc },
             { typeof(IHttpSysRequestInfoFeature), _identityFunc },
+            { typeof(IHttpResponseTrailersFeature), ctx => ctx.GetResponseTrailersFeature() },
         };
 
         private readonly FeatureContext _featureContext;

src/Servers/HttpSys/test/FunctionalTests/HttpsTests.cs

@@ -206,16 +206,14 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         private async Task<string> SendRequestAsync(string uri,
             X509Certificate cert = null)
         {
-            var handler = new WinHttpHandler();
+            var handler = new HttpClientHandler();
-            handler.ServerCertificateValidationCallback = (a, b, c, d) => true;
+            handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
             if (cert != null)
             {
                 handler.ClientCertificates.Add(cert);
             }
-            using (HttpClient client = new HttpClient(handler))
+            using HttpClient client = new HttpClient(handler);
-            {
+            return await client.GetStringAsync(uri);
-                return await client.GetStringAsync(uri);
-            }
         }
 
         private async Task<string> SendRequestAsync(string uri, string upload)

src/Servers/HttpSys/test/FunctionalTests/ResponseTrailersTests.cs

@@ -0,0 +1,368 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.HttpSys.Internal;
+using Microsoft.AspNetCore.Testing;
+using Microsoft.Extensions.Primitives;
+using Microsoft.Net.Http.Headers;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Server.HttpSys
+{
+    public class ResponseTrailersTests
+    {
+        [ConditionalFact]
+        public async Task ResponseTrailers_HTTP11_TrailersNotAvailable()
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                Assert.Equal("HTTP/1.1", httpContext.Request.Protocol);
+                Assert.False(httpContext.Response.SupportsTrailers());
+                return Task.FromResult(0);
+            }))
+            {
+                var response = await SendRequestAsync(address, http2: false);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version11, response.Version);
+                Assert.Empty(response.TrailingHeaders);
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_HTTP2_TrailersAvailable()
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                Assert.Equal("HTTP/2", httpContext.Request.Protocol);
+                Assert.True(httpContext.Response.SupportsTrailers());
+                return Task.FromResult(0);
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.Empty(response.TrailingHeaders);
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_ProhibitedTrailers_Blocked()
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                Assert.True(httpContext.Response.SupportsTrailers());
+                foreach (var header in HeaderCollection.DisallowedTrailers)
+                {
+                    Assert.Throws<InvalidOperationException>(() => httpContext.Response.AppendTrailer(header, "value"));
+                }
+                return Task.FromResult(0);
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.Empty(response.TrailingHeaders);
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_NoBody_TrailersSent()
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                httpContext.Response.DeclareTrailer("trailername");
+                httpContext.Response.AppendTrailer("trailername", "TrailerValue");
+                return Task.FromResult(0);
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.NotEmpty(response.TrailingHeaders);
+                Assert.Equal("TrailerValue", response.TrailingHeaders.GetValues("TrailerName").Single());
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_WithBody_TrailersSent()
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                await httpContext.Response.WriteAsync("Hello World");
+                httpContext.Response.AppendTrailer("TrailerName", "Trailer Value");
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.Equal("Hello World", await response.Content.ReadAsStringAsync());
+                Assert.NotEmpty(response.TrailingHeaders);
+                Assert.Equal("Trailer Value", response.TrailingHeaders.GetValues("TrailerName").Single());
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_WithContentLengthBody_TrailersNotSent()
+        {
+            var body = "Hello World";
+            var responseFinished = new TaskCompletionSource<int>(TaskCreationOptions.RunContinuationsAsynchronously);
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                httpContext.Response.ContentLength = body.Length;
+                await httpContext.Response.WriteAsync(body);
+                try
+                {
+                    Assert.Throws<InvalidOperationException>(() => httpContext.Response.AppendTrailer("TrailerName", "Trailer Value"));
+                    responseFinished.SetResult(0);
+                }
+                catch (Exception ex)
+                {
+                    responseFinished.SetException(ex);
+                }
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.Equal(body.Length.ToString(CultureInfo.InvariantCulture), response.Content.Headers.GetValues(HeaderNames.ContentLength).Single());
+                Assert.Equal(body, await response.Content.ReadAsStringAsync());
+                Assert.Empty(response.TrailingHeaders);
+                await responseFinished.Task;
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_WithTrailersBeforeContentLengthBody_TrailersSent()
+        {
+            var body = "Hello World";
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                httpContext.Response.ContentLength = body.Length * 2;
+                await httpContext.Response.WriteAsync(body);
+                httpContext.Response.AppendTrailer("TrailerName", "Trailer Value");
+                await httpContext.Response.WriteAsync(body);
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                // Avoid HttpContent's automatic content-length calculation.
+                Assert.True(response.Content.Headers.TryGetValues(HeaderNames.ContentLength, out var contentLength), HeaderNames.ContentLength);
+                Assert.Equal((2 * body.Length).ToString(CultureInfo.InvariantCulture), contentLength.First());
+                Assert.Equal(body + body, await response.Content.ReadAsStringAsync());
+                Assert.NotEmpty(response.TrailingHeaders);
+                Assert.Equal("Trailer Value", response.TrailingHeaders.GetValues("TrailerName").Single());
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_WithContentLengthBodyAndDeclared_TrailersSent()
+        {
+            var body = "Hello World";
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                httpContext.Response.ContentLength = body.Length;
+                httpContext.Response.DeclareTrailer("TrailerName");
+                await httpContext.Response.WriteAsync(body);
+                httpContext.Response.AppendTrailer("TrailerName", "Trailer Value");
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                // Avoid HttpContent's automatic content-length calculation.
+                Assert.True(response.Content.Headers.TryGetValues(HeaderNames.ContentLength, out var contentLength), HeaderNames.ContentLength);
+                Assert.Equal(body.Length.ToString(CultureInfo.InvariantCulture), contentLength.First());
+                Assert.Equal("TrailerName", response.Headers.Trailer.Single());
+                Assert.Equal(body, await response.Content.ReadAsStringAsync());
+                Assert.NotEmpty(response.TrailingHeaders);
+                Assert.Equal("Trailer Value", response.TrailingHeaders.GetValues("TrailerName").Single());
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_WithContentLengthBodyAndDeclaredButMissingTrailers_Completes()
+        {
+            var body = "Hello World";
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                httpContext.Response.ContentLength = body.Length;
+                httpContext.Response.DeclareTrailer("TrailerName");
+                await httpContext.Response.WriteAsync(body);
+                // If we declare trailers but don't send any make sure it completes anyways.
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                // Avoid HttpContent's automatic content-length calculation.
+                Assert.True(response.Content.Headers.TryGetValues(HeaderNames.ContentLength, out var contentLength), HeaderNames.ContentLength);
+                Assert.Equal(body.Length.ToString(CultureInfo.InvariantCulture), contentLength.First());
+                Assert.Equal("TrailerName", response.Headers.Trailer.Single());
+                Assert.Equal(body, await response.Content.ReadAsStringAsync());
+                Assert.Empty(response.TrailingHeaders);
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_CompleteAsyncNoBody_TrailersSent()
+        {
+            var trailersReceived = new TaskCompletionSource<int>(TaskCreationOptions.RunContinuationsAsynchronously);
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                httpContext.Response.AppendTrailer("trailername", "TrailerValue");
+                await httpContext.Response.CompleteAsync();
+                await trailersReceived.Task.WithTimeout();
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.NotEmpty(response.TrailingHeaders);
+                Assert.Equal("TrailerValue", response.TrailingHeaders.GetValues("TrailerName").Single());
+                trailersReceived.SetResult(0);
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_CompleteAsyncWithBody_TrailersSent()
+        {
+            var trailersReceived = new TaskCompletionSource<int>(TaskCreationOptions.RunContinuationsAsynchronously);
+            using (Utilities.CreateDynamicHttpsServer(out var address, async httpContext =>
+            {
+                await httpContext.Response.WriteAsync("Hello World");
+                httpContext.Response.AppendTrailer("TrailerName", "Trailer Value");
+                await httpContext.Response.CompleteAsync();
+                await trailersReceived.Task.WithTimeout();
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.Equal("Hello World", await response.Content.ReadAsStringAsync());
+                Assert.NotEmpty(response.TrailingHeaders);
+                Assert.Equal("Trailer Value", response.TrailingHeaders.GetValues("TrailerName").Single());
+                trailersReceived.SetResult(0);
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_MultipleValues_SentAsSeperateHeaders()
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                httpContext.Response.AppendTrailer("trailername", new StringValues(new[] { "TrailerValue0", "TrailerValue1" }));
+                return Task.FromResult(0);
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.NotEmpty(response.TrailingHeaders);
+                // We can't actually assert they are sent as seperate headers using HttpClient, we'd have to write a lower level test
+                // that read the header frames directly.
+                Assert.Equal(new[] { "TrailerValue0", "TrailerValue1" }, response.TrailingHeaders.GetValues("TrailerName"));
+            }
+        }
+
+        [ConditionalFact]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_LargeTrailers_Success()
+        {
+            var values = new[] {
+                        new string('a', 1024),
+                        new string('b', 1024 * 4),
+                        new string('c', 1024 * 8),
+                        new string('d', 1024 * 16),
+                        new string('e', 1024 * 32),
+                        new string('f', 1024 * 64 - 1) }; // Max header size
+
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                httpContext.Response.AppendTrailer("ThisIsALongerHeaderNameThatStillWorksForReals", new StringValues(values));
+                return Task.FromResult(0);
+            }))
+            {
+                var response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.Equal(HttpVersion.Version20, response.Version);
+                Assert.NotEmpty(response.TrailingHeaders);
+                // We can't actually assert they are sent in multiple frames using HttpClient, we'd have to write a lower level test
+                // that read the header frames directly. We at least verify that really large values work.
+                Assert.Equal(values, response.TrailingHeaders.GetValues("ThisIsALongerHeaderNameThatStillWorksForReals"));
+            }
+        }
+
+        [ConditionalTheory, MemberData(nameof(NullHeaderData))]
+        [MinimumOSVersion(OperatingSystems.Windows, "10.0.19505", SkipReason = "Requires HTTP/2 Trailers support.")]
+        public async Task ResponseTrailers_NullValues_Ignored(string headerName, StringValues headerValue, StringValues expectedValue)
+        {
+            using (Utilities.CreateDynamicHttpsServer(out var address, httpContext =>
+            {
+                httpContext.Response.AppendTrailer(headerName, headerValue);
+                return Task.FromResult(0);
+            }))
+            {
+                HttpResponseMessage response = await SendRequestAsync(address);
+                response.EnsureSuccessStatusCode();
+                var headers = response.TrailingHeaders;
+
+                if (StringValues.IsNullOrEmpty(expectedValue))
+                {
+                    Assert.False(headers.Contains(headerName));
+                }
+                else
+                {
+                    Assert.True(headers.Contains(headerName));
+                    Assert.Equal(headers.GetValues(headerName), expectedValue);
+                }
+            }
+        }
+
+        public static TheoryData<string, StringValues, StringValues> NullHeaderData
+        {
+            get
+            {
+                var dataset = new TheoryData<string, StringValues, StringValues>();
+
+                dataset.Add("NullString", (string)null, (string)null);
+                dataset.Add("EmptyString", "", "");
+                dataset.Add("NullStringArray", new string[] { null }, "");
+                dataset.Add("EmptyStringArray", new string[] { "" }, "");
+                dataset.Add("MixedStringArray", new string[] { null, "" }, new string[] { "", "" });
+                dataset.Add("WithValidStrings", new string[] { null, "Value", "" }, new string[] { "", "Value", "" });
+
+                return dataset;
+            }
+        }
+
+        private async Task<HttpResponseMessage> SendRequestAsync(string uri, bool http2 = true)
+        {
+            var handler = new HttpClientHandler();
+            handler.MaxResponseHeadersLength = 128;
+            handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
+            using HttpClient client = new HttpClient(handler);
+            client.DefaultRequestVersion = http2 ? HttpVersion.Version20 : HttpVersion.Version11;
+            return await client.GetAsync(uri);
+        }
+    }
+}

src/Shared/HttpSys/NativeInterop/HttpApiTypes.cs

@@ -88,6 +88,9 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
 
             [FieldOffset(8)]
             internal FromFileHandle fromFile;
+
+            [FieldOffset(8)]
+            internal Trailers trailers;
         }
 
         [StructLayout(LayoutKind.Sequential)]
@@ -106,6 +109,13 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
             internal IntPtr fileHandle;
         }
 
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct Trailers
+        {
+            internal ushort trailerCount;
+            internal IntPtr pTrailers;
+        }
+
         [StructLayout(LayoutKind.Sequential)]
         internal struct HTTPAPI_VERSION
         {
@@ -362,10 +372,12 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
 
         internal enum HTTP_DATA_CHUNK_TYPE : int
         {
-            HttpDataChunkFromMemory = 0,
+            HttpDataChunkFromMemory,
-            HttpDataChunkFromFileHandle = 1,
+            HttpDataChunkFromFileHandle,
-            HttpDataChunkFromFragmentCache = 2,
+            HttpDataChunkFromFragmentCache,
-            HttpDataChunkMaximum = 3,
+            HttpDataChunkFromFragmentCacheEx,
+            HttpDataChunkTrailers,
+            HttpDataChunkMaximum,
         }
 
         [StructLayout(LayoutKind.Sequential)]

src/Shared/HttpSys/RequestProcessing/HeaderCollection.cs

@@ -12,12 +12,42 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
 {
     internal class HeaderCollection : IHeaderDictionary
     {
+        // https://tools.ietf.org/html/rfc7230#section-4.1.2
+        internal static readonly HashSet<string> DisallowedTrailers = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+        {
+            // Message framing headers.
+            HeaderNames.TransferEncoding, HeaderNames.ContentLength,
+
+            // Routing headers.
+            HeaderNames.Host,
+
+            // Request modifiers: controls and conditionals.
+            // rfc7231#section-5.1: Controls.
+            HeaderNames.CacheControl, HeaderNames.Expect, HeaderNames.MaxForwards, HeaderNames.Pragma, HeaderNames.Range, HeaderNames.TE,
+
+            // rfc7231#section-5.2: Conditionals.
+            HeaderNames.IfMatch, HeaderNames.IfNoneMatch, HeaderNames.IfModifiedSince, HeaderNames.IfUnmodifiedSince, HeaderNames.IfRange,
+
+            // Authentication headers.
+            HeaderNames.WWWAuthenticate, HeaderNames.Authorization, HeaderNames.ProxyAuthenticate, HeaderNames.ProxyAuthorization, HeaderNames.SetCookie, HeaderNames.Cookie,
+
+            // Response control data.
+            // rfc7231#section-7.1: Control Data.
+            HeaderNames.Age, HeaderNames.Expires, HeaderNames.Date, HeaderNames.Location, HeaderNames.RetryAfter, HeaderNames.Vary, HeaderNames.Warning,
+
+            // Content-Encoding, Content-Type, Content-Range, and Trailer itself.
+            HeaderNames.ContentEncoding, HeaderNames.ContentType, HeaderNames.ContentRange, HeaderNames.Trailer
+        };
+
+        // Should this instance check for prohibited trailers?
+        private readonly bool _checkTrailers;
         private long? _contentLength;
         private StringValues _contentLengthText;
 
-        public HeaderCollection()
+        public HeaderCollection(bool checkTrailers = false)
             : this(new Dictionary<string, StringValues>(4, StringComparer.OrdinalIgnoreCase))
         {
+            _checkTrailers = checkTrailers;
         }
 
         public HeaderCollection(IDictionary<string, StringValues> store)
@@ -39,6 +69,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
             }
             set
             {
+                ValidateRestrictedTrailers(key);
                 ThrowIfReadOnly();
                 if (StringValues.IsNullOrEmpty(value))
                 {
@@ -58,6 +89,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
             get { return Store[key]; }
             set
             {
+                ValidateRestrictedTrailers(key);
                 ThrowIfReadOnly();
                 ValidateHeaderCharacters(key);
                 ValidateHeaderCharacters(value);
@@ -105,6 +137,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
             }
             set
             {
+                ValidateRestrictedTrailers(HeaderNames.ContentLength);
                 ThrowIfReadOnly();
 
                 if (value.HasValue)
@@ -128,6 +161,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
 
         public void Add(KeyValuePair<string, StringValues> item)
         {
+            ValidateRestrictedTrailers(item.Key);
             ThrowIfReadOnly();
             ValidateHeaderCharacters(item.Key);
             ValidateHeaderCharacters(item.Value);
@@ -136,6 +170,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
 
         public void Add(string key, StringValues value)
         {
+            ValidateRestrictedTrailers(key);
             ThrowIfReadOnly();
             ValidateHeaderCharacters(key);
             ValidateHeaderCharacters(value);
@@ -144,6 +179,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
 
         public void Append(string key, string value)
         {
+            ValidateRestrictedTrailers(key);
             ThrowIfReadOnly();
             ValidateHeaderCharacters(key);
             ValidateHeaderCharacters(value);
@@ -214,6 +250,11 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
         {
             if (IsReadOnly)
             {
+                if (_checkTrailers)
+                {
+                    throw new InvalidOperationException("The response trailers cannot be modified because the response has already completed. "
+                        + "If this is a Content-Length response then you need to call HttpResponse.DeclareTrailer before starting the body.");
+                }
                 throw new InvalidOperationException("The response headers cannot be modified because the response has already started.");
             }
         }
@@ -239,5 +280,13 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
                 }
             }
         }
+
+        private void ValidateRestrictedTrailers(string key)
+        {
+            if (_checkTrailers && DisallowedTrailers.Contains(key))
+            {
+                throw new InvalidOperationException($"The '{key}' header is not allowed in HTTP trailers.");
+            }
+        }
     }
-}
+}