Http/2 response trailers #622

benchmarks/Kestrel.Performance/Mocks/MockTrace.cs

@@ -49,6 +49,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
         public void Http2ConnectionError(string connectionId, Http2ConnectionErrorException ex) { }
         public void Http2StreamError(string connectionId, Http2StreamErrorException ex) { }
         public void HPackDecodingError(string connectionId, int streamId, HPackDecodingException ex) { }
+        public void HPackEncodingError(string connectionId, int streamId, HPackEncodingException ex) { }
         public void Http2StreamResetAbort(string traceIdentifier, Http2ErrorCode error, ConnectionAbortedException abortReason) { }
         public void Http2ConnectionClosing(string connectionId) { }
         public void Http2ConnectionClosed(string connectionId, int highestOpenedStreamId) { }

build/dependencies.props

@@ -11,13 +11,13 @@
     <MicrosoftAspNetCoreAllPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreAllPackageVersion>
     <MicrosoftAspNetCoreBenchmarkRunnerSourcesPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreBenchmarkRunnerSourcesPackageVersion>
     <MicrosoftAspNetCoreCertificatesGenerationSourcesPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreCertificatesGenerationSourcesPackageVersion>
-    <MicrosoftAspNetCoreHostingAbstractionsPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreHostingAbstractionsPackageVersion>
+    <MicrosoftAspNetCoreHostingAbstractionsPackageVersion>2.2.0-a-preview3-tratcher-trailers-17154</MicrosoftAspNetCoreHostingAbstractionsPackageVersion>
-    <MicrosoftAspNetCoreHostingPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreHostingPackageVersion>
+    <MicrosoftAspNetCoreHostingPackageVersion>2.2.0-a-preview3-tratcher-trailers-17154</MicrosoftAspNetCoreHostingPackageVersion>
-    <MicrosoftAspNetCoreHttpAbstractionsPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreHttpAbstractionsPackageVersion>
+    <MicrosoftAspNetCoreHttpAbstractionsPackageVersion>2.2.0-a-preview3-tratcher-trailers-16760</MicrosoftAspNetCoreHttpAbstractionsPackageVersion>
-    <MicrosoftAspNetCoreHttpFeaturesPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreHttpFeaturesPackageVersion>
+    <MicrosoftAspNetCoreHttpFeaturesPackageVersion>2.2.0-a-preview3-tratcher-trailers-16760</MicrosoftAspNetCoreHttpFeaturesPackageVersion>
-    <MicrosoftAspNetCoreHttpPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreHttpPackageVersion>
+    <MicrosoftAspNetCoreHttpPackageVersion>2.2.0-a-preview3-tratcher-trailers-16760</MicrosoftAspNetCoreHttpPackageVersion>
     <MicrosoftAspNetCoreTestingPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreTestingPackageVersion>
-    <MicrosoftAspNetCoreWebUtilitiesPackageVersion>2.2.0-preview3-35359</MicrosoftAspNetCoreWebUtilitiesPackageVersion>
+    <MicrosoftAspNetCoreWebUtilitiesPackageVersion>2.2.0-a-preview3-tratcher-trailers-16760</MicrosoftAspNetCoreWebUtilitiesPackageVersion>
     <MicrosoftExtensionsActivatorUtilitiesSourcesPackageVersion>2.2.0-preview3-35359</MicrosoftExtensionsActivatorUtilitiesSourcesPackageVersion>
     <MicrosoftExtensionsConfigurationBinderPackageVersion>2.2.0-preview3-35359</MicrosoftExtensionsConfigurationBinderPackageVersion>
     <MicrosoftExtensionsConfigurationCommandLinePackageVersion>2.2.0-preview3-35359</MicrosoftExtensionsConfigurationCommandLinePackageVersion>

samples/Http2SampleApp/Startup.cs

@@ -14,6 +14,7 @@ namespace Http2SampleApp
 
         public void Configure(IApplicationBuilder app, IHostingEnvironment env)
         {
+            app.UseTimingMiddleware();
             app.Run(context =>
             {
                 return context.Response.WriteAsync("Hello World! " + context.Request.Protocol);

samples/Http2SampleApp/TimingMiddleware.cs

@@ -0,0 +1,50 @@
+using System.Diagnostics;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+
+namespace Http2SampleApp
+{
+    // You may need to install the Microsoft.AspNetCore.Http.Abstractions package into your project
+    public class TimingMiddleware
+    {
+        private readonly RequestDelegate _next;
+
+        public TimingMiddleware(RequestDelegate next)
+        {
+            _next = next;
+        }
+
+        public async Task Invoke(HttpContext httpContext)
+        {
+            if (httpContext.Response.SupportsTrailers())
+            {
+                httpContext.Response.DeclareTrailer("Server-Timing");
+
+                var stopWatch = new Stopwatch();
+                stopWatch.Start();
+
+                await _next(httpContext);
+
+                stopWatch.Stop();
+                // Not yet supported in any browser dev tools
+                httpContext.Response.AppendTrailer("Server-Timing", $"app;dur={stopWatch.ElapsedMilliseconds}.0");
+            }
+            else
+            {
+                // Works in chrome
+                // httpContext.Response.Headers.Append("Server-Timing", $"app;dur=25.0");
+                await _next(httpContext);
+            }
+        }
+    }
+
+    // Extension method used to add the middleware to the HTTP request pipeline.
+    public static class TimingMiddlewareExtensions
+    {
+        public static IApplicationBuilder UseTimingMiddleware(this IApplicationBuilder builder)
+        {
+            return builder.UseMiddleware<TimingMiddleware>();
+        }
+    }
+}

src/Kestrel.Core/Internal/Http/HttpHeaders.Generated.cs

@@ -8998,4 +8998,235 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             }
         }
     }
+
+    public partial class HttpResponseTrailers
+    {
+        private static byte[] _headerBytes = new byte[]
+        {
+            13,10,69,84,97,103,58,32,
+        };
+
+        private long _bits = 0;
+        private HeaderReferences _headers;
+
+
+        
+        public StringValues HeaderETag
+        {
+            get
+            {
+                StringValues value;
+                if ((_bits & 1L) != 0)
+                {
+                    value = _headers._ETag;
+                }
+                return value;
+            }
+            set
+            {
+                _bits |= 1L;
+                _headers._ETag = value; 
+            }
+        }
+
+        protected override int GetCountFast()
+        {
+            return (_contentLength.HasValue ? 1 : 0 ) + BitCount(_bits) + (MaybeUnknown?.Count ?? 0);
+        }
+
+        protected override bool TryGetValueFast(string key, out StringValues value)
+        {
+            switch (key.Length)
+            {
+                case 4:
+                    {
+                        if ("ETag".Equals(key, StringComparison.OrdinalIgnoreCase))
+                        {
+                            if ((_bits & 1L) != 0)
+                            {
+                                value = _headers._ETag;
+                                return true;
+                            }
+                            return false;
+                        }
+                    }
+                    break;
+            }
+
+            return MaybeUnknown?.TryGetValue(key, out value) ?? false;
+        }
+
+        protected override void SetValueFast(string key, in StringValues value)
+        {
+            ValidateHeaderValueCharacters(value);
+            switch (key.Length)
+            {
+                case 4:
+                    {
+                        if ("ETag".Equals(key, StringComparison.OrdinalIgnoreCase))
+                        {
+                            _bits |= 1L;
+                            _headers._ETag = value;
+                            return;
+                        }
+                    }
+                    break;
+            }
+
+            SetValueUnknown(key, value);
+        }
+
+        protected override bool AddValueFast(string key, in StringValues value)
+        {
+            ValidateHeaderValueCharacters(value);
+            switch (key.Length)
+            {
+                case 4:
+                    {
+                        if ("ETag".Equals(key, StringComparison.OrdinalIgnoreCase))
+                        {
+                            if ((_bits & 1L) == 0)
+                            {
+                                _bits |= 1L;
+                                _headers._ETag = value;
+                                return true;
+                            }
+                            return false;
+                        }
+                    }
+                    break;
+            }
+
+            Unknown.Add(key, value);
+            // Return true, above will throw and exit for false
+            return true;
+        }
+
+        protected override bool RemoveFast(string key)
+        {
+            switch (key.Length)
+            {
+                case 4:
+                    {
+                        if ("ETag".Equals(key, StringComparison.OrdinalIgnoreCase))
+                        {
+                            if ((_bits & 1L) != 0)
+                            {
+                                _bits &= ~1L;
+                                _headers._ETag = default(StringValues);
+                                return true;
+                            }
+                            return false;
+                        }
+                    }
+                    break;
+            }
+
+            return MaybeUnknown?.Remove(key) ?? false;
+        }
+
+        protected override void ClearFast()
+        {
+            MaybeUnknown?.Clear();
+            _contentLength = null;
+            var tempBits = _bits;
+            _bits = 0;
+            if(HttpHeaders.BitCount(tempBits) > 12)
+            {
+                _headers = default(HeaderReferences);
+                return;
+            }
+            
+            if ((tempBits & 1L) != 0)
+            {
+                _headers._ETag = default(StringValues);
+                if((tempBits & ~1L) == 0)
+                {
+                    return;
+                }
+                tempBits &= ~1L;
+            }
+            
+        }
+
+        protected override bool CopyToFast(KeyValuePair<string, StringValues>[] array, int arrayIndex)
+        {
+            if (arrayIndex < 0)
+            {
+                return false;
+            }
+            
+                if ((_bits & 1L) != 0)
+                {
+                    if (arrayIndex == array.Length)
+                    {
+                        return false;
+                    }
+                    array[arrayIndex] = new KeyValuePair<string, StringValues>("ETag", _headers._ETag);
+                    ++arrayIndex;
+                }
+                if (_contentLength.HasValue)
+                {
+                    if (arrayIndex == array.Length)
+                    {
+                        return false;
+                    }
+                    array[arrayIndex] = new KeyValuePair<string, StringValues>("Content-Length", HeaderUtilities.FormatNonNegativeInt64(_contentLength.Value));
+                    ++arrayIndex;
+                }
+            ((ICollection<KeyValuePair<string, StringValues>>)MaybeUnknown)?.CopyTo(array, arrayIndex);
+
+            return true;
+        }
+        
+        
+
+        private struct HeaderReferences
+        {
+            public StringValues _ETag;
+            
+        }
+
+        public partial struct Enumerator
+        {
+            public bool MoveNext()
+            {
+                switch (_state)
+                {
+                    
+                    case 0:
+                        goto state0;
+                    
+                    case 1:
+                        goto state1;
+                    default:
+                        goto state_default;
+                }
+                
+                state0:
+                    if ((_bits & 1L) != 0)
+                    {
+                        _current = new KeyValuePair<string, StringValues>("ETag", _collection._headers._ETag);
+                        _state = 1;
+                        return true;
+                    }
+                
+                state1:
+                    if (_collection._contentLength.HasValue)
+                    {
+                        _current = new KeyValuePair<string, StringValues>("Content-Length", HeaderUtilities.FormatNonNegativeInt64(_collection._contentLength.Value));
+                        _state = 2;
+                        return true;
+                    }
+                state_default:
+                    if (!_hasUnknown || !_unknownEnumerator.MoveNext())
+                    {
+                        _current = default(KeyValuePair<string, StringValues>);
+                        return false;
+                    }
+                    _current = _unknownEnumerator.Current;
+                    return true;
+            }
+        }
+    }
 }

src/Kestrel.Core/Internal/Http/HttpHeaders.cs

@@ -97,7 +97,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             throw new ArgumentException(CoreStrings.KeyAlreadyExists);
         }
 
-        int ICollection<KeyValuePair<string, StringValues>>.Count => GetCountFast();
+        public int Count => GetCountFast();
 
         bool ICollection<KeyValuePair<string, StringValues>>.IsReadOnly => _isReadOnly;
 

src/Kestrel.Core/Internal/Http/HttpProtocol.FeatureCollection.cs

@@ -209,9 +209,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             _currentIHttpUpgradeFeature = this;
         }
 
-        protected void ResetIHttp2StreamIdFeature()
+        protected void ResetHttp2Features()
         {
             _currentIHttp2StreamIdFeature = this;
+            _currentIHttpResponseTrailersFeature = this;
         }
 
         void IHttpResponseFeature.OnStarting(Func<object, Task> callback, object state)

src/Kestrel.Core/Internal/Http/HttpProtocol.Generated.cs

@@ -24,6 +24,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         private static readonly Type IFormFeatureType = typeof(IFormFeature);
         private static readonly Type IHttpUpgradeFeatureType = typeof(IHttpUpgradeFeature);
         private static readonly Type IHttp2StreamIdFeatureType = typeof(IHttp2StreamIdFeature);
+        private static readonly Type IHttpResponseTrailersFeatureType = typeof(IHttpResponseTrailersFeature);
         private static readonly Type IResponseCookiesFeatureType = typeof(IResponseCookiesFeature);
         private static readonly Type IItemsFeatureType = typeof(IItemsFeature);
         private static readonly Type ITlsConnectionFeatureType = typeof(ITlsConnectionFeature);
@@ -46,6 +47,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         private object _currentIFormFeature;
         private object _currentIHttpUpgradeFeature;
         private object _currentIHttp2StreamIdFeature;
+        private object _currentIHttpResponseTrailersFeature;
         private object _currentIResponseCookiesFeature;
         private object _currentIItemsFeature;
         private object _currentITlsConnectionFeature;
@@ -79,6 +81,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             _currentIFormFeature = null;
             _currentIHttpUpgradeFeature = null;
             _currentIHttp2StreamIdFeature = null;
+            _currentIHttpResponseTrailersFeature = null;
             _currentIResponseCookiesFeature = null;
             _currentIItemsFeature = null;
             _currentITlsConnectionFeature = null;
@@ -183,6 +186,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 {
                     feature = _currentIHttp2StreamIdFeature;
                 }
+                else if (key == IHttpResponseTrailersFeatureType)
+                {
+                    feature = _currentIHttpResponseTrailersFeature;
+                }
                 else if (key == IResponseCookiesFeatureType)
                 {
                     feature = _currentIResponseCookiesFeature;
@@ -279,6 +286,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 {
                     _currentIHttp2StreamIdFeature = value;
                 }
+                else if (key == IHttpResponseTrailersFeatureType)
+                {
+                    _currentIHttpResponseTrailersFeature = value;
+                }
                 else if (key == IResponseCookiesFeatureType)
                 {
                     _currentIResponseCookiesFeature = value;
@@ -373,6 +384,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 feature = (TFeature)_currentIHttp2StreamIdFeature;
             }
+            else if (typeof(TFeature) == typeof(IHttpResponseTrailersFeature))
+            {
+                feature = (TFeature)_currentIHttpResponseTrailersFeature;
+            }
             else if (typeof(TFeature) == typeof(IResponseCookiesFeature))
             {
                 feature = (TFeature)_currentIResponseCookiesFeature;
@@ -473,6 +488,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 _currentIHttp2StreamIdFeature = feature;
             }
+            else if (typeof(TFeature) == typeof(IHttpResponseTrailersFeature))
+            {
+                _currentIHttpResponseTrailersFeature = feature;
+            }
             else if (typeof(TFeature) == typeof(IResponseCookiesFeature))
             {
                 _currentIResponseCookiesFeature = feature;
@@ -565,6 +584,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 yield return new KeyValuePair<Type, object>(IHttp2StreamIdFeatureType, _currentIHttp2StreamIdFeature);
             }
+            if (_currentIHttpResponseTrailersFeature != null)
+            {
+                yield return new KeyValuePair<Type, object>(IHttpResponseTrailersFeatureType, _currentIHttpResponseTrailersFeature);
+            }
             if (_currentIResponseCookiesFeature != null)
             {
                 yield return new KeyValuePair<Type, object>(IResponseCookiesFeatureType, _currentIResponseCookiesFeature);

src/Kestrel.Core/Internal/Http/HttpResponseTrailers.cs

@@ -0,0 +1,65 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using Microsoft.Extensions.Primitives;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
+{
+    public partial class HttpResponseTrailers : HttpHeaders
+    {
+        public Enumerator GetEnumerator()
+        {
+            return new Enumerator(this);
+        }
+
+        protected override IEnumerator<KeyValuePair<string, StringValues>> GetEnumeratorFast()
+        {
+            return GetEnumerator();
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void SetValueUnknown(string key, in StringValues value)
+        {
+            ValidateHeaderNameCharacters(key);
+            Unknown[key] = value;
+        }
+
+        public partial struct Enumerator : IEnumerator<KeyValuePair<string, StringValues>>
+        {
+            private readonly HttpResponseTrailers _collection;
+            private readonly long _bits;
+            private int _state;
+            private KeyValuePair<string, StringValues> _current;
+            private readonly bool _hasUnknown;
+            private Dictionary<string, StringValues>.Enumerator _unknownEnumerator;
+
+            internal Enumerator(HttpResponseTrailers collection)
+            {
+                _collection = collection;
+                _bits = collection._bits;
+                _state = 0;
+                _current = default;
+                _hasUnknown = collection.MaybeUnknown != null;
+                _unknownEnumerator = _hasUnknown
+                    ? collection.MaybeUnknown.GetEnumerator()
+                    : default;
+            }
+
+            public KeyValuePair<string, StringValues> Current => _current;
+
+            object IEnumerator.Current => _current;
+
+            public void Dispose()
+            {
+            }
+
+            public void Reset()
+            {
+                _state = 0;
+            }
+        }
+    }
+}

src/Kestrel.Core/Internal/Http2/Http2Connection.cs

@@ -90,7 +90,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             var http2Limits = httpLimits.Http2;
 
             _context = context;
-            _frameWriter = new Http2FrameWriter(context.Transport.Output, context.ConnectionContext, _outputFlowControl, context.TimeoutControl, context.ConnectionId, context.ServiceContext.Log);
+            _frameWriter = new Http2FrameWriter(context.Transport.Output, context.ConnectionContext, this, _outputFlowControl, context.TimeoutControl, context.ConnectionId, context.ServiceContext.Log);
             _serverSettings.MaxConcurrentStreams = (uint)http2Limits.MaxStreamsPerConnection;
             _serverSettings.MaxFrameSize = (uint)http2Limits.MaxFrameSize;
             _serverSettings.HeaderTableSize = (uint)http2Limits.HeaderTableSize;

src/Kestrel.Core/Internal/Http2/Http2FrameWriter.cs

@@ -31,6 +31,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
         private readonly PipeWriter _outputWriter;
         private bool _aborted;
         private readonly ConnectionContext _connectionContext;
+        private readonly Http2Connection _http2Connection;
         private readonly OutputFlowControl _connectionOutputFlowControl;
         private readonly string _connectionId;
         private readonly IKestrelTrace _log;
@@ -41,6 +42,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
         public Http2FrameWriter(
             PipeWriter outputPipeWriter,
             ConnectionContext connectionContext,
+            Http2Connection http2Connection,
             OutputFlowControl connectionOutputFlowControl,
             ITimeoutControl timeoutControl,
             string connectionId,
@@ -48,6 +50,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
         {
             _outputWriter = outputPipeWriter;
             _connectionContext = connectionContext;
+            _http2Connection = http2Connection;
             _connectionOutputFlowControl = connectionOutputFlowControl;
             _connectionId = connectionId;
             _log = log;
@@ -157,42 +160,72 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                     _outgoingFrame.PrepareHeaders(Http2HeadersFrameFlags.NONE, streamId);
                     var buffer = _headerEncodingBuffer.AsSpan();
                     var done = _hpackEncoder.BeginEncode(statusCode, EnumerateHeaders(headers), buffer, out var payloadLength);
-
+                    FinishWritingHeaders(streamId, payloadLength, done);
-                    _outgoingFrame.PayloadLength = payloadLength;
-
-                    if (done)
-                    {
-                        _outgoingFrame.HeadersFlags = Http2HeadersFrameFlags.END_HEADERS;
-                    }
-
-                    WriteHeaderUnsynchronized();
-                    _outputWriter.Write(buffer.Slice(0, payloadLength));
-
-                    while (!done)
-                    {
-                        _outgoingFrame.PrepareContinuation(Http2ContinuationFrameFlags.NONE, streamId);
-
-                        done = _hpackEncoder.Encode(buffer, out payloadLength);
-                        _outgoingFrame.PayloadLength = payloadLength;
-
-                        if (done)
-                        {
-                            _outgoingFrame.ContinuationFlags = Http2ContinuationFrameFlags.END_HEADERS;
-                        }
-
-                        WriteHeaderUnsynchronized();
-                        _outputWriter.Write(buffer.Slice(0, payloadLength));
-                    }
                 }
                 catch (HPackEncodingException hex)
                 {
-                    // Header errors are fatal to the connection. We don't have a direct way to signal this to the Http2Connection.
+                    _log.HPackEncodingError(_connectionId, streamId, hex);
-                    _connectionContext.Abort(new ConnectionAbortedException("", hex));
+                    _http2Connection.Abort(new ConnectionAbortedException(hex.Message, hex));
-                    throw new InvalidOperationException("", hex); // Report the error to the user if this was the first write.
+                    throw new InvalidOperationException(hex.Message, hex); // Report the error to the user if this was the first write.
                 }
             }
         }
 
+        public Task WriteResponseTrailers(int streamId, HttpResponseTrailers headers)
+        {
+            lock (_writeLock)
+            {
+                if (_completed)
+                {
+                    return Task.CompletedTask;
+                }
+
+                try
+                {
+                    _outgoingFrame.PrepareHeaders(Http2HeadersFrameFlags.END_STREAM, streamId);
+                    var buffer = _headerEncodingBuffer.AsSpan();
+                    var done = _hpackEncoder.BeginEncode(EnumerateHeaders(headers), buffer, out var payloadLength);
+                    FinishWritingHeaders(streamId, payloadLength, done);
+                }
+                catch (HPackEncodingException hex)
+                {
+                    _log.HPackEncodingError(_connectionId, streamId, hex);
+                    _http2Connection.Abort(new ConnectionAbortedException(hex.Message, hex));
+                }
+
+                return _flusher.FlushAsync();
+            }
+        }
+
+        private void FinishWritingHeaders(int streamId, int payloadLength, bool done)
+        {
+            var buffer = _headerEncodingBuffer.AsSpan();
+            _outgoingFrame.PayloadLength = payloadLength;
+            if (done)
+            {
+                _outgoingFrame.HeadersFlags |= Http2HeadersFrameFlags.END_HEADERS;
+            }
+
+            WriteHeaderUnsynchronized();
+            _outputWriter.Write(buffer.Slice(0, payloadLength));
+
+            while (!done)
+            {
+                _outgoingFrame.PrepareContinuation(Http2ContinuationFrameFlags.NONE, streamId);
+
+                done = _hpackEncoder.Encode(buffer, out payloadLength);
+                _outgoingFrame.PayloadLength = payloadLength;
+
+                if (done)
+                {
+                    _outgoingFrame.ContinuationFlags = Http2ContinuationFrameFlags.END_HEADERS;
+                }
+
+                WriteHeaderUnsynchronized();
+                _outputWriter.Write(buffer.Slice(0, payloadLength));
+            }
+        }
+
         public Task WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence<byte> data, bool endStream)
         {
             // The Length property of a ReadOnlySequence can be expensive, so we cache the value.

src/Kestrel.Core/Internal/Http2/Http2OutputProducer.cs

@@ -24,7 +24,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
         // This should only be accessed via the FrameWriter. The connection-level output flow control is protected by the
         // FrameWriter's connection-level write lock.
         private readonly StreamOutputFlowControl _flowControl;
-
+        private readonly Http2Stream _stream;
         private readonly object _dataWriterLock = new object();
         private readonly Pipe _dataPipe;
         private readonly Task _dataWriteProcessingTask;
@@ -37,11 +37,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
             Http2FrameWriter frameWriter,
             StreamOutputFlowControl flowControl,
             ITimeoutControl timeoutControl,
-            MemoryPool<byte> pool)
+            MemoryPool<byte> pool,
+            Http2Stream stream)
         {
             _streamId = streamId;
             _frameWriter = frameWriter;
             _flowControl = flowControl;
+            _stream = stream;
             _dataPipe = CreateDataPipe(pool);
             _flusher = new TimingPipeFlusher(_dataPipe.Writer, timeoutControl);
             _dataWriteProcessingTask = ProcessDataWrites();
@@ -200,7 +202,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                 {
                     readResult = await _dataPipe.Reader.ReadAsync();
 
-                    await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: readResult.IsCompleted);
+                    if (readResult.IsCompleted && _stream.Trailers?.Count > 0)
+                    {
+                        if (readResult.Buffer.Length > 0)
+                        {
+                            await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: false);
+                        }
+
+                        await _frameWriter.WriteResponseTrailers(_streamId, _stream.Trailers);
+                    }
+                    else
+                    {
+                        await _frameWriter.WriteDataAsync(_streamId, _flowControl, readResult.Buffer, endStream: readResult.IsCompleted);
+                    }
 
                     _dataPipe.Reader.AdvanceTo(readResult.Buffer.End);
                 } while (!readResult.IsCompleted);

src/Kestrel.Core/Internal/Http2/Http2Stream.FeatureCollection.cs

@@ -1,12 +1,34 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
 {
-    public partial class Http2Stream : IHttp2StreamIdFeature
+    public partial class Http2Stream : IHttp2StreamIdFeature, IHttpResponseTrailersFeature
     {
+        internal HttpResponseTrailers Trailers { get; set; }
+        private IHeaderDictionary _userTrailers;
+
+        IHeaderDictionary IHttpResponseTrailersFeature.Trailers
+        {
+            get
+            {
+                if (Trailers == null)
+                {
+                    Trailers = new HttpResponseTrailers();
+                }
+                return _userTrailers ?? Trailers;
+            }
+            set
+            {
+                _userTrailers = value;
+            }
+        }
+
         int IHttp2StreamIdFeature.StreamId => _context.StreamId;
     }
 }

src/Kestrel.Core/Internal/Http2/Http2Stream.cs

@@ -40,7 +40,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
                 _context.ServerPeerSettings.InitialWindowSize / 2);
 
             _outputFlowControl = new StreamOutputFlowControl(context.ConnectionOutputFlowControl, context.ClientPeerSettings.InitialWindowSize);
-            _http2Output = new Http2OutputProducer(context.StreamId, context.FrameWriter, _outputFlowControl, context.TimeoutControl, context.MemoryPool);
+            _http2Output = new Http2OutputProducer(context.StreamId, context.FrameWriter, _outputFlowControl, context.TimeoutControl, context.MemoryPool, this);
 
             RequestBodyPipe = CreateRequestBodyPipe(_context.ServerPeerSettings.InitialWindowSize);
             Output = _http2Output;
@@ -59,7 +59,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
 
         protected override void OnReset()
         {
-            ResetIHttp2StreamIdFeature();
+            ResetHttp2Features();
         }
 
         protected override void OnRequestProcessingEnded()

src/Kestrel.Core/Internal/Infrastructure/IKestrelTrace.cs

@@ -67,6 +67,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
 
         void HPackDecodingError(string connectionId, int streamId, HPackDecodingException ex);
 
+        void HPackEncodingError(string connectionId, int streamId, HPackEncodingException ex);
+
         void Http2FrameReceived(string connectionId, Http2Frame frame);
 
         void Http2FrameSending(string connectionId, Http2Frame frame);

src/Kestrel.Core/Internal/Infrastructure/KestrelTrace.cs

@@ -107,6 +107,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
             LoggerMessage.Define<string, Http2FrameType, int, int, object>(LogLevel.Trace, new EventId(37, nameof(Http2FrameReceived)),
                 @"Connection id ""{ConnectionId}"" sending {type} frame for stream ID {id} with length {length} and flags {flags}");
 
+        private static readonly Action<ILogger, string, int, Exception> _hpackEncodingError =
+            LoggerMessage.Define<string, int>(LogLevel.Information, new EventId(38, nameof(HPackEncodingError)),
+                @"Connection id ""{ConnectionId}"": HPACK encoding error while encoding headers for stream ID {StreamId}.");
+
         protected readonly ILogger _logger;
 
         public KestrelTrace(ILogger logger)
@@ -254,6 +258,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
             _hpackDecodingError(_logger, connectionId, streamId, ex);
         }
 
+        public virtual void HPackEncodingError(string connectionId, int streamId, HPackEncodingException ex)
+        {
+            _hpackEncodingError(_logger, connectionId, streamId, ex);
+        }
+
         public void Http2FrameReceived(string connectionId, Http2Frame frame)
         {
             _http2FrameReceived(_logger, connectionId, frame.Type, frame.StreamId, frame.PayloadLength, frame.ShowFlags(), null);

test/Kestrel.InMemory.FunctionalTests/Http2/Http2StreamTests.cs

@@ -13,6 +13,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.HPack;
 using Microsoft.Extensions.Logging;
 using Microsoft.Net.Http.Headers;
 using Xunit;
@@ -1340,6 +1341,223 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]);
         }
 
+        [Fact]
+        public async Task ResponseTrailers_WithoutData_Sent()
+        {
+            await InitializeConnectionAsync(context =>
+            {
+                context.Response.AppendTrailer("CustomName", "Custom Value");
+                return Task.CompletedTask;
+            });
+
+            await StartStreamAsync(1, _browserRequestHeaders, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 55,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            var trailersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 25,
+                withFlags: (byte)(Http2HeadersFrameFlags.END_HEADERS | Http2HeadersFrameFlags.END_STREAM),
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]);
+
+            _decodedHeaders.Clear();
+
+            _hpackDecoder.Decode(trailersFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Single(_decodedHeaders);
+            Assert.Equal("Custom Value", _decodedHeaders["CustomName"]);
+        }
+
+        [Fact]
+        public async Task ResponseTrailers_WithData_Sent()
+        {
+            await InitializeConnectionAsync(async context =>
+            {
+                await context.Response.WriteAsync("Hello World");
+                context.Response.AppendTrailer("CustomName", "Custom Value");
+            });
+
+            await StartStreamAsync(1, _browserRequestHeaders, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 11,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+
+            var trailersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 25,
+                withFlags: (byte)(Http2HeadersFrameFlags.END_HEADERS | Http2HeadersFrameFlags.END_STREAM),
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+
+            _decodedHeaders.Clear();
+
+            _hpackDecoder.Decode(trailersFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Single(_decodedHeaders);
+            Assert.Equal("Custom Value", _decodedHeaders["CustomName"]);
+        }
+
+        [Fact]
+        public async Task ResponseTrailers_WithContinuation_Sent()
+        {
+            var largeHeader = new string('a', 1024 * 3);
+            await InitializeConnectionAsync(async context =>
+            {
+                await context.Response.WriteAsync("Hello World");
+                // The first five fill the first frame
+                context.Response.AppendTrailer("CustomName0", largeHeader);
+                context.Response.AppendTrailer("CustomName1", largeHeader);
+                context.Response.AppendTrailer("CustomName2", largeHeader);
+                context.Response.AppendTrailer("CustomName3", largeHeader);
+                context.Response.AppendTrailer("CustomName4", largeHeader);
+                // This one spills over to the next frame
+                context.Response.AppendTrailer("CustomName5", largeHeader);
+            });
+
+            await StartStreamAsync(1, _browserRequestHeaders, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 11,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+
+            var trailersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 15440,
+                withFlags: (byte)Http2HeadersFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            var trailersContinuationFrame = await ExpectAsync(Http2FrameType.CONTINUATION,
+                withLength: 3088,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+
+            _decodedHeaders.Clear();
+
+            _hpackDecoder.Decode(trailersFrame.PayloadSequence, endHeaders: false, handler: this);
+
+            Assert.Equal(5, _decodedHeaders.Count);
+            Assert.Equal(largeHeader, _decodedHeaders["CustomName0"]);
+            Assert.Equal(largeHeader, _decodedHeaders["CustomName1"]);
+            Assert.Equal(largeHeader, _decodedHeaders["CustomName2"]);
+            Assert.Equal(largeHeader, _decodedHeaders["CustomName3"]);
+            Assert.Equal(largeHeader, _decodedHeaders["CustomName4"]);
+
+            _decodedHeaders.Clear();
+
+            _hpackDecoder.Decode(trailersContinuationFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Single(_decodedHeaders);
+            Assert.Equal(largeHeader, _decodedHeaders["CustomName5"]);
+        }
+
+        [Fact]
+        public async Task ResponseTrailers_WithNonAscii_Throws()
+        {
+            await InitializeConnectionAsync(async context =>
+            {
+                await context.Response.WriteAsync("Hello World");
+                Assert.Throws<InvalidOperationException>(() => context.Response.AppendTrailer("Custom你好Name", "Custom Value"));
+                Assert.Throws<InvalidOperationException>(() => context.Response.AppendTrailer("CustomName", "Custom 你好 Value"));
+            });
+
+            await StartStreamAsync(1, _browserRequestHeaders, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 11,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 0,
+                withFlags: (byte)Http2DataFrameFlags.END_STREAM,
+                withStreamId: 1);
+
+            await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false);
+
+            _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: true, handler: this);
+
+            Assert.Equal(2, _decodedHeaders.Count);
+            Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
+            Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+        }
+
+        [Fact]
+        public async Task ResponseTrailers_TooLong_Throws()
+        {
+            await InitializeConnectionAsync(async context =>
+            {
+                await context.Response.WriteAsync("Hello World");
+                context.Response.AppendTrailer("too_long", new string('a', (int)Http2PeerSettings.DefaultMaxFrameSize));
+            });
+
+            await StartStreamAsync(1, _browserRequestHeaders, endStream: true);
+
+            var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
+                withStreamId: 1);
+
+            await ExpectAsync(Http2FrameType.DATA,
+                withLength: 11,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 1);
+
+            var goAway = await ExpectAsync(Http2FrameType.GOAWAY,
+                withLength: 8,
+                withFlags: (byte)Http2DataFrameFlags.NONE,
+                withStreamId: 0);
+
+            VerifyGoAway(goAway, 1, Http2ErrorCode.INTERNAL_ERROR);
+
+            _pair.Application.Output.Complete();
+            await _connectionTask;
+
+            var message = Assert.Single(TestApplicationErrorLogger.Messages, m => m.Exception is HPackEncodingException);
+            Assert.Contains(CoreStrings.HPackErrorNotEnoughBuffer, message.Exception.Message);
+        }
+
         [Fact]
         public async Task ApplicationException_BeforeFirstWrite_Sends500()
         {
@@ -1750,6 +1968,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
 
             var message = await appFinished.Task.DefaultTimeout();
             Assert.Equal(CoreStrings.HPackErrorNotEnoughBuffer, message);
+
+            // Just the StatusCode gets written before aborting in the continuation frame
+            await ExpectAsync(Http2FrameType.HEADERS,
+                withLength: 37,
+                withFlags: (byte)Http2HeadersFrameFlags.NONE,
+                withStreamId: 1);
+
+            _pair.Application.Output.Complete();
+
+            await WaitForConnectionErrorAsync<HPackEncodingException>(ignoreNonGoAwayFrames: false, expectedLastStreamId: 1, Http2ErrorCode.INTERNAL_ERROR,
+                CoreStrings.HPackErrorNotEnoughBuffer);
         }
     }
 }

test/shared/CompositeKestrelTrace.cs

@@ -189,6 +189,12 @@ namespace Microsoft.AspNetCore.Testing
             _trace2.HPackDecodingError(connectionId, streamId, ex);
         }
 
+        public void HPackEncodingError(string connectionId, int streamId, HPackEncodingException ex)
+        {
+            _trace1.HPackEncodingError(connectionId, streamId, ex);
+            _trace2.HPackEncodingError(connectionId, streamId, ex);
+        }
+
         public void Http2StreamResetAbort(string traceIdentifier, Http2ErrorCode error, ConnectionAbortedException abortReason)
         {
             _trace1.Http2StreamResetAbort(traceIdentifier, error, abortReason);

tools/CodeGenerator/HttpProtocolFeatureCollection.cs

@@ -30,6 +30,7 @@ namespace CodeGenerator
             {
                 "IHttpUpgradeFeature",
                 "IHttp2StreamIdFeature",
+                "IHttpResponseTrailersFeature",
                 "IResponseCookiesFeature",
                 "IItemsFeature",
                 "ITlsConnectionFeature",

tools/CodeGenerator/KnownHeaders.cs

@@ -269,6 +269,21 @@ namespace CodeGenerator
                 PrimaryHeader = responsePrimaryHeaders.Contains("Content-Length")
             }})
             .ToArray();
+
+            var responseTrailers = new[]
+            {
+                "ETag",
+            }
+            .Select((header, index) => new KnownHeader
+            {
+                Name = header,
+                Index = index,
+                EnhancedSetter = enhancedHeaders.Contains(header),
+                ExistenceCheck = responseHeadersExistence.Contains(header),
+                PrimaryHeader = responsePrimaryHeaders.Contains(header)
+            })
+            .ToArray();
+
             // 63 for responseHeaders as it steals one bit for Content-Length in CopyTo(ref MemoryPoolIterator output)
             Debug.Assert(responseHeaders.Length <= 63);
             Debug.Assert(responseHeaders.Max(x => x.Index) <= 62);
@@ -288,6 +303,13 @@ namespace CodeGenerator
                     HeadersByLength = responseHeaders.GroupBy(x => x.Name.Length),
                     ClassName = "HttpResponseHeaders",
                     Bytes = responseHeaders.SelectMany(header => header.Bytes).ToArray()
+                },
+                new
+                {
+                    Headers = responseTrailers,
+                    HeadersByLength = responseTrailers.GroupBy(x => x.Name.Length),
+                    ClassName = "HttpResponseTrailers",
+                    Bytes = responseTrailers.SelectMany(header => header.Bytes).ToArray()
                 }
             };
             foreach (var loop in loops.Where(l => l.Bytes != null))
@@ -402,7 +424,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         }}
 
         protected override void SetValueFast(string key, in StringValues value)
-        {{{(loop.ClassName == "HttpResponseHeaders" ? @"
+        {{{(loop.ClassName != "HttpRequestHeaders" ? @"
             ValidateHeaderValueCharacters(value);" : "")}
             switch (key.Length)
             {{{Each(loop.HeadersByLength, byLength => $@"
@@ -424,7 +446,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         }}
 
         protected override bool AddValueFast(string key, in StringValues value)
-        {{{(loop.ClassName == "HttpResponseHeaders" ? @"
+        {{{(loop.ClassName != "HttpRequestHeaders" ? @"
             ValidateHeaderValueCharacters(value);" : "")}
             switch (key.Length)
             {{{Each(loop.HeadersByLength, byLength => $@"