Use less stack for HttpResponseHeaders.CopyToFast (#7724)

src/Servers/Kestrel/Core/src/Internal/Http/HttpHeaders.cs

@@ -10,6 +10,7 @@ using System.Runtime.Intrinsics.X86;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.Extensions.Primitives;
+using Microsoft.Net.Http.Headers;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 {

src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestHeaders.cs

@@ -67,7 +67,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         {
             private readonly HttpRequestHeaders _collection;
             private readonly long _bits;
-            private int _state;
+            private int _next;
             private KeyValuePair<string, StringValues> _current;
             private readonly bool _hasUnknown;
             private Dictionary<string, StringValues>.Enumerator _unknownEnumerator;
@@ -76,7 +76,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 _collection = collection;
                 _bits = collection._bits;
-                _state = 0;
+                _next = 0;
                 _current = default(KeyValuePair<string, StringValues>);
                 _hasUnknown = collection.MaybeUnknown != null;
                 _unknownEnumerator = _hasUnknown
@@ -94,7 +94,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
             public void Reset()
             {
-                _state = 0;
+                _next = 0;
             }
         }
     }

src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseHeaders.cs

@@ -58,6 +58,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             return parsed;
         }
 
+        private static void ThrowInvalidContentLengthException(string value)
+        {
+            throw new InvalidOperationException(CoreStrings.FormatInvalidContentLength_InvalidNumber(value));
+        }
+
         [MethodImpl(MethodImplOptions.NoInlining)]
         private void SetValueUnknown(string key, in StringValues value)
         {
@@ -65,16 +70,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             Unknown[key] = value;
         }
 
-        private static void ThrowInvalidContentLengthException(string value)
-        {
-            throw new InvalidOperationException(CoreStrings.FormatInvalidContentLength_InvalidNumber(value));
-        }
-
         public partial struct Enumerator : IEnumerator<KeyValuePair<string, StringValues>>
         {
             private readonly HttpResponseHeaders _collection;
             private readonly long _bits;
-            private int _state;
+            private int _next;
             private KeyValuePair<string, StringValues> _current;
             private readonly bool _hasUnknown;
             private Dictionary<string, StringValues>.Enumerator _unknownEnumerator;
@@ -83,7 +83,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 _collection = collection;
                 _bits = collection._bits;
-                _state = 0;
+                _next = 0;
                 _current = default;
                 _hasUnknown = collection.MaybeUnknown != null;
                 _unknownEnumerator = _hasUnknown
@@ -101,7 +101,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
             public void Reset()
             {
-                _state = 0;
+                _next = 0;
             }
         }
 

src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseTrailers.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
@@ -31,7 +32,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
         {
             private readonly HttpResponseTrailers _collection;
             private readonly long _bits;
-            private int _state;
+            private int _next;
             private KeyValuePair<string, StringValues> _current;
             private readonly bool _hasUnknown;
             private Dictionary<string, StringValues>.Enumerator _unknownEnumerator;
@@ -40,7 +41,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             {
                 _collection = collection;
                 _bits = collection._bits;
-                _state = 0;
+                _next = 0;
                 _current = default;
                 _hasUnknown = collection.MaybeUnknown != null;
                 _unknownEnumerator = _hasUnknown
@@ -58,7 +59,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
             public void Reset()
             {
-                _state = 0;
+                _next = 0;
             }
         }
     }

src/Servers/Kestrel/perf/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs

@@ -8,25 +8,23 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
-using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.Server.Kestrel.Core;
-using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
-using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal;
-using Microsoft.AspNetCore.Testing;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
     public class ResponseHeadersWritingBenchmark
     {
+        private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: " + Constants.ServerName);
         private static readonly byte[] _helloWorldPayload = Encoding.ASCII.GetBytes("Hello, World!");
 
-        private TestHttp1Connection _http1Connection;
+        private HttpResponseHeaders _responseHeaders;
+        private IHeaderDictionary _responseHeadersDict;
+        private DateHeaderValueManager _dateHeaderValueManager;
+        private Writer _writer;
 
-        private MemoryPool<byte> _memoryPool;
+        private DateHeaderValueManager.DateHeaderValues DateHeaderValues => _dateHeaderValueManager.GetDateHeaderValues();
-
-        private DuplexPipe.DuplexPipePair _pair;
 
         [Params(
             BenchmarkTypes.TechEmpowerPlaintext,
@@ -38,118 +36,113 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
         public BenchmarkTypes Type { get; set; }
 
         [Benchmark]
-        public async Task Output()
+        public void Output()
         {
-            _http1Connection.Reset();
-            _http1Connection.StatusCode = 200;
-            _http1Connection.HttpVersionEnum = HttpVersion.Http11;
-            _http1Connection.KeepAlive = true;
-
-            Task writeTask = Task.CompletedTask;
             switch (Type)
             {
                 case BenchmarkTypes.TechEmpowerPlaintext:
-                    writeTask = TechEmpowerPlaintext();
+                    TechEmpowerPlaintext();
                     break;
                 case BenchmarkTypes.PlaintextChunked:
-                    writeTask = PlaintextChunked();
+                    PlaintextChunked();
                     break;
                 case BenchmarkTypes.PlaintextWithCookie:
-                    writeTask = PlaintextWithCookie();
+                    PlaintextWithCookie();
                     break;
                 case BenchmarkTypes.PlaintextChunkedWithCookie:
-                    writeTask = PlaintextChunkedWithCookie();
+                    PlaintextChunkedWithCookie();
                     break;
                 case BenchmarkTypes.LiveAspNet:
-                    writeTask = LiveAspNet();
+                    LiveAspNet();
                     break;
             }
-
-            await writeTask;
-            await _http1Connection.ProduceEndAsync();
         }
 
-        private Task TechEmpowerPlaintext()
+        private void TechEmpowerPlaintext()
         {
-            var responseHeaders = _http1Connection.ResponseHeaders;
+            var responseHeaders = _responseHeaders;
-            responseHeaders["Content-Type"] = "text/plain";
+            responseHeaders.HeaderContentType = "text/plain";
             responseHeaders.ContentLength = _helloWorldPayload.Length;
-            return _http1Connection.WriteAsync(new ArraySegment<byte>(_helloWorldPayload), default(CancellationToken));
+
+            var writer = new BufferWriter<PipeWriter>(_writer);
+            _responseHeaders.CopyTo(ref writer);
         }
 
-        private Task PlaintextChunked()
+        private void PlaintextChunked()
         {
-            var responseHeaders = _http1Connection.ResponseHeaders;
+            var responseHeaders = _responseHeaders;
-            responseHeaders["Content-Type"] = "text/plain";
+            responseHeaders.HeaderContentType = "text/plain";
-            return _http1Connection.WriteAsync(new ArraySegment<byte>(_helloWorldPayload), default(CancellationToken));
+
+            var writer = new BufferWriter<PipeWriter>(_writer);
+            _responseHeaders.CopyTo(ref writer);
         }
 
-        private Task LiveAspNet()
+        private void LiveAspNet()
         {
-            var responseHeaders = _http1Connection.ResponseHeaders;
+            var responseHeaders = _responseHeaders;
-            responseHeaders["Content-Encoding"] = "gzip";
+            responseHeaders.HeaderContentEncoding = "gzip";
-            responseHeaders["Content-Type"] = "text/html; charset=utf-8";
+            responseHeaders.HeaderContentType = "text/html; charset=utf-8";
-            responseHeaders["Strict-Transport-Security"] = "max-age=31536000; includeSubdomains";
+            _responseHeadersDict["Strict-Transport-Security"] = "max-age=31536000; includeSubdomains";
-            responseHeaders["Vary"] = "Accept-Encoding";
+            responseHeaders.HeaderVary = "Accept-Encoding";
-            responseHeaders["X-Powered-By"] = "ASP.NET";
+            _responseHeadersDict["X-Powered-By"] = "ASP.NET";
-            return _http1Connection.WriteAsync(new ArraySegment<byte>(_helloWorldPayload), default(CancellationToken));
+
+            var writer = new BufferWriter<PipeWriter>(_writer);
+            _responseHeaders.CopyTo(ref writer);
         }
 
-        private Task PlaintextWithCookie()
+        private void PlaintextWithCookie()
         {
-            var responseHeaders = _http1Connection.ResponseHeaders;
+            var responseHeaders = _responseHeaders;
-            responseHeaders["Content-Type"] = "text/plain";
+            responseHeaders.HeaderContentType = "text/plain";
-            responseHeaders["Set-Cookie"] = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric";
+            responseHeaders.HeaderSetCookie = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric";
             responseHeaders.ContentLength = _helloWorldPayload.Length;
-            return _http1Connection.WriteAsync(new ArraySegment<byte>(_helloWorldPayload), default(CancellationToken));
+
+            var writer = new BufferWriter<PipeWriter>(_writer);
+            _responseHeaders.CopyTo(ref writer);
         }
 
-        private Task PlaintextChunkedWithCookie()
+        private void PlaintextChunkedWithCookie()
         {
-            var responseHeaders = _http1Connection.ResponseHeaders;
+            var responseHeaders = _responseHeaders;
-            responseHeaders["Content-Type"] = "text/plain";
+            responseHeaders.HeaderContentType = "text/plain";
-            responseHeaders["Set-Cookie"] = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric";
+            responseHeaders.HeaderSetCookie = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric";
-            return _http1Connection.WriteAsync(new ArraySegment<byte>(_helloWorldPayload), default(CancellationToken));
+            responseHeaders.HeaderTransferEncoding = "chunked";
+
+            var writer = new BufferWriter<PipeWriter>(_writer);
+            _responseHeaders.CopyTo(ref writer);
+        }
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            _responseHeaders = new HttpResponseHeaders();
+            _responseHeadersDict = _responseHeaders;
+            _dateHeaderValueManager = new DateHeaderValueManager();
+            _dateHeaderValueManager.OnHeartbeat(DateTimeOffset.Now);
+            _writer = new Writer();
         }
 
         [IterationSetup]
-        public void Setup()
+        public void IterationSetup()
         {
-            _memoryPool = KestrelMemoryPool.Create();
+            _responseHeaders.Reset();
-            var options = new PipeOptions(_memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
+            _responseHeaders.SetRawServer(Constants.ServerName, _bytesServer);
-            _pair = DuplexPipe.CreateConnectionPair(options, options);
+            _responseHeaders.SetRawDate(DateHeaderValues.String, DateHeaderValues.Bytes);
-
-            var serviceContext = new ServiceContext
-            {
-                DateHeaderValueManager = new DateHeaderValueManager(),
-                ServerOptions = new KestrelServerOptions(),
-                Log = new MockTrace(),
-                HttpParser = new HttpParser<Http1ParsingHandler>()
-            };
-
-            var http1Connection = new TestHttp1Connection(new HttpConnectionContext
-            {
-                ServiceContext = serviceContext,
-                ConnectionFeatures = new FeatureCollection(),
-                MemoryPool = _memoryPool,
-                TimeoutControl = new TimeoutControl(timeoutHandler: null),
-                Transport = _pair.Transport
-            });
-
-            http1Connection.Reset();
-            serviceContext.DateHeaderValueManager.OnHeartbeat(DateTimeOffset.UtcNow);
-
-            _http1Connection = http1Connection;
         }
 
-        [IterationCleanup]
+        public class Writer : PipeWriter
-        public void Cleanup()
         {
-            _pair.Application.Input.Complete();
+            private Memory<byte> _memory = new byte[4096 * 4];
-            _pair.Application.Output.Complete();
+
-            _pair.Transport.Input.Complete();
+            public override Memory<byte> GetMemory(int sizeHint = 0) => _memory;
-            _pair.Transport.Output.Complete();
+
-            _memoryPool.Dispose();
+            public override Span<byte> GetSpan(int sizeHint = 0) => _memory.Span;
+
+            public override void Advance(int bytes) { }
+            public override void CancelPendingFlush() { }
+            public override void Complete(Exception exception = null)  { }
+            public override ValueTask<FlushResult> FlushAsync(CancellationToken cancellationToken = default) => default;
+            public override void OnReaderCompleted(Action<Exception, object> callback, object state) { }
         }
 
         public enum BenchmarkTypes

src/Servers/Kestrel/tools/CodeGenerator/KnownHeaders.cs

@@ -65,12 +65,12 @@ namespace CodeGenerator
             public bool FastCount { get; set; }
             public bool EnhancedSetter { get; set; }
             public bool PrimaryHeader { get; set; }
-            public string TestBit() => $"(_bits & {1L << Index}L) != 0";
+            public string TestBit() => $"(_bits & {"0x" + (1L << Index).ToString("x")}L) != 0";
-            public string TestTempBit() => $"(tempBits & {1L << Index}L) != 0";
+            public string TestTempBit() => $"(tempBits & {"0x" + (1L << Index).ToString("x")}L) != 0";
-            public string TestNotTempBit() => $"(tempBits & ~{1L << Index}L) == 0";
+            public string TestNotTempBit() => $"(tempBits & ~{"0x" + (1L << Index).ToString("x")}L) == 0";
-            public string TestNotBit() => $"(_bits & {1L << Index}L) == 0";
+            public string TestNotBit() => $"(_bits & {"0x" + (1L << Index).ToString("x")}L) == 0";
-            public string SetBit() => $"_bits |= {1L << Index}L";
+            public string SetBit() => $"_bits |= {"0x" + (1L << Index).ToString("x")}L";
-            public string ClearBit() => $"_bits &= ~{1L << Index}L";
+            public string ClearBit() => $"_bits &= ~{"0x" + (1L << Index).ToString("x")}L";
 
             public string EqualIgnoreCaseBytes()
             {
@@ -133,6 +133,7 @@ namespace CodeGenerator
                 "Date",
                 "Content-Type",
                 "Server",
+                "Content-Length",
             };
             var commonHeaders = new[]
             {
@@ -264,7 +265,7 @@ namespace CodeGenerator
             .Concat(new[] { new KnownHeader
             {
                 Name = "Content-Length",
-                Index = -1,
+                Index = 63,
                 EnhancedSetter = enhancedHeaders.Contains("Content-Length"),
                 PrimaryHeader = responsePrimaryHeaders.Contains("Content-Length")
             }})
@@ -286,7 +287,7 @@ namespace CodeGenerator
 
             // 63 for responseHeaders as it steals one bit for Content-Length in CopyTo(ref MemoryPoolIterator output)
             Debug.Assert(responseHeaders.Length <= 63);
-            Debug.Assert(responseHeaders.Max(x => x.Index) <= 62);
+            Debug.Assert(responseHeaders.Count(x => x.Index == 63) == 1);
 
             var loops = new[]
             {
@@ -327,9 +328,9 @@ namespace CodeGenerator
 
 using System;
 using System.Collections.Generic;
-using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using System.Buffers;
 using System.IO.Pipelines;
+using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
 
@@ -339,7 +340,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
     public partial class {loop.ClassName}
     {{{(loop.Bytes != null ?
         $@"
-        private static byte[] _headerBytes = new byte[]
+        private static ReadOnlySpan<byte> HeaderBytes => new byte[]
         {{
             {Each(loop.Bytes, b => $"{b},")}
         }};"
@@ -529,7 +530,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
                 {{
                     return;
                 }}
-                tempBits &= ~{1L << header.Index}L;
+                tempBits &= ~{"0x" + (1L << header.Index).ToString("x")}L;
             }}
             ")}
         }}
@@ -564,47 +565,66 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
             return true;
         }}
         {(loop.ClassName == "HttpResponseHeaders" ? $@"
-        internal void CopyToFast(ref BufferWriter<PipeWriter> output)
+        internal unsafe void CopyToFast(ref BufferWriter<PipeWriter> output)
         {{
-            var tempBits = _bits | (_contentLength.HasValue ? {1L << 63}L : 0);
+            var tempBits = (ulong)_bits | (_contentLength.HasValue ? {"0x" + (1L << 63).ToString("x")}L : 0);
-            {Each(loop.Headers.Where(header => header.Identifier != "ContentLength").OrderBy(h => !h.PrimaryHeader), header => $@"
+            var next = 0;
-                if ({header.TestTempBit()})
+            var keyStart = 0;
-                {{ {(header.EnhancedSetter == false ? "" : $@"
+            var keyLength = 0;
-                    if (_headers._raw{header.Identifier} != null)
+            ref readonly StringValues values = ref Unsafe.AsRef<StringValues>(null);
-                    {{
+
-                        output.Write(_headers._raw{header.Identifier});
+            do
-                    }}
+            {{
-                    else ")}
+                switch (next)
-                    {{
+                {{{Each(loop.Headers.OrderBy(h => !h.PrimaryHeader).Select((h, i) => (Header: h, Index: i)), hi => $@"
-                        var valueCount = _headers._{header.Identifier}.Count;
+                    case {hi.Index}: // Header: ""{hi.Header.Name}""
-                        for (var i = 0; i < valueCount; i++)
+                        if ({hi.Header.TestTempBit()})
                         {{
-                            var value = _headers._{header.Identifier}[i];
+                            tempBits ^= {"0x" + (1L << hi.Header.Index).ToString("x")}L;{(hi.Header.Identifier != "ContentLength" ? $@"{(hi.Header.EnhancedSetter == false ? $@"
-                            if (value != null)
+                            values = ref _headers._{hi.Header.Identifier};
+                            keyStart = {hi.Header.BytesOffset};
+                            keyLength = {hi.Header.BytesCount};
+                            next = {hi.Index + 1};
+                            break; // OutputHeader" : $@"
+                            if (_headers._raw{hi.Header.Identifier} != null)
                             {{
-                                output.Write(new ReadOnlySpan<byte>(_headerBytes, {header.BytesOffset}, {header.BytesCount}));
+                                output.Write(_headers._raw{hi.Header.Identifier});
-                                output.WriteAsciiNoValidation(value);
                             }}
+                            else
+                            {{
+                                values = ref _headers._{hi.Header.Identifier};
+                                keyStart = {hi.Header.BytesOffset};
+                                keyLength = {hi.Header.BytesCount};
+                                next = {hi.Index + 1};
+                                break; // OutputHeader
+                            }}")}" : $@"
+                            output.Write(HeaderBytes.Slice({hi.Header.BytesOffset}, {hi.Header.BytesCount}));
+                            output.WriteNumeric((ulong)ContentLength.Value);
+                            if (tempBits == 0)
+                            {{
+                                return;
+                            }}")}
+                        }}
+                        {(hi.Index + 1 < loop.Headers.Count() ? $"goto case {hi.Index + 1};" : "return;")}")}
+                    default:
+                        return;
+                }}
+
+                // OutputHeader
+                {{
+                    var valueCount = values.Count;
+                    var headerKey = HeaderBytes.Slice(keyStart, keyLength);
+                    for (var i = 0; i < valueCount; i++)
+                    {{
+                        var value = values[i];
+                        if (value != null)
+                        {{
+                            output.Write(headerKey);
+                            output.WriteAsciiNoValidation(value);
                         }}
                     }}
-
+                }}
-                    if({header.TestNotTempBit()})
+            }} while (tempBits != 0);
-                    {{
-                        return;
-                    }}
-                    tempBits &= ~{1L << header.Index}L;
-                }}{(header.Identifier == "Server" ? $@"
-                if ((tempBits & {1L << 63}L) != 0)
-                {{
-                    output.Write(new ReadOnlySpan<byte>(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount}));
-                    output.WriteNumeric((ulong)ContentLength.Value);
-
-                    if((tempBits & ~{1L << 63}L) == 0)
-                    {{
-                        return;
-                    }}
-                    tempBits &= ~{1L << 63}L;
-                }}" : "")}")}
         }}" : "")}
         {(loop.ClassName == "HttpRequestHeaders" ? $@"
         public unsafe void Append(byte* pKeyBytes, int keyLength, string value)
@@ -632,36 +652,34 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http
 
         public partial struct Enumerator
         {{
+            // Compiled to Jump table
             public bool MoveNext()
             {{
-                switch (_state)
+                switch (_next)
-                {{
+                {{{Each(loop.Headers.Where(header => header.Identifier != "ContentLength"), header => $@"
-                    {Each(loop.Headers.Where(header => header.Identifier != "ContentLength"), header => $@"
                     case {header.Index}:
-                        goto state{header.Index};
+                        goto Header{header.Identifier};")}
-                    ")}
+                    {(!loop.ClassName.Contains("Trailers") ? $@"case {loop.Headers.Count() - 1}:
-                    case {loop.Headers.Count()}:
+                        goto HeaderContentLength;" : "")}
-                        goto state{loop.Headers.Count()};
                     default:
-                        goto state_default;
+                        goto ExtraHeaders;
                 }}
                 {Each(loop.Headers.Where(header => header.Identifier != "ContentLength"), header => $@"
-                state{header.Index}:
+                Header{header.Identifier}: // case {header.Index}
                     if ({header.TestBit()})
                     {{
                         _current = new KeyValuePair<string, StringValues>(""{header.Name}"", _collection._headers._{header.Identifier});
-                        _state = {header.Index + 1};
+                        _next = {header.Index + 1};
                         return true;
-                    }}
+                    }}")}
-                ")}
+                {(!loop.ClassName.Contains("Trailers") ? $@"HeaderContentLength: // case {loop.Headers.Count() - 1}
-                state{loop.Headers.Count()}:
                     if (_collection._contentLength.HasValue)
                     {{
                         _current = new KeyValuePair<string, StringValues>(""Content-Length"", HeaderUtilities.FormatNonNegativeInt64(_collection._contentLength.Value));
-                        _state = {loop.Headers.Count() + 1};
+                        _next = {loop.Headers.Count()};
                         return true;
-                    }}
+                    }}" : "")}
-                state_default:
+                ExtraHeaders:
                     if (!_hasUnknown || !_unknownEnumerator.MoveNext())
                     {{
                         _current = default(KeyValuePair<string, StringValues>);