Single span optimizations (#1421)

- Added a fast path for single span in the start line parsing
- Added a fast path for single span header parsing
- Changed the out header loop to be pointer based (instead of slicing)

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/KestrelHttpParser.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Buffers;
 using System.IO.Pipelines;
 using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure;
 using Microsoft.Extensions.Logging;
@@ -31,8 +32,27 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             consumed = buffer.Start;
             examined = buffer.End;
 
+            ReadCursor end;
+            Span<byte> span;
+
+            // If the buffer is a single span then use it to find the LF
+            if (buffer.IsSingleSpan)
+            {
+                var startLineSpan = buffer.First.Span;
+                var lineIndex = startLineSpan.IndexOfVectorized(ByteLF);
+
+                if (lineIndex == -1)
+                {
+                    return false;
+                }
+
+                end = buffer.Move(consumed, lineIndex + 1);
+                span = startLineSpan.Slice(0, lineIndex + 1);
+            }
+            else
+            {
                 var start = buffer.Start;
-            if (ReadCursorOperations.Seek(start, buffer.End, out var end, ByteLF) == -1)
+                if (ReadCursorOperations.Seek(start, buffer.End, out end, ByteLF) == -1)
                 {
                     return false;
                 }
@@ -41,7 +61,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
                 end = buffer.Move(end, 1);
                 var startLineBuffer = buffer.Slice(start, end);
 
-            Span<byte> span;
                 if (startLineBuffer.IsSingleSpan)
                 {
                     // No copies, directly use the one and only span
@@ -53,6 +72,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
                     span = new Span<byte>(new byte[startLineBuffer.Length]);
                     startLineBuffer.CopyTo(span);
                 }
+            }
 
             var pathStart = -1;
             var queryStart = -1;
@@ -243,6 +263,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             examined = buffer.End;
             consumedBytes = 0;
 
+            if (buffer.IsSingleSpan)
+            {
+                var result = TakeMessageHeadersSingleSpan(handler, buffer.First.Span, out consumedBytes);
+                consumed = buffer.Move(consumed, consumedBytes);
+
+                if (result)
+                {
+                    examined = consumed;
+                }
+
+                return result;
+            }
+
             var bufferEnd = buffer.End;
 
             var reader = new ReadableBufferReader(buffer);
@@ -307,23 +340,180 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
                     headerBuffer.CopyTo(span);
                 }
 
-                var nameStart = 0;
+                if (!TakeSingleHeader(span, out var nameStart, out var nameEnd, out var valueStart, out var valueEnd))
-                var nameEnd = -1;
+                {
-                var valueStart = -1;
+                    return false;
-                var valueEnd = -1;
+                }
+
+                // Skip the reader forward past the header line
+                reader.Skip(span.Length);
+
+                // Before accepting the header line, we need to see at least one character
+                // > so we can make sure there's no space or tab
+                var next = reader.Peek();
+
+                // TODO: We don't need to reject the line here, we can use the state machine
+                // to store the fact that we're reading a header value
+                if (next == -1)
+                {
+                    // If we can't see the next char then reject the entire line
+                    return false;
+                }
+
+                if (next == ByteSpace || next == ByteTab)
+                {
+                    // From https://tools.ietf.org/html/rfc7230#section-3.2.4:
+                    //
+                    // Historically, HTTP header field values could be extended over
+                    // multiple lines by preceding each extra line with at least one space
+                    // or horizontal tab (obs-fold).  This specification deprecates such
+                    // line folding except within the message/http media type
+                    // (Section 8.3.1).  A sender MUST NOT generate a message that includes
+                    // line folding (i.e., that has any field-value that contains a match to
+                    // the obs-fold rule) unless the message is intended for packaging
+                    // within the message/http media type.
+                    //
+                    // A server that receives an obs-fold in a request message that is not
+                    // within a message/http container MUST either reject the message by
+                    // sending a 400 (Bad Request), preferably with a representation
+                    // explaining that obsolete line folding is unacceptable, or replace
+                    // each received obs-fold with one or more SP octets prior to
+                    // interpreting the field value or forwarding the message downstream.
+                    RejectRequest(RequestRejectionReason.HeaderValueLineFoldingNotSupported);
+                }
+
+                var nameBuffer = span.Slice(nameStart, nameEnd - nameStart);
+                var valueBuffer = span.Slice(valueStart, valueEnd - valueStart);
+                consumedBytes += span.Length;
+
+                handler.OnHeader(nameBuffer, valueBuffer);
+
+                consumed = reader.Cursor;
+            }
+        }
+
+        private unsafe bool TakeMessageHeadersSingleSpan<T>(T handler, Span<byte> headersSpan, out int consumedBytes) where T : IHttpHeadersHandler
+        {
+            consumedBytes = 0;
+
+            var remaining = headersSpan.Length;
+            var index = 0;
+            fixed (byte* data = &headersSpan.DangerousGetPinnableReference())
+            {
+                while (true)
+                {
+                    if (remaining < 2)
+                    {
+                        return false;
+                    }
+
+                    var ch1 = data[index];
+                    var ch2 = data[index + 1];
+
+                    if (ch1 == ByteCR)
+                    {
+                        // Check for final CRLF.
+                        if (ch2 == ByteLF)
+                        {
+                            consumedBytes += 2;
+                            return true;
+                        }
+
+                        // Headers don't end in CRLF line.
+                        RejectRequest(RequestRejectionReason.HeadersCorruptedInvalidHeaderSequence);
+                    }
+                    else if (ch1 == ByteSpace || ch1 == ByteTab)
+                    {
+                        RejectRequest(RequestRejectionReason.HeaderLineMustNotStartWithWhitespace);
+                    }
+
+                    var endOfLineIndex = IndexOf(data, index, headersSpan.Length, ByteLF);
+
+                    if (endOfLineIndex == -1)
+                    {
+                        return false;
+                    }
+
+                    var span = new Span<byte>(data + index, (endOfLineIndex - index + 1));
+                    index += span.Length;
+
+                    if (!TakeSingleHeader(span, out var nameStart, out var nameEnd, out var valueStart, out var valueEnd))
+                    {
+                        return false;
+                    }
+
+                    if ((endOfLineIndex + 1) >= headersSpan.Length)
+                    {
+                        return false;
+                    }
+
+                    // Before accepting the header line, we need to see at least one character
+                    // > so we can make sure there's no space or tab
+                    var next = data[index];
+
+                    if (next == ByteSpace || next == ByteTab)
+                    {
+                        // From https://tools.ietf.org/html/rfc7230#section-3.2.4:
+                        //
+                        // Historically, HTTP header field values could be extended over
+                        // multiple lines by preceding each extra line with at least one space
+                        // or horizontal tab (obs-fold).  This specification deprecates such
+                        // line folding except within the message/http media type
+                        // (Section 8.3.1).  A sender MUST NOT generate a message that includes
+                        // line folding (i.e., that has any field-value that contains a match to
+                        // the obs-fold rule) unless the message is intended for packaging
+                        // within the message/http media type.
+                        //
+                        // A server that receives an obs-fold in a request message that is not
+                        // within a message/http container MUST either reject the message by
+                        // sending a 400 (Bad Request), preferably with a representation
+                        // explaining that obsolete line folding is unacceptable, or replace
+                        // each received obs-fold with one or more SP octets prior to
+                        // interpreting the field value or forwarding the message downstream.
+                        RejectRequest(RequestRejectionReason.HeaderValueLineFoldingNotSupported);
+                    }
+
+                    var nameBuffer = span.Slice(nameStart, nameEnd - nameStart);
+                    var valueBuffer = span.Slice(valueStart, valueEnd - valueStart);
+
+                    handler.OnHeader(nameBuffer, valueBuffer);
+
+                    consumedBytes += span.Length;
+                    remaining -= span.Length;
+                }
+            }
+        }
+
+        private unsafe int IndexOf(byte* data, int index, int length, byte value)
+        {
+            for (int i = index; i < length; i++)
+            {
+                if (data[i] == value)
+                {
+                    return i;
+                }
+            }
+            return -1;
+        }
+
+        private unsafe bool TakeSingleHeader(Span<byte> span, out int nameStart, out int nameEnd, out int valueStart, out int valueEnd)
+        {
+            nameStart = 0;
+            nameEnd = -1;
+            valueStart = -1;
+            valueEnd = -1;
+            var headerLineLength = span.Length;
             var nameHasWhitespace = false;
             var previouslyWhitespace = false;
-                var headerLineLength = span.Length;
 
             int i = 0;
-                var length = span.Length;
+            var done = false;
-                bool done = false;
             fixed (byte* data = &span.DangerousGetPinnableReference())
             {
                 switch (HeaderState.Name)
                 {
                     case HeaderState.Name:
-                            for (; i < length; i++)
+                        for (; i < headerLineLength; i++)
                         {
                             var ch = data[i];
                             if (ch == ByteColon)
@@ -349,7 +539,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
 
                         break;
                     case HeaderState.Whitespace:
-                            for (; i < length; i++)
+                        for (; i < headerLineLength; i++)
                         {
                             var ch = data[i];
                             var whitespace = ch == ByteTab || ch == ByteSpace || ch == ByteCR;
@@ -373,7 +563,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
 
                         break;
                     case HeaderState.ExpectValue:
-                            for (; i < length; i++)
+                        for (; i < headerLineLength; i++)
                         {
                             var ch = data[i];
                             var whitespace = ch == ByteTab || ch == ByteSpace;
@@ -429,55 +619,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
                 }
             }
 
-                if (!done)
+            return done;
-                {
-                    return false;
-                }
-
-                // Skip the reader forward past the header line
-                reader.Skip(headerLineLength);
-
-                // Before accepting the header line, we need to see at least one character
-                // > so we can make sure there's no space or tab
-                var next = reader.Peek();
-
-                // TODO: We don't need to reject the line here, we can use the state machine
-                // to store the fact that we're reading a header value
-                if (next == -1)
-                {
-                    // If we can't see the next char then reject the entire line
-                    return false;
-                }
-
-                if (next == ByteSpace || next == ByteTab)
-                {
-                    // From https://tools.ietf.org/html/rfc7230#section-3.2.4:
-                    //
-                    // Historically, HTTP header field values could be extended over
-                    // multiple lines by preceding each extra line with at least one space
-                    // or horizontal tab (obs-fold).  This specification deprecates such
-                    // line folding except within the message/http media type
-                    // (Section 8.3.1).  A sender MUST NOT generate a message that includes
-                    // line folding (i.e., that has any field-value that contains a match to
-                    // the obs-fold rule) unless the message is intended for packaging
-                    // within the message/http media type.
-                    //
-                    // A server that receives an obs-fold in a request message that is not
-                    // within a message/http container MUST either reject the message by
-                    // sending a 400 (Bad Request), preferably with a representation
-                    // explaining that obsolete line folding is unacceptable, or replace
-                    // each received obs-fold with one or more SP octets prior to
-                    // interpreting the field value or forwarding the message downstream.
-                    RejectRequest(RequestRejectionReason.HeaderValueLineFoldingNotSupported);
-                }
-
-                var nameBuffer = span.Slice(nameStart, nameEnd - nameStart);
-                var valueBuffer = span.Slice(valueStart, valueEnd - valueStart);
-                consumedBytes += headerLineLength;
-
-                handler.OnHeader(nameBuffer, valueBuffer);
-                consumed = reader.Cursor;
-            }
         }
 
         private static bool IsValidTokenChar(char c)

test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs

@@ -143,9 +143,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 
                 Frame.Reset();
 
-                ReadCursor consumed;
+                if (!Frame.TakeStartLine(readableBuffer, out var consumed, out var examined))
-                ReadCursor examined;
-                if (!Frame.TakeStartLine(readableBuffer, out consumed, out examined))
                 {
                     ThrowInvalidStartLine();
                 }