Speed up ParseRequestLine (#1463)

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs

@@ -1216,15 +1216,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             Log.ApplicationError(ConnectionId, ex);
         }
 
-        public void OnStartLine(HttpMethod method, HttpVersion version, Span<byte> target, Span<byte> path, Span<byte> query, Span<byte> customMethod)
+        public void OnStartLine(HttpMethod method, HttpVersion version, Span<byte> target, Span<byte> path, Span<byte> query, Span<byte> customMethod, bool pathEncoded)
         {
             // URIs are always encoded/escaped to ASCII https://tools.ietf.org/html/rfc3986#page-11
             // Multibyte Internationalized Resource Identifiers (IRIs) are first converted to utf8;
             // then encoded/escaped to ASCII  https://www.ietf.org/rfc/rfc3987.txt "Mapping of IRIs to URIs"
             string requestUrlPath;
             string rawTarget;
-            var needDecode = path.IndexOf(BytePercentage) >= 0;
+            if (pathEncoded)
-            if (needDecode)
             {
                 // Read raw target before mutating memory.
                 rawTarget = target.GetAsciiStringNonNullCharacters();

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/IHttpRequestLineHandler.cs

@@ -7,6 +7,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
 {
     public interface IHttpRequestLineHandler
     {
-        void OnStartLine(HttpMethod method, HttpVersion version, Span<byte> target, Span<byte> path, Span<byte> query, Span<byte> customMethod);
+        void OnStartLine(HttpMethod method, HttpVersion version, Span<byte> target, Span<byte> path, Span<byte> query, Span<byte> customMethod, bool pathEncoded);
     }
 }

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/KestrelHttpParser.cs

@@ -34,219 +34,130 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             consumed = buffer.Start;
             examined = buffer.End;
 
-            ReadCursor end;
+            // Prepare the first span
-            Span<byte> span;
+            var span = buffer.First.Span;
-
+            var lineIndex = span.IndexOfVectorized(ByteLF);
-            // If the buffer is a single span then use it to find the LF
+            if (lineIndex >= 0)
-            if (buffer.IsSingleSpan)
             {
-                var startLineSpan = buffer.First.Span;
+                consumed = buffer.Move(consumed, lineIndex + 1);
-                var lineIndex = startLineSpan.IndexOfVectorized(ByteLF);
+                span = span.Slice(0, lineIndex + 1);
-
-                if (lineIndex == -1)
-                {
-                    return false;
-                }
-
-                end = buffer.Move(consumed, lineIndex + 1);
-                span = startLineSpan.Slice(0, lineIndex + 1);
             }
-            else
+            else if (buffer.IsSingleSpan || !TryGetNewLineSpan(ref buffer, ref span, out consumed))
             {
-                var start = buffer.Start;
+                // No request line end
-                if (ReadCursorOperations.Seek(start, buffer.End, out end, ByteLF) == -1)
+                return false;
-                {
-                    return false;
-                }
-
-                // Move 1 byte past the \n
-                end = buffer.Move(end, 1);
-                var startLineBuffer = buffer.Slice(start, end);
-
-                span = startLineBuffer.ToSpan();
             }
 
-            var pathStart = -1;
+            // Fix and parse the span
-            var queryStart = -1;
-            var queryEnd = -1;
-            var pathEnd = -1;
-            var versionStart = -1;
-
-            var httpVersion = HttpVersion.Unknown;
-            HttpMethod method;
-            Span<byte> customMethod;
-            var i = 0;
-            var length = span.Length;
-            var done = false;
-
             fixed (byte* data = &span.DangerousGetPinnableReference())
             {
-                switch (StartLineState.KnownMethod)
+                ParseRequestLine(handler, data, span.Length);
+            }
+
+            examined = consumed;
+            return true;
+        }
+
+        private unsafe void ParseRequestLine<T>(T handler, byte* data, int length) where T : IHttpRequestLineHandler
+        {
+            int offset;
+            Span<byte> customMethod;
+            // Get Method and set the offset
+            var method = HttpUtilities.GetKnownMethod(data, length, out offset);
+            if (method == HttpMethod.Custom)
+            {
+                customMethod = GetUnknownMethod(data, length, out offset);
+            }
+
+            // Skip space
+            offset++;
+
+            byte ch = 0;
+            // Target = Path and Query
+            var pathEncoded = false;
+            var pathStart = -1;
+            for (; offset < length; offset++)
+            {
+                ch = data[offset];
+                if (ch == ByteSpace)
                 {
-                    case StartLineState.KnownMethod:
+                    if (pathStart == -1)
-                        if (span.GetKnownMethod(out method, out var methodLength))
+                    {
-                        {
+                        // Empty path is illegal
-                            // Update the index, current char, state and jump directly
+                        RejectRequestLine(data, length);
-                            // to the next state
+                    }
-                            i += methodLength + 1;
 
-                            goto case StartLineState.Path;
+                    break;
-                        }
+                }
-                        goto case StartLineState.UnknownMethod;
+                else if (ch == ByteQuestionMark)
+                {
+                    if (pathStart == -1)
+                    {
+                        // Empty path is illegal
+                        RejectRequestLine(data, length);
+                    }
 
-                    case StartLineState.UnknownMethod:
+                    break;
-                        for (; i < length; i++)
+                }
-                        {
+                else if (ch == BytePercentage)
-                            var ch = data[i];
+                {
+                    if (pathStart == -1)
+                    {
+                        // Path starting with % is illegal
+                        RejectRequestLine(data, length);
+                    }
 
-                            if (ch == ByteSpace)
+                    pathEncoded = true;
-                            {
+                }
-                                customMethod = span.Slice(0, i);
+                else if (pathStart == -1)
-
+                {
-                                if (customMethod.Length == 0)
+                    pathStart = offset;
-                                {
-                                    RejectRequestLine(span);
-                                }
-                                // Consume space
-                                i++;
-
-                                goto case StartLineState.Path;
-                            }
-
-                            if (!IsValidTokenChar((char)ch))
-                            {
-                                RejectRequestLine(span);
-                            }
-                        }
-
-                        break;
-                    case StartLineState.Path:
-                        for (; i < length; i++)
-                        {
-                            var ch = data[i];
-                            if (ch == ByteSpace)
-                            {
-                                pathEnd = i;
-
-                                if (pathStart == -1)
-                                {
-                                    // Empty path is illegal
-                                    RejectRequestLine(span);
-                                }
-
-                                // No query string found
-                                queryStart = queryEnd = i;
-
-                                // Consume space
-                                i++;
-
-                                goto case StartLineState.KnownVersion;
-                            }
-                            else if (ch == ByteQuestionMark)
-                            {
-                                pathEnd = i;
-
-                                if (pathStart == -1)
-                                {
-                                    // Empty path is illegal
-                                    RejectRequestLine(span);
-                                }
-
-                                queryStart = i;
-                                goto case StartLineState.QueryString;
-                            }
-                            else if (ch == BytePercentage)
-                            {
-                                if (pathStart == -1)
-                                {
-                                    RejectRequestLine(span);
-                                }
-                            }
-
-                            if (pathStart == -1)
-                            {
-                                pathStart = i;
-                            }
-                        }
-                        break;
-                    case StartLineState.QueryString:
-                        for (; i < length; i++)
-                        {
-                            var ch = data[i];
-                            if (ch == ByteSpace)
-                            {
-                                queryEnd = i;
-
-                                // Consume space
-                                i++;
-
-                                goto case StartLineState.KnownVersion;
-                            }
-                        }
-                        break;
-                    case StartLineState.KnownVersion:
-                        // REVIEW: We don't *need* to slice here but it makes the API
-                        // nicer, slicing should be free :)
-                        if (span.Slice(i).GetKnownVersion(out httpVersion, out var versionLenght))
-                        {
-                            // Update the index, current char, state and jump directly
-                            // to the next state
-                            i += versionLenght + 1;
-                            goto case StartLineState.NewLine;
-                        }
-
-                        versionStart = i;
-
-                        goto case StartLineState.UnknownVersion;
-
-                    case StartLineState.UnknownVersion:
-                        for (; i < length; i++)
-                        {
-                            var ch = data[i];
-                            if (ch == ByteCR)
-                            {
-                                var versionSpan = span.Slice(versionStart, i - versionStart);
-
-                                if (versionSpan.Length == 0)
-                                {
-                                    RejectRequestLine(span);
-                                }
-                                else
-                                {
-                                    RejectRequest(RequestRejectionReason.UnrecognizedHTTPVersion,
-                                        versionSpan.GetAsciiStringEscaped(32));
-                                }
-                            }
-                        }
-                        break;
-                    case StartLineState.NewLine:
-                        if (data[i] != ByteLF)
-                        {
-                            RejectRequestLine(span);
-                        }
-                        i++;
-
-                        goto case StartLineState.Complete;
-                    case StartLineState.Complete:
-                        done = true;
-                        break;
                 }
             }
 
-            if (!done)
+            if (pathStart == -1)
             {
-                RejectRequestLine(span);
+                // End of path not found
+                RejectRequestLine(data, length);
             }
 
-            var pathBuffer = span.Slice(pathStart, pathEnd - pathStart);
+            var pathBuffer = new Span<byte>(data + pathStart, offset - pathStart);
-            var targetBuffer = span.Slice(pathStart, queryEnd - pathStart);
-            var query = span.Slice(queryStart, queryEnd - queryStart);
 
-            handler.OnStartLine(method, httpVersion, targetBuffer, pathBuffer, query, customMethod);
+            var queryStart = offset;
+            // Query string
+            if (ch == ByteQuestionMark)
+            {
+                // We have a query string
+                for (; offset < length; offset++)
+                {
+                    ch = data[offset];
+                    if (ch == ByteSpace)
+                    {
+                        break;
+                    }
+                }
+            }
 
-            consumed = end;
+            var targetBuffer = new Span<byte>(data + pathStart, offset - pathStart);
-            examined = consumed;
+            var query = new Span<byte>(data + queryStart, offset - queryStart);
-            return true;
+
+            // Consume space
+            offset++;
+
+            // Version
+            var httpVersion = HttpUtilities.GetKnownVersion(data + offset, length - offset);
+            if (httpVersion == HttpVersion.Unknown)
+            {
+                RejectUnknownVersion(data, length, offset);
+            }
+
+            //  After version 8 bytes and cr 1 byte, expect lf
+            if (data[offset + 8 + 1] != ByteLF)
+            {
+                RejectRequestLine(data, length);
+            }
+
+            handler.OnStartLine(method, httpVersion, targetBuffer, pathBuffer, query, customMethod, pathEncoded);
         }
 
         public unsafe bool ParseHeaders<T>(T handler, ReadableBuffer buffer, out ReadCursor consumed, out ReadCursor examined, out int consumedBytes) where T : IHttpHeadersHandler
@@ -502,6 +413,48 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             return true;
         }
 
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static bool TryGetNewLineSpan(ref ReadableBuffer buffer, ref Span<byte> span, out ReadCursor end)
+        {
+            var start = buffer.Start;
+            if (ReadCursorOperations.Seek(start, buffer.End, out end, ByteLF) != -1)
+            {
+                // Move 1 byte past the \n
+                end = buffer.Move(end, 1);
+                span = buffer.Slice(start, end).ToSpan();
+                return true;
+            }
+            return false;
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private unsafe Span<byte> GetUnknownMethod(byte* data, int length, out int methodLength)
+        {
+            methodLength = 0;
+            for (var i = 0; i < length; i++)
+            {
+                var ch = data[i];
+
+                if (ch == ByteSpace)
+                {
+                    if (i == 0)
+                    {
+                        RejectRequestLine(data, length);
+                    }
+
+                    methodLength = i;
+                    break;
+                }
+                else if (!IsValidTokenChar((char)ch))
+                {
+                    RejectRequestLine(data, length);
+                }
+            }
+
+            return new Span<byte>(data, methodLength);
+        }
+
         private static bool IsValidTokenChar(char c)
         {
             // Determines if a character is valid as a 'token' as defined in the
@@ -532,9 +485,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             throw BadHttpRequestException.GetException(reason);
         }
 
-        public static void RejectRequest(RequestRejectionReason reason, string value)
+        private unsafe void RejectUnknownVersion(byte* data, int length, int versionStart)
         {
-            throw BadHttpRequestException.GetException(reason, value);
+            throw GetRejectUnknownVersion(data, length, versionStart);
+        }
+
+        private unsafe void RejectRequestLine(byte* data, int length)
+        {
+            throw GetRejectRequestLineException(new Span<byte>(data, length));
         }
 
         private void RejectRequestLine(Span<byte> span)
@@ -549,6 +507,30 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
                 Log.IsEnabled(LogLevel.Information) ? span.GetAsciiStringEscaped(MaxRequestLineError) : string.Empty);
         }
 
+        private unsafe BadHttpRequestException GetRejectUnknownVersion(byte* data, int length, int versionStart)
+        {
+            var span = new Span<byte>(data, length);
+            length -= versionStart;
+            for (var i = 0; i < length; i++)
+            {
+                var ch = span[i + versionStart];
+                if (ch == ByteCR)
+                {
+                    if (i == 0)
+                    {
+                        return GetRejectRequestLineException(span);
+                    }
+                    else
+                    {
+                        return BadHttpRequestException.GetException(RequestRejectionReason.UnrecognizedHTTPVersion,
+                            span.Slice(versionStart, i).GetAsciiStringEscaped(32));
+                    }
+                }
+            }
+
+            return GetRejectRequestLineException(span);
+        }
+
         private unsafe void RejectRequestHeader(byte* headerLine, int length)
         {
             RejectRequestHeader(new Span<byte>(headerLine, length));
@@ -578,26 +560,5 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
             // https://github.com/dotnet/coreclr/issues/7459#issuecomment-253965670
             return Vector.AsVectorByte(new Vector<uint>(vectorByte * 0x01010101u));
         }
-
-        private enum HeaderState
-        {
-            Name,
-            Whitespace,
-            ExpectValue,
-            ExpectNewLine,
-            Complete
-        }
-
-        private enum StartLineState
-        {
-            KnownMethod,
-            UnknownMethod,
-            Path,
-            QueryString,
-            KnownVersion,
-            UnknownVersion,
-            NewLine,
-            Complete
-        }
     }
 }

src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/HttpUtilities.cs

@@ -147,34 +147,46 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure
         /// </remarks>
         /// <returns><c>true</c> if the input matches a known string, <c>false</c> otherwise.</returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static bool GetKnownMethod(this Span<byte> span, out HttpMethod method, out int length)
+        public static unsafe bool GetKnownMethod(this Span<byte> span, out HttpMethod method, out int length)
         {
-            if (span.TryRead<uint>(out var possiblyGet))
+            fixed (byte* data = &span.DangerousGetPinnableReference())
             {
-                if (possiblyGet == _httpGetMethodInt)
+                method = GetKnownMethod(data, span.Length, out length);
-                {
+                return method != HttpMethod.Custom;
-                    length = 3;
-                    method = HttpMethod.Get;
-                    return true;
-                }
             }
+        }
 
-            if (span.TryRead<ulong>(out var value))
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal unsafe static HttpMethod GetKnownMethod(byte* data, int length, out int methodLength)
+        {
+            methodLength = 0;
+            if (length < sizeof(uint))
             {
+                return HttpMethod.Custom;
+            }
+            else if (*(uint*)data == _httpGetMethodInt)
+            {
+                methodLength = 3;
+                return HttpMethod.Get;
+            }
+            else if (length < sizeof(ulong))
+            {
+                return HttpMethod.Custom;
+            }
+            else
+            {
+                var value = *(ulong*)data;
                 foreach (var x in _knownMethods)
                 {
                     if ((value & x.Item1) == x.Item2)
                     {
-                        method = x.Item3;
+                        methodLength = x.Item4;
-                        length = x.Item4;
+                        return x.Item3;
-                        return true;
                     }
                 }
             }
 
-            method = HttpMethod.Custom;
+            return HttpMethod.Custom;
-            length = 0;
-            return false;
         }
 
         /// <summary>
@@ -189,36 +201,56 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure
         /// </remarks>
         /// <returns><c>true</c> if the input matches a known string, <c>false</c> otherwise.</returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static bool GetKnownVersion(this Span<byte> span, out HttpVersion knownVersion, out byte length)
+        public static unsafe bool GetKnownVersion(this Span<byte> span, out HttpVersion knownVersion, out byte length)
         {
-            if (span.TryRead<ulong>(out var version))
+            fixed (byte* data = &span.DangerousGetPinnableReference())
             {
-                if (version == _http11VersionLong)
+                knownVersion = GetKnownVersion(data, span.Length);
+                if (knownVersion != HttpVersion.Unknown)
                 {
                     length = sizeof(ulong);
-                    knownVersion = HttpVersion.Http11;
-                }
-                else if (version == _http10VersionLong)
-                {
-                    length = sizeof(ulong);
-                    knownVersion = HttpVersion.Http10;
-                }
-                else
-                {
-                    length = 0;
-                    knownVersion = HttpVersion.Unknown;
-                    return false;
-                }
-
-                if (span[sizeof(ulong)] == (byte)'\r')
-                {
                     return true;
                 }
+
+                length = 0;
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Checks 9 bytes from <paramref name="location"/>  correspond to a known HTTP version.
+        /// </summary>
+        /// <remarks>
+        /// A "known HTTP version" Is is either HTTP/1.0 or HTTP/1.1.
+        /// Since those fit in 8 bytes, they can be optimally looked up by reading those bytes as a long. Once
+        /// in that format, it can be checked against the known versions.
+        /// The Known versions will be checked with the required '\r'.
+        /// To optimize performance the HTTP/1.1 will be checked first.
+        /// </remarks>
+        /// <returns><c>true</c> if the input matches a known string, <c>false</c> otherwise.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal unsafe static HttpVersion GetKnownVersion(byte* location, int length)
+        {
+            HttpVersion knownVersion;
+            var version = *(ulong*)location;
+            if (length < sizeof(ulong) + 1 || location[sizeof(ulong)] != (byte)'\r')
+            {
+                knownVersion = HttpVersion.Unknown;
+            }
+            else if (version == _http11VersionLong)
+            {
+                knownVersion = HttpVersion.Http11;
+            }
+            else if (version == _http10VersionLong)
+            {
+                knownVersion = HttpVersion.Http10;
+            }
+            else
+            {
+                knownVersion = HttpVersion.Unknown;
             }
 
-            knownVersion = HttpVersion.Unknown;
+            return knownVersion;
-            length = 0;
-            return false;
         }
 
         public static string VersionToString(HttpVersion httpVersion)

test/Microsoft.AspNetCore.Server.Kestrel.Performance/FrameParsingOverhead.cs

@@ -119,7 +119,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 
             public bool ParseRequestLine<T>(T handler, ReadableBuffer buffer, out ReadCursor consumed, out ReadCursor examined) where T : IHttpRequestLineHandler
             {
-                handler.OnStartLine(HttpMethod.Get, HttpVersion.Http11, new Span<byte>(_target), new Span<byte>(_target), Span<byte>.Empty, Span<byte>.Empty);
+                handler.OnStartLine(HttpMethod.Get, HttpVersion.Http11, new Span<byte>(_target), new Span<byte>(_target), Span<byte>.Empty, Span<byte>.Empty, false);
 
                 consumed = buffer.Start;
                 examined = buffer.End;

test/Microsoft.AspNetCore.Server.Kestrel.Performance/KestrelHttpParser.cs

@@ -66,7 +66,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
             }
         }
 
-        public void OnStartLine(HttpMethod method, HttpVersion version, Span<byte> target, Span<byte> path, Span<byte> query, Span<byte> customMethod)
+        public void OnStartLine(HttpMethod method, HttpVersion version, Span<byte> target, Span<byte> path, Span<byte> query, Span<byte> customMethod, bool pathEncoded)
         {
         }
 

test/Microsoft.AspNetCore.Server.KestrelTests/HttpParserTests.cs

@@ -49,14 +49,16 @@ namespace Microsoft.AspNetCore.Server.KestrelTests
                     It.IsAny<Span<byte>>(),
                     It.IsAny<Span<byte>>(),
                     It.IsAny<Span<byte>>(),
-                    It.IsAny<Span<byte>>()))
+                    It.IsAny<Span<byte>>(),
-                .Callback<HttpMethod, HttpVersion, Span<byte>, Span<byte>, Span<byte>, Span<byte>>((method, version, target, path, query, customMethod) =>
+                    It.IsAny<bool>()))
+                .Callback<HttpMethod, HttpVersion, Span<byte>, Span<byte>, Span<byte>, Span<byte>, bool>((method, version, target, path, query, customMethod, pathEncoded) =>
                 {
                     parsedMethod = method != HttpMethod.Custom ? HttpUtilities.MethodToString(method) : customMethod.GetAsciiStringNonNullCharacters();
                     parsedVersion = HttpUtilities.VersionToString(version);
                     parsedRawTarget = target.GetAsciiStringNonNullCharacters();
                     parsedRawPath = path.GetAsciiStringNonNullCharacters();
                     parsedQuery = query.GetAsciiStringNonNullCharacters();
+                    pathEncoded = false;
                 });
 
             Assert.True(parser.ParseRequestLine(requestLineHandler.Object, buffer, out var consumed, out var examined));