diff --git a/.gitignore b/.gitignore index 6acc284439..af0898e29a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ runtimes/ .build/ .testPublish/ launchSettings.json +BenchmarkDotNet.Artifacts/ +BDN.Generated/ diff --git a/KestrelHttpServer.sln b/KestrelHttpServer.sln index 1a5cda8a81..d78ceec5ad 100644 --- a/KestrelHttpServer.sln +++ b/KestrelHttpServer.sln @@ -37,7 +37,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{0EF2AC test\shared\DummyApplication.cs = test\shared\DummyApplication.cs test\shared\HttpClientSlim.cs = test\shared\HttpClientSlim.cs test\shared\LifetimeNotImplemented.cs = test\shared\LifetimeNotImplemented.cs + test\shared\MockConnection.cs = test\shared\MockConnection.cs + test\shared\MockFrameControl.cs = test\shared\MockFrameControl.cs test\shared\MockSystemClock.cs = test\shared\MockSystemClock.cs + test\shared\SocketInputExtensions.cs = test\shared\SocketInputExtensions.cs test\shared\TestApplicationErrorLogger.cs = test\shared\TestApplicationErrorLogger.cs test\shared\TestConnection.cs = test\shared\TestConnection.cs test\shared\TestKestrelTrace.cs = test\shared\TestKestrelTrace.cs @@ -45,6 +48,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{0EF2AC test\shared\TestServiceContext.cs = test\shared\TestServiceContext.cs EndProjectSection EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.AspNetCore.Server.Kestrel.Performance", "test\Microsoft.AspNetCore.Server.Kestrel.Performance\Microsoft.AspNetCore.Server.Kestrel.Performance.xproj", "{70567566-524C-4B67-9B59-E5C206D6C2EB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -79,6 +84,10 @@ Global {9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Debug|Any CPU.Build.0 = Debug|Any CPU {9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Release|Any CPU.ActiveCfg = Release|Any CPU {9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Release|Any CPU.Build.0 = Release|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -92,5 +101,6 @@ Global {5F64B3C3-0C2E-431A-B820-A81BBFC863DA} = {2D5D5227-4DBD-499A-96B1-76A36B03B750} {9559A5F1-080C-4909-B6CF-7E4B3DC55748} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} {0EF2ACDF-012F-4472-A13A-4272419E2903} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} + {70567566-524C-4B67-9B59-E5C206D6C2EB} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} EndGlobalSection EndGlobal diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs index 307ce7bc49..2b7f56e882 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Net; -using System.Numerics; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -24,6 +23,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { public abstract partial class Frame : IFrameControl { + // byte types don't have a data type annotation so we pre-cast them; to avoid in-place casts + private const byte ByteCR = (byte)'\r'; + private const byte ByteLF = (byte)'\n'; + private const byte ByteColon = (byte)':'; + private const byte ByteSpace = (byte)' '; + private const byte ByteTab = (byte)'\t'; + private const byte ByteQuestionMark = (byte)'?'; + private const byte BytePercentage = (byte)'%'; + private static readonly ArraySegment _endChunkedResponseBytes = CreateAsciiByteArraySegment("0\r\n\r\n"); private static readonly ArraySegment _continueBytes = CreateAsciiByteArraySegment("HTTP/1.1 100 Continue\r\n\r\n"); @@ -35,14 +43,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http private static readonly byte[] _bytesEndHeaders = Encoding.ASCII.GetBytes("\r\n\r\n"); private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: Kestrel"); - private static Vector _vectorCRs = new Vector((byte)'\r'); - private static Vector _vectorLFs = new Vector((byte)'\n'); - private static Vector _vectorColons = new Vector((byte)':'); - private static Vector _vectorSpaces = new Vector((byte)' '); - private static Vector _vectorTabs = new Vector((byte)'\t'); - private static Vector _vectorQuestionMarks = new Vector((byte)'?'); - private static Vector _vectorPercentages = new Vector((byte)'%'); - private readonly object _onStartingSync = new Object(); private readonly object _onCompletedSync = new Object(); @@ -952,7 +952,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http _requestProcessingStatus = RequestProcessingStatus.RequestStarted; int bytesScanned; - if (end.Seek(ref _vectorLFs, out bytesScanned, ServerOptions.Limits.MaxRequestLineSize) == -1) + if (end.Seek(ByteLF, out bytesScanned, ServerOptions.Limits.MaxRequestLineSize) == -1) { if (bytesScanned >= ServerOptions.Limits.MaxRequestLineSize) { @@ -969,13 +969,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var begin = scan; if (!begin.GetKnownMethod(out method)) { - if (scan.Seek(ref _vectorSpaces, ref end) == -1) + if (scan.Seek(ByteSpace, ref end) == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); } - method = begin.GetAsciiString(scan); + method = begin.GetAsciiString(ref scan); if (method == null) { @@ -1002,16 +1002,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http scan.Take(); begin = scan; var needDecode = false; - var chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref _vectorPercentages, ref end); + var chFound = scan.Seek(ByteSpace, ByteQuestionMark, BytePercentage, ref end); if (chFound == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); } - else if (chFound == '%') + else if (chFound == BytePercentage) { needDecode = true; - chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref end); + chFound = scan.Seek(ByteSpace, ByteQuestionMark, ref end); if (chFound == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, @@ -1023,20 +1023,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var pathEnd = scan; var queryString = ""; - if (chFound == '?') + if (chFound == ByteQuestionMark) { begin = scan; - if (scan.Seek(ref _vectorSpaces, ref end) == -1) + if (scan.Seek(ByteSpace, ref end) == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); } - queryString = begin.GetAsciiString(scan); + queryString = begin.GetAsciiString(ref scan); } var queryEnd = scan; - if (pathBegin.Peek() == ' ') + if (pathBegin.Peek() == ByteSpace) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1044,7 +1044,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http scan.Take(); begin = scan; - if (scan.Seek(ref _vectorCRs, ref end) == -1) + if (scan.Seek(ByteCR, ref end) == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1067,7 +1067,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } scan.Take(); // consume CR - if (scan.Take() != '\n') + if (scan.Take() != ByteLF) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1081,16 +1081,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http if (needDecode) { // Read raw target before mutating memory. - rawTarget = pathBegin.GetAsciiString(queryEnd); + rawTarget = pathBegin.GetAsciiString(ref queryEnd); // URI was encoded, unescape and then parse as utf8 pathEnd = UrlPathDecoder.Unescape(pathBegin, pathEnd); - requestUrlPath = pathBegin.GetUtf8String(pathEnd); + requestUrlPath = pathBegin.GetUtf8String(ref pathEnd); } else { // URI wasn't encoded, parse as ASCII - requestUrlPath = pathBegin.GetAsciiString(pathEnd); + requestUrlPath = pathBegin.GetAsciiString(ref pathEnd); if (queryString.Length == 0) { @@ -1100,7 +1100,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } else { - rawTarget = pathBegin.GetAsciiString(queryEnd); + rawTarget = pathBegin.GetAsciiString(ref queryEnd); } } @@ -1208,7 +1208,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { return false; } - else if (ch == '\r') + else if (ch == ByteCR) { // Check for final CRLF. end.Take(); @@ -1218,7 +1218,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { return false; } - else if (ch == '\n') + else if (ch == ByteLF) { ConnectionControl.CancelTimeout(); consumed = end; @@ -1228,7 +1228,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http // Headers don't end in CRLF line. RejectRequest(RequestRejectionReason.HeadersCorruptedInvalidHeaderSequence); } - else if (ch == ' ' || ch == '\t') + else if (ch == ByteSpace || ch == ByteTab) { RejectRequest(RequestRejectionReason.HeaderLineMustNotStartWithWhitespace); } @@ -1241,7 +1241,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } int bytesScanned; - if (end.Seek(ref _vectorLFs, out bytesScanned, _remainingRequestHeadersBytesAllowed) == -1) + if (end.Seek(ByteLF, out bytesScanned, _remainingRequestHeadersBytesAllowed) == -1) { if (bytesScanned >= _remainingRequestHeadersBytesAllowed) { @@ -1254,7 +1254,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } var beginName = scan; - if (scan.Seek(ref _vectorColons, ref end) == -1) + if (scan.Seek(ByteColon, ref end) == -1) { RejectRequest(RequestRejectionReason.NoColonCharacterFoundInHeaderLine); } @@ -1263,7 +1263,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http scan.Take(); var validateName = beginName; - if (validateName.Seek(ref _vectorSpaces, ref _vectorTabs, ref endName) != -1) + if (validateName.Seek(ByteSpace, ByteTab, ref endName) != -1) { RejectRequest(RequestRejectionReason.WhitespaceIsNotAllowedInHeaderName); } @@ -1271,14 +1271,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var beginValue = scan; ch = scan.Take(); - while (ch == ' ' || ch == '\t') + while (ch == ByteSpace || ch == ByteTab) { beginValue = scan; ch = scan.Take(); } scan = beginValue; - if (scan.Seek(ref _vectorCRs, ref end) == -1) + if (scan.Seek(ByteCR, ref end) == -1) { RejectRequest(RequestRejectionReason.MissingCRInHeaderLine); } @@ -1287,7 +1287,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http ch = scan.Take(); // expecting '\n' end = scan; - if (ch != '\n') + if (ch != ByteLF) { RejectRequest(RequestRejectionReason.HeaderValueMustNotContainCR); } @@ -1297,7 +1297,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { return false; } - else if (next == ' ' || next == '\t') + else if (next == ByteSpace || next == ByteTab) { // From https://tools.ietf.org/html/rfc7230#section-3.2.4: // @@ -1330,18 +1330,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var endValue = scan; do { - ws.Seek(ref _vectorSpaces, ref _vectorTabs, ref _vectorCRs); + ws.Seek(ByteSpace, ByteTab, ByteCR); endValue = ws; ch = ws.Take(); - while (ch == ' ' || ch == '\t') + while (ch == ByteSpace || ch == ByteTab) { ch = ws.Take(); } - } while (ch != '\r'); + } while (ch != ByteCR); var name = beginName.GetArraySegment(endName); - var value = beginValue.GetAsciiString(endValue); + var value = beginValue.GetAsciiString(ref endValue); consumed = scan; requestHeaders.Append(name.Array, name.Offset, name.Count, value); diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs index 5fb8fa2c3a..77e9b9adcd 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using System.Numerics; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; @@ -390,9 +389,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http /// private class ForChunkedEncoding : MessageBody { - // This causes an InvalidProgramException if made static - // https://github.com/dotnet/corefx/issues/8825 - private Vector _vectorCRs = new Vector((byte)'\r'); + // byte consts don't have a data type annotation so we pre-cast it + private const byte ByteCR = (byte)'\r'; private readonly SocketInput _input; private readonly FrameRequestHeaders _requestHeaders; @@ -613,7 +611,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http // Just drain the data do { - if (scan.Seek(ref _vectorCRs) == -1) + if (scan.Seek(ByteCR) == -1) { // End marker not found yet consumed = scan; diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs index 77968bca17..3acb659218 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Runtime.InteropServices; +using System.Text; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 5f7e908c36..b1b45eb6e8 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -4,12 +4,23 @@ using System; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; +using System.Text; using System.Threading; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { public struct MemoryPoolIterator { + private const ulong _xorPowerOfTwoToHighByte = (0x07ul | + 0x06ul << 8 | + 0x05ul << 16 | + 0x04ul << 24 | + 0x03ul << 32 | + 0x02ul << 40 | + 0x01ul << 48 ) + 1; + private static readonly int _vectorSpan = Vector.Count; private MemoryPoolBlock _block; @@ -30,36 +41,50 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public bool IsEnd { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - if (_block == null) + var block = _block; + if (block == null) { return true; } - else if (_index < _block.End) + else if (_index < block.End) { return false; } - else + else if (block.Next == null) { - var block = _block.Next; - while (block != null) - { - if (block.Start < block.End) - { - return false; // subsequent block has data - IsEnd is false - } - block = block.Next; - } return true; } + else + { + return IsEndMultiBlock(); + } } } + [MethodImpl(MethodImplOptions.NoInlining)] + private bool IsEndMultiBlock() + { + var block = _block.Next; + do + { + if (block.Start < block.End) + { + return false; // subsequent block has data - IsEnd is false + } + block = block.Next; + } while (block != null); + + return true; + } + public MemoryPoolBlock Block => _block; public int Index => _index; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Take() { var block = _block; @@ -69,6 +94,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } var index = _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss var wasLastBlock = block.Next == null; if (index < block.End) @@ -77,19 +103,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return block.Array[index]; } + return wasLastBlock ? -1 : TakeMultiBlock(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private int TakeMultiBlock() + { + var block = _block; do { - if (wasLastBlock) - { - return -1; - } - else - { - block = block.Next; - index = block.Start; - } + block = block.Next; + var index = block.Start; - wasLastBlock = block.Next == null; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { @@ -97,18 +124,26 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _index = index + 1; return block.Array[index]; } + + if (wasLastBlock) + { + return -1; + } } while (true); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Skip(int bytesToSkip) { - if (_block == null) + var block = _block; + if (block == null && bytesToSkip > 0) { - return; + ThrowInvalidOperationException_SkipMoreThanAvailable(); } - var wasLastBlock = _block.Next == null; - var following = _block.End - _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; + var following = block.End - _index; if (following >= bytesToSkip) { @@ -116,22 +151,26 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return; } - var block = _block; - var index = _index; - while (true) + if (wasLastBlock) { - if (wasLastBlock) - { - throw new InvalidOperationException("Attempted to skip more bytes than available."); - } - else - { - bytesToSkip -= following; - block = block.Next; - index = block.Start; - } + ThrowInvalidOperationException_SkipMoreThanAvailable(); + } - wasLastBlock = block.Next == null; + SkipMultiBlock(bytesToSkip, following); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void SkipMultiBlock(int bytesToSkip, int following) + { + var block = _block; + do + { + bytesToSkip -= following; + block = block.Next; + var index = block.Start; + + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; following = block.End - index; if (following >= bytesToSkip) @@ -140,9 +179,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _index = index + bytesToSkip; return; } - } + + if (wasLastBlock) + { + ThrowInvalidOperationException_SkipMoreThanAvailable(); + } + } while (true); } + private static void ThrowInvalidOperationException_SkipMoreThanAvailable() + { + throw new InvalidOperationException("Attempted to skip more bytes than available."); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Peek() { var block = _block; @@ -151,113 +201,123 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } - var wasLastBlock = _block.Next == null; var index = _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { return block.Array[index]; } + return wasLastBlock ? -1 : PeekMultiBlock(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private int PeekMultiBlock() + { + var block = _block; do { - if (wasLastBlock) - { - return -1; - } - else - { - block = block.Next; - index = block.Start; - } + block = block.Next; + var index = block.Start; - wasLastBlock = block.Next == null; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { return block.Array[index]; } + if (wasLastBlock) + { + return -1; + } } while (true); } // NOTE: Little-endian only! + [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe bool TryPeekLong(out ulong longValue) { longValue = 0; - if (_block == null) + var block = _block; + if (block == null) { return false; } - var wasLastBlock = _block.Next == null; - var blockBytes = _block.End - _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; + var blockBytes = block.End - _index; if (blockBytes >= sizeof(ulong)) { - longValue = *(ulong*)(_block.DataFixedPtr + _index); + longValue = *(ulong*)(block.DataFixedPtr + _index); return true; } - else if (wasLastBlock) + + return wasLastBlock ? false : TryPeekLongMultiBlock(ref longValue, blockBytes); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe bool TryPeekLongMultiBlock(ref ulong longValue, int blockBytes) + { + // Each block will be filled with at least 2048 bytes before the Next pointer is set, so a long + // will cross at most one block boundary assuming there are at least 8 bytes following the iterator. + var nextBytes = sizeof(ulong) - blockBytes; + + var block = _block; + if (block.Next.End - block.Next.Start < nextBytes) { return false; } + + var nextLong = *(ulong*)(block.Next.DataFixedPtr + block.Next.Start); + + if (blockBytes == 0) + { + // This case can not fall through to the else block since that would cause a 64-bit right shift + // on blockLong which is equivalent to no shift at all instead of shifting in all zeros. + // https://msdn.microsoft.com/en-us/library/xt18et0d.aspx + longValue = nextLong; + } else { - // Each block will be filled with at least 2048 bytes before the Next pointer is set, so a long - // will cross at most one block boundary assuming there are at least 8 bytes following the iterator. - var nextBytes = sizeof(ulong) - blockBytes; + var blockLong = *(ulong*)(block.DataFixedPtr + block.End - sizeof(ulong)); - if (_block.Next.End - _block.Next.Start < nextBytes) - { - return false; - } - - var nextLong = *(ulong*)(_block.Next.DataFixedPtr + _block.Next.Start); - - if (blockBytes == 0) - { - // This case can not fall through to the else block since that would cause a 64-bit right shift - // on blockLong which is equivalent to no shift at all instead of shifting in all zeros. - // https://msdn.microsoft.com/en-us/library/xt18et0d.aspx - longValue = nextLong; - } - else - { - var blockLong = *(ulong*)(_block.DataFixedPtr + _block.End - sizeof(ulong)); - - // Ensure that the right shift has a ulong operand so a logical shift is performed. - longValue = (blockLong >> nextBytes * 8) | (nextLong << blockBytes * 8); - } - - return true; + // Ensure that the right shift has a ulong operand so a logical shift is performed. + longValue = (blockLong >> nextBytes * 8) | (nextLong << blockBytes * 8); } + + return true; } - public int Seek(ref Vector byte0Vector) + public int Seek(byte byte0) { int bytesScanned; - return Seek(ref byte0Vector, out bytesScanned); + return Seek(byte0, out bytesScanned); } public unsafe int Seek( - ref Vector byte0Vector, + byte byte0, out int bytesScanned, int limit = int.MaxValue) { bytesScanned = 0; - if (IsDefault || limit <= 0) + var block = _block; + if (block == null || limit <= 0) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; - var byte0 = byte0Vector[0]; + var byte0Vector = GetVector(byte0); while (true) { @@ -307,7 +367,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(byte0Equals); var vectorBytesScanned = firstEqualByteIndex + 1; if (bytesScanned + vectorBytesScanned > limit) @@ -350,20 +410,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } public unsafe int Seek( - ref Vector byte0Vector, + byte byte0, ref MemoryPoolIterator limit) { - if (IsDefault) + var block = _block; + if (block == null) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - byte[] array; - var byte0 = byte0Vector[0]; while (true) { @@ -383,7 +441,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure wasLastBlock = block.Next == null; following = block.End - index; } - array = block.Array; + var array = block.Array; while (following > 0) { // Need unit tests to test Vector path @@ -394,7 +452,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure #endif if (following >= _vectorSpan) { - var byte0Equals = Vector.Equals(new Vector(array, index), byte0Vector); + var byte0Equals = Vector.Equals(new Vector(array, index), GetVector(byte0)); if (byte0Equals.Equals(Vector.Zero)) { @@ -413,7 +471,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(byte0Equals); if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) { @@ -451,31 +509,27 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public int Seek(ref Vector byte0Vector, ref Vector byte1Vector) + public int Seek(byte byte0, byte byte1) { var limit = new MemoryPoolIterator(); - return Seek(ref byte0Vector, ref byte1Vector, ref limit); + return Seek(byte0, byte1, ref limit); } public unsafe int Seek( - ref Vector byte0Vector, - ref Vector byte1Vector, + byte byte0, + byte byte1, ref MemoryPoolIterator limit) { - if (IsDefault) + var block = _block; + if (block == null) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - byte[] array; - int byte0Index = int.MaxValue; - int byte1Index = int.MaxValue; - var byte0 = byte0Vector[0]; - var byte1 = byte1Vector[0]; + int byteIndex = int.MaxValue; while (true) { @@ -494,7 +548,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure wasLastBlock = block.Next == null; following = block.End - index; } - array = block.Array; + var array = block.Array; while (following > 0) { @@ -507,19 +561,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (following >= _vectorSpan) { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); - if (!byte0Equals.Equals(Vector.Zero)) + var byteEquals = Vector.Equals(data, GetVector(byte0)); + byteEquals = Vector.ConditionalSelect(byteEquals, byteEquals, Vector.Equals(data, GetVector(byte1))); + + if (!byteEquals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); - } - if (!byte1Equals.Equals(Vector.Zero)) - { - byte1Index = FindFirstEqualByte(ref byte1Equals); + byteIndex = LocateFirstFoundByte(byteEquals); } - if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) + if (byteIndex == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; @@ -537,21 +588,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - if (byte0Index < byte1Index) - { - _index = index + byte0Index; - - if (block == limit.Block && _index > limit.Index) - { - // Ensure iterator is left at limit position - _index = limit.Index; - return -1; - } - - return byte0; - } - - _index = index + byte1Index; + _index = index + byteIndex; if (block == limit.Block && _index > limit.Index) { @@ -560,7 +597,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } - return byte1; + _index = index + byteIndex; + + if (block == limit.Block && _index > limit.Index) + { + // Ensure iterator is left at limit position + _index = limit.Index; + return -1; + } + + return block.Array[index + byteIndex]; } // Need unit tests to test Vector path #if !DEBUG @@ -592,34 +638,28 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public int Seek(ref Vector byte0Vector, ref Vector byte1Vector, ref Vector byte2Vector) + public int Seek(byte byte0, byte byte1, byte byte2) { var limit = new MemoryPoolIterator(); - return Seek(ref byte0Vector, ref byte1Vector, ref byte2Vector, ref limit); + return Seek(byte0, byte1, byte2, ref limit); } public unsafe int Seek( - ref Vector byte0Vector, - ref Vector byte1Vector, - ref Vector byte2Vector, + byte byte0, + byte byte1, + byte byte2, ref MemoryPoolIterator limit) { - if (IsDefault) + var block = _block; + if (block == null) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - byte[] array; - int byte0Index = int.MaxValue; - int byte1Index = int.MaxValue; - int byte2Index = int.MaxValue; - var byte0 = byte0Vector[0]; - var byte1 = byte1Vector[0]; - var byte2 = byte2Vector[0]; + int byteIndex = int.MaxValue; while (true) { @@ -638,7 +678,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure wasLastBlock = block.Next == null; following = block.End - index; } - array = block.Array; + var array = block.Array; while (following > 0) { // Need unit tests to test Vector path @@ -650,24 +690,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (following >= _vectorSpan) { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); - var byte2Equals = Vector.Equals(data, byte2Vector); - if (!byte0Equals.Equals(Vector.Zero)) + var byteEquals = Vector.Equals(data, GetVector(byte0)); + byteEquals = Vector.ConditionalSelect(byteEquals, byteEquals, Vector.Equals(data, GetVector(byte1))); + byteEquals = Vector.ConditionalSelect(byteEquals, byteEquals, Vector.Equals(data, GetVector(byte2))); + + if (!byteEquals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); - } - if (!byte1Equals.Equals(Vector.Zero)) - { - byte1Index = FindFirstEqualByte(ref byte1Equals); - } - if (!byte2Equals.Equals(Vector.Zero)) - { - byte2Index = FindFirstEqualByte(ref byte2Equals); + byteIndex = LocateFirstFoundByte(byteEquals); } - if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) + if (byteIndex == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; @@ -685,35 +718,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - int toReturn, toMove; - if (byte0Index < byte1Index) - { - if (byte0Index < byte2Index) - { - toReturn = byte0; - toMove = byte0Index; - } - else - { - toReturn = byte2; - toMove = byte2Index; - } - } - else - { - if (byte1Index < byte2Index) - { - toReturn = byte1; - toMove = byte1Index; - } - else - { - toReturn = byte2; - toMove = byte2Index; - } - } - - _index = index + toMove; + _index = index + byteIndex; if (block == limit.Block && _index > limit.Index) { @@ -722,7 +727,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } - return toReturn; + return block.Array[index + byteIndex]; } // Need unit tests to test Vector path #if !DEBUG @@ -761,60 +766,31 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } /// - /// Find first byte + /// Locate the first of the found bytes /// /// /// The first index of the result vector - /// byteEquals = 0 - internal static int FindFirstEqualByte(ref Vector byteEquals) + // Force inlining (64 IL bytes, 91 bytes asm) Issue: https://github.com/dotnet/coreclr/issues/7386 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LocateFirstFoundByte(Vector byteEquals) { - if (!BitConverter.IsLittleEndian) return FindFirstEqualByteSlow(ref byteEquals); - - // Quasi-tree search - var vector64 = Vector.AsVectorInt64(byteEquals); - for (var i = 0; i < Vector.Count; i++) + var vector64 = Vector.AsVectorUInt64(byteEquals); + ulong longValue = 0; + var i = 0; + // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001 + for (; i < Vector.Count; i++) { - var longValue = vector64[i]; + longValue = vector64[i]; if (longValue == 0) continue; - - return (i << 3) + - ((longValue & 0x00000000ffffffff) > 0 - ? (longValue & 0x000000000000ffff) > 0 - ? (longValue & 0x00000000000000ff) > 0 ? 0 : 1 - : (longValue & 0x0000000000ff0000) > 0 ? 2 : 3 - : (longValue & 0x0000ffff00000000) > 0 - ? (longValue & 0x000000ff00000000) > 0 ? 4 : 5 - : (longValue & 0x00ff000000000000) > 0 ? 6 : 7); + break; } - throw new InvalidOperationException(); - } - // Internal for testing - internal static int FindFirstEqualByteSlow(ref Vector byteEquals) - { - // Quasi-tree search - var vector64 = Vector.AsVectorInt64(byteEquals); - for (var i = 0; i < Vector.Count; i++) - { - var longValue = vector64[i]; - if (longValue == 0) continue; - - var shift = i << 1; - var offset = shift << 2; - var vector32 = Vector.AsVectorInt32(byteEquals); - if (vector32[shift] != 0) - { - if (byteEquals[offset] != 0) return offset; - if (byteEquals[offset + 1] != 0) return offset + 1; - if (byteEquals[offset + 2] != 0) return offset + 2; - return offset + 3; - } - if (byteEquals[offset + 4] != 0) return offset + 4; - if (byteEquals[offset + 5] != 0) return offset + 5; - if (byteEquals[offset + 6] != 0) return offset + 6; - return offset + 7; - } - throw new InvalidOperationException(); + // Flag least significant power of two bit + var powerOfTwoFlag = (longValue ^ (longValue - 1)); + // Shift all powers of two into the high byte and extract + var foundByteIndex = (int)((powerOfTwoFlag * _xorPowerOfTwoToHighByte) >> 57); + // Single LEA instruction with jitted const (using function result) + return i * 8 + foundByteIndex; } /// @@ -822,17 +798,44 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// /// The byte to be saved. /// true if the operation successes. false if can't find available space. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool Put(byte data) { - if (_block == null) + var block = _block; + if (block == null) { - return false; + ThrowInvalidOperationException_PutPassedEndOfBlock(); } - var block = _block; var index = _index; - while (true) + + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; + if (index < block.End) { + _index = index + 1; + block.Array[index] = data; + return true; + } + + if (wasLastBlock) + { + ThrowInvalidOperationException_PutPassedEndOfBlock(); + } + + return PutMultiBlock(data); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private bool PutMultiBlock(byte data) + { + var block = _block; + do + { + block = block.Next; + var index = block.Start; + + // Always set wasLastBlock before checking .End to avoid race which may cause data loss var wasLastBlock = block.Next == null; if (index < block.End) @@ -840,27 +843,48 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; _index = index + 1; block.Array[index] = data; - return true; + break; } - else if (wasLastBlock) + if (wasLastBlock) { + ThrowInvalidOperationException_PutPassedEndOfBlock(); return false; } - else - { - block = block.Next; - index = block.Start; - } - } + } while (true); + + return true; } + private static void ThrowInvalidOperationException_PutPassedEndOfBlock() + { + throw new InvalidOperationException("Attempted to put passed end of block."); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetLength(MemoryPoolIterator end) { - if (IsDefault || end.IsDefault) + var block = _block; + if (block == null || end.IsDefault) { - return -1; + ThrowInvalidOperationException_GetLengthNullBlock(); } + if (block == end._block) + { + return end._index - _index; + } + + return GetLengthMultiBlock(ref end); + } + + private static void ThrowInvalidOperationException_GetLengthNullBlock() + { + throw new InvalidOperationException("Attempted GetLength of non existent block."); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public int GetLengthMultiBlock(ref MemoryPoolIterator end) + { var block = _block; var index = _index; var length = 0; @@ -888,13 +912,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public MemoryPoolIterator CopyTo(byte[] array, int offset, int count, out int actual) { - if (IsDefault) + var block = _block; + if (block == null) { actual = 0; return this; } - var block = _block; var index = _index; var remaining = count; while (true) @@ -949,17 +973,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public void CopyFrom(byte[] data, int offset, int count) { - if (IsDefault) + var block = _block; + if (block == null) { return; } - Debug.Assert(_block != null); - Debug.Assert(_block.Next == null); - Debug.Assert(_block.End == _index); + Debug.Assert(block.Next == null); + Debug.Assert(block.End == _index); - var pool = _block.Pool; - var block = _block; + var pool = block.Pool; var blockIndex = _index; var bufferIndex = offset; @@ -996,17 +1019,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public unsafe void CopyFromAscii(string data) { - if (IsDefault) + var block = _block; + if (block == null) { return; } - Debug.Assert(_block != null); - Debug.Assert(_block.Next == null); - Debug.Assert(_block.End == _index); + Debug.Assert(block.Next == null); + Debug.Assert(block.End == _index); - var pool = _block.Pool; - var block = _block; + var pool = block.Pool; var blockIndex = _index; var length = data.Length; @@ -1059,5 +1081,178 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; _index = blockIndex; } + + public unsafe string GetAsciiString(ref MemoryPoolIterator end) + { + var block = _block; + if (block == null || end.IsDefault) + { + return null; + } + + var length = GetLength(end); + + if (length == 0) + { + return null; + } + + var inputOffset = _index; + + var asciiString = new string('\0', length); + + fixed (char* outputStart = asciiString) + { + var output = outputStart; + var remaining = length; + + var endBlock = end.Block; + var endIndex = end.Index; + + var outputOffset = 0; + while (true) + { + int following = (block != endBlock ? block.End : endIndex) - inputOffset; + + if (following > 0) + { + if (!AsciiUtilities.TryGetAsciiString(block.DataFixedPtr + inputOffset, output + outputOffset, following)) + { + throw BadHttpRequestException.GetException(RequestRejectionReason.NonAsciiOrNullCharactersInInputString); + } + + outputOffset += following; + remaining -= following; + } + + if (remaining == 0) + { + break; + } + + block = block.Next; + inputOffset = block.Start; + } + } + + return asciiString; + } + + public string GetUtf8String(ref MemoryPoolIterator end) + { + var block = _block; + if (block == null || end.IsDefault) + { + return default(string); + } + + var index = _index; + if (end.Block == block) + { + return Encoding.UTF8.GetString(block.Array, index, end.Index - index); + } + + var decoder = Encoding.UTF8.GetDecoder(); + + var length = GetLength(end); + var charLength = length; + // Worse case is 1 byte = 1 char + var chars = new char[charLength]; + var charIndex = 0; + + var remaining = length; + while (true) + { + int bytesUsed; + int charsUsed; + bool completed; + var following = block.End - index; + if (remaining <= following) + { + decoder.Convert( + block.Array, + index, + remaining, + chars, + charIndex, + charLength - charIndex, + true, + out bytesUsed, + out charsUsed, + out completed); + return new string(chars, 0, charIndex + charsUsed); + } + else if (block.Next == null) + { + decoder.Convert( + block.Array, + index, + following, + chars, + charIndex, + charLength - charIndex, + true, + out bytesUsed, + out charsUsed, + out completed); + return new string(chars, 0, charIndex + charsUsed); + } + else + { + decoder.Convert( + block.Array, + index, + following, + chars, + charIndex, + charLength - charIndex, + false, + out bytesUsed, + out charsUsed, + out completed); + charIndex += charsUsed; + remaining -= following; + block = block.Next; + index = block.Start; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ArraySegment GetArraySegment(MemoryPoolIterator end) + { + var block = _block; + if (block == null || end.IsDefault) + { + return default(ArraySegment); + } + + var index = _index; + if (end.Block == block) + { + return new ArraySegment(block.Array, index, end.Index - index); + } + + return GetArraySegmentMultiBlock(ref end); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private ArraySegment GetArraySegmentMultiBlock(ref MemoryPoolIterator end) + { + var length = GetLength(end); + var array = new byte[length]; + CopyTo(array, 0, length, out length); + return new ArraySegment(array, 0, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector GetVector(byte vectorByte) + { + // Vector .ctor doesn't become an intrinsic due to detection issue + // However this does cause it to become an intrinsic (with additional multiply and reg->reg copy) + // https://github.com/dotnet/coreclr/issues/7459#issuecomment-253965670 + return Vector.AsVectorByte(new Vector(vectorByte * 0x01010101u)); + } + } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs index d59f689662..1839d3309f 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs @@ -3,16 +3,14 @@ using System; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Text; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { public static class MemoryPoolIteratorExtensions { - private static readonly Encoding _utf8 = Encoding.UTF8; - public const string Http10Version = "HTTP/1.0"; public const string Http11Version = "HTTP/1.1"; @@ -71,62 +69,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public unsafe static string GetAsciiString(this MemoryPoolIterator start, MemoryPoolIterator end) - { - if (start.IsDefault || end.IsDefault) - { - return null; - } - - var length = start.GetLength(end); - - if (length == 0) - { - return null; - } - - var inputOffset = start.Index; - var block = start.Block; - - var asciiString = new string('\0', length); - - fixed (char* outputStart = asciiString) - { - var output = outputStart; - var remaining = length; - - var endBlock = end.Block; - var endIndex = end.Index; - - var outputOffset = 0; - while (true) - { - int following = (block != endBlock ? block.End : endIndex) - inputOffset; - - if (following > 0) - { - if (!AsciiUtilities.TryGetAsciiString(block.DataFixedPtr + inputOffset, output + outputOffset, following)) - { - throw BadHttpRequestException.GetException(RequestRejectionReason.NonAsciiOrNullCharactersInInputString); - } - - outputOffset += following; - remaining -= following; - } - - if (remaining == 0) - { - break; - } - - block = block.Next; - inputOffset = block.Start; - } - } - - return asciiString; - } - public static string GetAsciiStringEscaped(this MemoryPoolIterator start, MemoryPoolIterator end, int maxChars) { var sb = new StringBuilder(); @@ -147,102 +89,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return sb.ToString(); } - public static string GetUtf8String(this MemoryPoolIterator start, MemoryPoolIterator end) - { - if (start.IsDefault || end.IsDefault) - { - return default(string); - } - if (end.Block == start.Block) - { - return _utf8.GetString(start.Block.Array, start.Index, end.Index - start.Index); - } - - var decoder = _utf8.GetDecoder(); - - var length = start.GetLength(end); - var charLength = length; - // Worse case is 1 byte = 1 char - var chars = new char[charLength]; - var charIndex = 0; - - var block = start.Block; - var index = start.Index; - var remaining = length; - while (true) - { - int bytesUsed; - int charsUsed; - bool completed; - var following = block.End - index; - if (remaining <= following) - { - decoder.Convert( - block.Array, - index, - remaining, - chars, - charIndex, - charLength - charIndex, - true, - out bytesUsed, - out charsUsed, - out completed); - return new string(chars, 0, charIndex + charsUsed); - } - else if (block.Next == null) - { - decoder.Convert( - block.Array, - index, - following, - chars, - charIndex, - charLength - charIndex, - true, - out bytesUsed, - out charsUsed, - out completed); - return new string(chars, 0, charIndex + charsUsed); - } - else - { - decoder.Convert( - block.Array, - index, - following, - chars, - charIndex, - charLength - charIndex, - false, - out bytesUsed, - out charsUsed, - out completed); - charIndex += charsUsed; - remaining -= following; - block = block.Next; - index = block.Start; - } - } - } - - public static ArraySegment GetArraySegment(this MemoryPoolIterator start, MemoryPoolIterator end) - { - if (start.IsDefault || end.IsDefault) - { - return default(ArraySegment); - } - if (end.Block == start.Block) - { - return new ArraySegment(start.Block.Array, start.Index, end.Index - start.Index); - } - - var length = start.GetLength(end); - var array = new byte[length]; - start.CopyTo(array, 0, length, out length); - return new ArraySegment(array, 0, length); - } - public static ArraySegment PeekArraySegment(this MemoryPoolIterator iter) { if (iter.IsDefault || iter.IsEnd) @@ -283,6 +129,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// The iterator from which to start the known string lookup. /// A reference to a pre-allocated known string, if the input matches any. /// true if the input matches a known string, false otherwise. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool GetKnownMethod(this MemoryPoolIterator begin, out string knownMethod) { knownMethod = null; @@ -323,6 +170,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// The iterator from which to start the known string lookup. /// A reference to a pre-allocated known string, if the input matches any. /// true if the input matches a known string, false otherwise. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool GetKnownVersion(this MemoryPoolIterator begin, out string knownVersion) { knownVersion = null; diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs index 8989a4b649..86b8eff609 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.AspNetCore.Server.Kestrel.FunctionalTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] [assembly: InternalsVisibleTo("Microsoft.AspNetCore.Server.KestrelTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] +[assembly: InternalsVisibleTo("Microsoft.AspNetCore.Server.Kestrel.Performance, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] [assembly: AssemblyMetadata("Serviceable", "True")] [assembly: NeutralResourcesLanguage("en-us")] [assembly: AssemblyCompany("Microsoft Corporation.")] diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj new file mode 100644 index 0000000000..3bbc5601e0 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj @@ -0,0 +1,22 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 70567566-524c-4b67-9b59-e5c206d6c2eb + Microsoft.AspNetCore.Server.Kestrel.Performance + .\obj + .\bin\ + v4.6.2 + + + 2.0 + + + + + + \ No newline at end of file diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs new file mode 100644 index 0000000000..b627755d76 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Properties; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class Program + { + public static void Main(string[] args) + { + var options = (uint[])Enum.GetValues(typeof(BenchmarkType)); + BenchmarkType type; + if (args.Length != 1 || !Enum.TryParse(args[0], out type)) + { + Console.WriteLine($"Please add benchmark to run as parameter:"); + for (var i = 0; i < options.Length; i++) + { + Console.WriteLine($" {((BenchmarkType)options[i]).ToString()}"); + } + + return; + } + + RunSelectedBenchmarks(type); + } + + private static void RunSelectedBenchmarks(BenchmarkType type) + { + if (type.HasFlag(BenchmarkType.RequestParsing)) + { + BenchmarkRunner.Run(); + } + } + } + + [Flags] + public enum BenchmarkType : uint + { + RequestParsing = 1, + // add new ones in powers of two - e.g. 2,4,8,16... + + All = uint.MaxValue + } +} diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md new file mode 100644 index 0000000000..b98f36ff5c --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md @@ -0,0 +1,11 @@ +Compile the solution in Release mode (so Kestrel is available in release) + +To run a specific benchmark add it as parameter +``` +dotnet run RequestParsing +``` +To run all use `All` as parameter +``` +dotnet run All +``` +Using no parameter will list all available benchmarks \ No newline at end of file diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs new file mode 100644 index 0000000000..5ae06776f5 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs @@ -0,0 +1,188 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Attributes; +using Microsoft.AspNetCore.Server.Kestrel.Internal; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure; +using Microsoft.AspNetCore.Testing; +using RequestLineStatus = Microsoft.AspNetCore.Server.Kestrel.Internal.Http.Frame.RequestLineStatus; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + [Config(typeof(CoreConfig))] + public class RequestParsing + { + private const int InnerLoopCount = 512; + private const int Pipelining = 16; + + private const string plaintextRequest = "GET /plaintext HTTP/1.1\r\nHost: www.example.com\r\n\r\n"; + + private const string liveaspnetRequest = "GET https://live.asp.net/ HTTP/1.1\r\n" + + "Host: live.asp.net\r\n" + + "Connection: keep-alive\r\n" + + "Upgrade-Insecure-Requests: 1\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + + "DNT: 1\r\n" + + "Accept-Encoding: gzip, deflate, sdch, br\r\n" + + "Accept-Language: en-US,en;q=0.8\r\n" + + "Cookie: __unam=7a67379-1s65dc575c4-6d778abe-1; omniID=9519gfde_3347_4762_8762_df51458c8ec2\r\n\r\n"; + + private const string unicodeRequest = + "GET http://stackoverflow.com/questions/40148683/why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric HTTP/1.1\r\n" + + "Accept: text/html, application/xhtml+xml, image/jxr, */*\r\n" + + "Accept-Language: en-US,en-GB;q=0.7,en;q=0.3\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.14965\r\n" + + "Accept-Encoding: gzip, deflate\r\n" + + "Host: stackoverflow.com\r\n" + + "Connection: Keep-Alive\r\n" + + "Cache-Control: max-age=0\r\n" + + "Upgrade-Insecure-Requests: 1\r\n" + + "DNT: 1\r\n" + + "Referer: http://stackoverflow.com/?tab=month\r\n" + + "Pragma: no-cache\r\n" + + "Cookie: prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric\r\n\r\n"; + + private static readonly byte[] _plaintextPipelinedRequests = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(plaintextRequest, Pipelining))); + private static readonly byte[] _plaintextRequest = Encoding.ASCII.GetBytes(plaintextRequest); + + private static readonly byte[] _liveaspnentPipelinedRequests = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(liveaspnetRequest, Pipelining))); + private static readonly byte[] _liveaspnentRequest = Encoding.ASCII.GetBytes(liveaspnetRequest); + + private static readonly byte[] _unicodePipelinedRequests = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(unicodeRequest, Pipelining))); + private static readonly byte[] _unicodeRequest = Encoding.ASCII.GetBytes(unicodeRequest); + + private KestrelTrace Trace; + private LoggingThreadPool ThreadPool; + private MemoryPool MemoryPool; + private SocketInput SocketInput; + private Frame Frame; + + [Benchmark(Baseline = true, OperationsPerInvoke = InnerLoopCount)] + public void ParsePlaintext() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_plaintextRequest); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount * Pipelining)] + public void ParsePipelinedPlaintext() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_plaintextPipelinedRequests); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount)] + public void ParseLiveAspNet() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_liveaspnentRequest); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount * Pipelining)] + public void ParsePipelinedLiveAspNet() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_liveaspnentPipelinedRequests); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount)] + public void ParseUnicode() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_unicodeRequest); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount * Pipelining)] + public void ParseUnicodePipelined() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_unicodePipelinedRequests); + + ParseData(); + } + } + + private void InsertData(byte[] dataBytes) + { + SocketInput.IncomingData(dataBytes, 0, dataBytes.Length); + } + + private void ParseData() + { + while (SocketInput.GetAwaiter().IsCompleted) + { + Frame.Reset(); + + if (Frame.TakeStartLine(SocketInput) != RequestLineStatus.Done) + { + ThrowInvalidStartLine(); + } + + Frame.InitializeHeaders(); + + if (!Frame.TakeMessageHeaders(SocketInput, (FrameRequestHeaders) Frame.RequestHeaders)) + { + ThrowInvalidMessageHeaders(); + } + } + } + + private void ThrowInvalidStartLine() + { + throw new InvalidOperationException("Invalid StartLine"); + } + + private void ThrowInvalidMessageHeaders() + { + throw new InvalidOperationException("Invalid MessageHeaders"); + } + + [Setup] + public void Setup() + { + Trace = new KestrelTrace(new TestKestrelTrace()); + ThreadPool = new LoggingThreadPool(Trace); + MemoryPool = new MemoryPool(); + SocketInput = new SocketInput(MemoryPool, ThreadPool); + + var connectionContext = new MockConnection(new KestrelServerOptions()); + connectionContext.SocketInput = SocketInput; + + Frame = new Frame(application: null, context: connectionContext); + } + + [Cleanup] + public void Cleanup() + { + SocketInput.IncomingFin(); + SocketInput.Dispose(); + MemoryPool.Dispose(); + } + } +} diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs new file mode 100644 index 0000000000..12e9969ed2 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class RpsColumn : IColumn + { + private static int NanosPerSecond = 1000 * 1000 * 1000; + + public string GetValue(Summary summary, Benchmark benchmark) + { + var totalNanos = summary.Reports.First(r => r.Benchmark == benchmark).ResultStatistics.Mean; + // Make sure we don't divide by zero!! + return Math.Abs(totalNanos) > 0.0 ? (NanosPerSecond / totalNanos).ToString("N2") : "N/A"; + } + + public bool IsDefault(Summary summary, Benchmark benchmark) => false; + public bool IsAvailable(Summary summary) => true; + public string Id => "RPS-Column"; + public string ColumnName => "RPS"; + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Custom; + public int PriorityInCategory => 1; + } +} diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs new file mode 100644 index 0000000000..efdf192205 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Validators; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class CoreConfig : ManualConfig + { + public CoreConfig() + { + Add(JitOptimizationsValidator.FailOnError); + Add(new RpsColumn()); + + Add(Job.Default. + With(BenchmarkDotNet.Environments.Runtime.Core). + WithRemoveOutliers(false). + With(new GcMode() { Server = true }). + With(RunStrategy.Throughput). + WithLaunchCount(3). + WithWarmupCount(5). + WithTargetCount(10)); + } + } +} diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json new file mode 100644 index 0000000000..33f0f54e92 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json @@ -0,0 +1,3 @@ +{ + "projects": [ "..\\" ] +} diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json new file mode 100644 index 0000000000..a74be5fefb --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json @@ -0,0 +1,42 @@ +{ + "version": "1.0.0-*", + "dependencies": { + "BenchmarkDotNet": "0.10.0", + "Microsoft.AspNetCore.Server.Kestrel": "1.2.0-*" + }, + "frameworks": { + "netcoreapp1.0": { + "dependencies": { + "Microsoft.NETCore.App": { + "version": "1.0.1-*", + "type": "platform" + } + } + } + }, + "buildOptions": { + "emitEntryPoint": true, + "compile": { + "include": [ + "../shared/SocketInputExtensions.cs", + "../shared/TestKestrelTrace.cs", + "../shared/TestApplicationErrorLogger.cs", + "../shared/MockConnection.cs" + ] + }, + "keyFile": "../../tools/Key.snk", + "copyToOutput": { + "include": "TestResources/testCert.pfx" + } + }, + "runtimeOptions": { + "configProperties": { + "System.GC.Server": true + } + }, + "publishOptions": { + "include": [ + "TestResources/testCert.pfx" + ] + } +} \ No newline at end of file diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs index 87990788c6..220ae3d80b 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs @@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem.GetIterator(); var end = GetIterator(begin, byteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(ref end); Assert.Equal(s.Length, byteRange.Length); @@ -58,7 +58,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem.GetIterator(); var end = GetIterator(begin, byteRange.Length); - Assert.Throws(() => begin.GetAsciiString(end)); + Assert.Throws(() => begin.GetAsciiString(ref end)); pool.Return(mem); } @@ -94,7 +94,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem0.GetIterator(); var end = GetIterator(begin, expectedByteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(ref end); Assert.Equal(s.Length, expectedByteRange.Length); @@ -135,7 +135,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem0.GetIterator(); var end = GetIterator(begin, expectedByteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(ref end); Assert.Equal(expectedByteRange.Length, s.Length); diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs index a9c5652e4e..4cde6c079a 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs @@ -19,35 +19,31 @@ namespace Microsoft.AspNetCore.Server.KestrelTests block.Array[block.End++] = ch; } - var vectorMaxValues = new Vector(byte.MaxValue); - var iterator = block.GetIterator(); foreach (var ch in Enumerable.Range(0, 256).Select(x => (byte)x)) { - var vectorCh = new Vector(ch); - var hit = iterator; - hit.Seek(ref vectorCh); + hit.Seek(ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh); + hit.Seek(byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues); + hit.Seek(byte.MaxValue, ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); } @@ -77,35 +73,31 @@ namespace Microsoft.AspNetCore.Server.KestrelTests block3.Array[block3.End++] = ch; } - var vectorMaxValues = new Vector(byte.MaxValue); - var iterator = block1.GetIterator(); foreach (var ch in Enumerable.Range(0, 256).Select(x => (byte)x)) { - var vectorCh = new Vector(ch); - var hit = iterator; - hit.Seek(ref vectorCh); + hit.Seek(ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh); + hit.Seek(byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues); + hit.Seek(byte.MaxValue, ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorMaxValues, ref vectorCh); + hit.Seek(byte.MaxValue, byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); } diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 454f66291f..1db21257a3 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests for (int i = 0; i < Vector.Count; i++) { Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByte(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(vector)); bytes[i] = 0; } @@ -40,27 +40,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests { bytes[i] = 1; Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByte(ref vector)); - bytes[i] = 0; - } - } - - [Fact] - public void TestFindFirstEqualByteSlow() - { - var bytes = Enumerable.Repeat(0xff, Vector.Count).ToArray(); - for (int i = 0; i < Vector.Count; i++) - { - Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByteSlow(ref vector)); - bytes[i] = 0; - } - - for (int i = 0; i < Vector.Count; i++) - { - bytes[i] = 1; - Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByteSlow(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(vector)); bytes[i] = 0; } } @@ -98,21 +78,15 @@ namespace Microsoft.AspNetCore.Server.KestrelTests int found = -1; if (searchFor.Length == 1) { - var search0 = new Vector((byte) searchFor[0]); - found = begin.Seek(ref search0); + found = begin.Seek((byte)searchFor[0]); } else if (searchFor.Length == 2) { - var search0 = new Vector((byte) searchFor[0]); - var search1 = new Vector((byte) searchFor[1]); - found = begin.Seek(ref search0, ref search1); + found = begin.Seek((byte)searchFor[0], (byte)searchFor[1]); } else if (searchFor.Length == 3) { - var search0 = new Vector((byte) searchFor[0]); - var search1 = new Vector((byte) searchFor[1]); - var search2 = new Vector((byte) searchFor[2]); - found = begin.Seek(ref search0, ref search1, ref search2); + found = begin.Seek((byte)searchFor[0], (byte)searchFor[1], (byte)searchFor[2]); } else { @@ -180,7 +154,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests } // Can't put anything by the end - Assert.False(head.Put(0xFF)); + Assert.ThrowsAny(() => head.Put(0xFF)); for (var i = 0; i < 4; ++i) { @@ -544,20 +518,25 @@ namespace Microsoft.AspNetCore.Server.KestrelTests public void SkipThrowsWhenSkippingMoreBytesThanAvailableInMultipleBlocks() { // Arrange - var block = _pool.Lease(); - block.End += 3; + var firstBlock = _pool.Lease(); + firstBlock.End += 3; - var nextBlock = _pool.Lease(); - nextBlock.End += 2; - block.Next = nextBlock; + var middleBlock = _pool.Lease(); + middleBlock.End += 1; + firstBlock.Next = middleBlock; - var scan = block.GetIterator(); + var finalBlock = _pool.Lease(); + finalBlock.End += 2; + middleBlock.Next = finalBlock; + + var scan = firstBlock.GetIterator(); // Act/Assert Assert.ThrowsAny(() => scan.Skip(8)); - _pool.Return(block); - _pool.Return(nextBlock); + _pool.Return(firstBlock); + _pool.Return(middleBlock); + _pool.Return(finalBlock); } [Theory] @@ -759,7 +738,6 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); block = _pool.Lease(); var chars = input.ToString().ToCharArray().Select(c => (byte)c).ToArray(); @@ -769,7 +747,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests // Act int bytesScanned; - var returnValue = scan.Seek(ref seekVector, out bytesScanned, limit); + var returnValue = scan.Seek((byte)seek, out bytesScanned, limit); // Assert Assert.Equal(expectedBytesScanned, bytesScanned); @@ -799,8 +777,6 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); - var input1 = input.Substring(0, input.Length / 2); block1 = _pool.Lease(); var chars1 = input1.ToCharArray().Select(c => (byte)c).ToArray(); @@ -821,7 +797,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests // Act int bytesScanned; - var returnValue = scan.Seek(ref seekVector, out bytesScanned, limit); + var returnValue = scan.Seek((byte)seek, out bytesScanned, limit); // Assert Assert.Equal(expectedBytesScanned, bytesScanned); @@ -855,9 +831,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); - var limitAtVector = new Vector((byte)limitAt); - var afterSeekVector = new Vector((byte)'B'); + var afterSeek = (byte)'B'; block = _pool.Lease(); var chars = input.ToCharArray().Select(c => (byte)c).ToArray(); @@ -872,13 +846,13 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = scan1; // Act - var endReturnValue = end.Seek(ref limitAtVector); - var returnValue1 = scan1.Seek(ref seekVector, ref end); - var returnValue2_1 = scan2_1.Seek(ref seekVector, ref afterSeekVector, ref end); - var returnValue2_2 = scan2_2.Seek(ref afterSeekVector, ref seekVector, ref end); - var returnValue3_1 = scan3_1.Seek(ref seekVector, ref afterSeekVector, ref afterSeekVector, ref end); - var returnValue3_2 = scan3_2.Seek(ref afterSeekVector, ref seekVector, ref afterSeekVector, ref end); - var returnValue3_3 = scan3_3.Seek(ref afterSeekVector, ref afterSeekVector, ref seekVector, ref end); + var endReturnValue = end.Seek((byte)limitAt); + var returnValue1 = scan1.Seek((byte)seek, ref end); + var returnValue2_1 = scan2_1.Seek((byte)seek, afterSeek, ref end); + var returnValue2_2 = scan2_2.Seek(afterSeek, (byte)seek, ref end); + var returnValue3_1 = scan3_1.Seek((byte)seek, afterSeek, afterSeek, ref end); + var returnValue3_2 = scan3_2.Seek(afterSeek, (byte)seek, afterSeek, ref end); + var returnValue3_3 = scan3_3.Seek(afterSeek, afterSeek, (byte)seek, ref end); // Assert Assert.Equal(input.Contains(limitAt) ? limitAt : -1, endReturnValue); @@ -922,9 +896,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); - var limitAtVector = new Vector((byte)limitAt); - var afterSeekVector = new Vector((byte)'B'); + var afterSeek = (byte)'B'; var input1 = input.Substring(0, input.Length / 2); block1 = _pool.Lease(); @@ -951,13 +923,13 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = scan1; // Act - var endReturnValue = end.Seek(ref limitAtVector); - var returnValue1 = scan1.Seek(ref seekVector, ref end); - var returnValue2_1 = scan2_1.Seek(ref seekVector, ref afterSeekVector, ref end); - var returnValue2_2 = scan2_2.Seek(ref afterSeekVector, ref seekVector, ref end); - var returnValue3_1 = scan3_1.Seek(ref seekVector, ref afterSeekVector, ref afterSeekVector, ref end); - var returnValue3_2 = scan3_2.Seek(ref afterSeekVector, ref seekVector, ref afterSeekVector, ref end); - var returnValue3_3 = scan3_3.Seek(ref afterSeekVector, ref afterSeekVector, ref seekVector, ref end); + var endReturnValue = end.Seek((byte)limitAt); + var returnValue1 = scan1.Seek((byte)seek, ref end); + var returnValue2_1 = scan2_1.Seek((byte)seek, afterSeek, ref end); + var returnValue2_2 = scan2_2.Seek(afterSeek, (byte)seek, ref end); + var returnValue3_1 = scan3_1.Seek((byte)seek, afterSeek, afterSeek, ref end); + var returnValue3_2 = scan3_2.Seek(afterSeek, (byte)seek, afterSeek, ref end); + var returnValue3_3 = scan3_3.Seek(afterSeek, afterSeek, (byte)seek, ref end); // Assert Assert.Equal(input.Contains(limitAt) ? limitAt : -1, endReturnValue); @@ -999,6 +971,216 @@ namespace Microsoft.AspNetCore.Server.KestrelTests } } + [Fact] + public void EmptyIteratorBehaviourIsValid() + { + const byte byteCr = (byte) '\n'; + ulong longValue; + var end = default(MemoryPoolIterator); + + Assert.False(default(MemoryPoolIterator).TryPeekLong(out longValue)); + Assert.Null(default(MemoryPoolIterator).GetAsciiString(ref end)); + Assert.Null(default(MemoryPoolIterator).GetUtf8String(ref end)); + // Assert.Equal doesn't work for default(ArraySegments) + Assert.True(default(MemoryPoolIterator).GetArraySegment(end).Equals(default(ArraySegment))); + Assert.True(default(MemoryPoolIterator).IsDefault); + Assert.True(default(MemoryPoolIterator).IsEnd); + Assert.Equal(default(MemoryPoolIterator).Take(), -1); + Assert.Equal(default(MemoryPoolIterator).Peek(), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr, ref end), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr, byteCr), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr, byteCr, byteCr), -1); + + default(MemoryPoolIterator).CopyFrom(default(ArraySegment)); + default(MemoryPoolIterator).CopyFromAscii(""); + Assert.ThrowsAny(() => default(MemoryPoolIterator).Put(byteCr)); + Assert.ThrowsAny(() => default(MemoryPoolIterator).GetLength(end)); + Assert.ThrowsAny(() => default(MemoryPoolIterator).Skip(1)); + } + + [Fact] + public void TestGetArraySegment() + { + MemoryPoolBlock block0 = null; + MemoryPoolBlock block1 = null; + + var byteRange = Enumerable.Range(1, 127).Select(x => (byte)x).ToArray(); + try + { + // Arrange + block0 = _pool.Lease(); + block1 = _pool.Lease(); + + block0.GetIterator().CopyFrom(byteRange); + block1.GetIterator().CopyFrom(byteRange); + + block0.Next = block1; + + var begin = block0.GetIterator(); + var end0 = begin; + var end1 = begin; + + end0.Skip(byteRange.Length); + end1.Skip(byteRange.Length * 2); + + // Act + var as0 = begin.GetArraySegment(end0); + var as1 = begin.GetArraySegment(end1); + + // Assert + Assert.Equal(as0.Count, byteRange.Length); + Assert.Equal(as1.Count, byteRange.Length * 2); + + for (var i = 1; i < byteRange.Length; i++) + { + var asb0 = as0.Array[i + as0.Offset]; + var asb1 = as1.Array[i + as1.Offset]; + var b = byteRange[i]; + + Assert.Equal(asb0, b); + Assert.Equal(asb1, b); + } + + for (var i = 1 + byteRange.Length; i < byteRange.Length * 2; i++) + { + var asb1 = as1.Array[i + as1.Offset]; + var b = byteRange[i - byteRange.Length]; + + Assert.Equal(asb1, b); + } + + } + finally + { + if (block0 != null) _pool.Return(block0); + if (block1 != null) _pool.Return(block1); + } + } + + [Fact] + public void TestTake() + { + MemoryPoolBlock block0 = null; + MemoryPoolBlock block1 = null; + MemoryPoolBlock block2 = null; + MemoryPoolBlock emptyBlock0 = null; + MemoryPoolBlock emptyBlock1 = null; + + var byteRange = Enumerable.Range(1, 127).Select(x => (byte)x).ToArray(); + try + { + // Arrange + block0 = _pool.Lease(); + block1 = _pool.Lease(); + block2 = _pool.Lease(); + emptyBlock0 = _pool.Lease(); + emptyBlock1 = _pool.Lease(); + + block0.GetIterator().CopyFrom(byteRange); + block1.GetIterator().CopyFrom(byteRange); + block2.GetIterator().CopyFrom(byteRange); + + var begin = block0.GetIterator(); + + // Single block + for (var i = 0; i < byteRange.Length; i++) + { + var t = begin.Take(); + var b = byteRange[i]; + + Assert.Equal(t, b); + } + + Assert.Equal(begin.Take(), -1); + + // Dual block + block0.Next = block1; + begin = block0.GetIterator(); + + for (var block = 0; block < 2; block++) + { + for (var i = 0; i < byteRange.Length; i++) + { + var t = begin.Take(); + var b = byteRange[i]; + + Assert.Equal(t, b); + } + } + + Assert.Equal(begin.Take(), -1); + + // Multi block + block1.Next = emptyBlock0; + emptyBlock0.Next = emptyBlock1; + emptyBlock1.Next = block2; + begin = block0.GetIterator(); + + for (var block = 0; block < 3; block++) + { + for (var i = 0; i < byteRange.Length; i++) + { + var t = begin.Take(); + var b = byteRange[i]; + + Assert.Equal(t, b); + } + } + + Assert.Equal(begin.Take(), -1); + } + finally + { + if (block0 != null) _pool.Return(block0); + if (block1 != null) _pool.Return(block1); + if (block2 != null) _pool.Return(block2); + if (emptyBlock0 != null) _pool.Return(emptyBlock0); + if (emptyBlock1 != null) _pool.Return(emptyBlock1); + } + } + + [Fact] + public void TestTakeEmptyBlocks() + { + MemoryPoolBlock emptyBlock0 = null; + MemoryPoolBlock emptyBlock1 = null; + MemoryPoolBlock emptyBlock2 = null; + try + { + // Arrange + emptyBlock0 = _pool.Lease(); + emptyBlock1 = _pool.Lease(); + emptyBlock2 = _pool.Lease(); + + var beginEmpty = emptyBlock0.GetIterator(); + + // Assert + + // No blocks + Assert.Equal(default(MemoryPoolIterator).Take(), -1); + + // Single empty block + Assert.Equal(beginEmpty.Take(), -1); + + // Dual empty block + emptyBlock0.Next = emptyBlock1; + beginEmpty = emptyBlock0.GetIterator(); + Assert.Equal(beginEmpty.Take(), -1); + + // Multi empty block + emptyBlock1.Next = emptyBlock2; + beginEmpty = emptyBlock0.GetIterator(); + Assert.Equal(beginEmpty.Take(), -1); + } + finally + { + if (emptyBlock0 != null) _pool.Return(emptyBlock0); + if (emptyBlock1 != null) _pool.Return(emptyBlock1); + if (emptyBlock2 != null) _pool.Return(emptyBlock2); + } + } + [Theory] [InlineData("a", "a", 1)] [InlineData("ab", "a...", 1)] diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs index a5ed8d7c9f..414bb208e3 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs @@ -10,11 +10,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Server.Kestrel; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; -using Microsoft.AspNetCore.Server.KestrelTests.TestHelpers; using Microsoft.Extensions.Internal; using Moq; using Xunit; using Xunit.Sdk; +using Microsoft.AspNetCore.Testing; namespace Microsoft.AspNetCore.Server.KestrelTests { diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs index c33c8e5168..0acfc27fe8 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs @@ -167,7 +167,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, rawLength); var end2 = UrlPathDecoder.Unescape(begin, end); - var result = begin.GetUtf8String(end2); + var result = begin.GetUtf8String(ref end2); Assert.Equal(expectLength, result.Length); Assert.Equal(expect, result); @@ -201,7 +201,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, raw.Length); var result = UrlPathDecoder.Unescape(begin, end); - Assert.Equal(expect, begin.GetUtf8String(result)); + Assert.Equal(expect, begin.GetUtf8String(ref result)); } private void PositiveAssert(MemoryPoolBlock mem, string raw) @@ -210,7 +210,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, raw.Length); var result = UrlPathDecoder.Unescape(begin, end); - Assert.NotEqual(raw.Length, begin.GetUtf8String(result).Length); + Assert.NotEqual(raw.Length, begin.GetUtf8String(ref result).Length); } private void NegativeAssert(MemoryPoolBlock mem, string raw) @@ -219,7 +219,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, raw.Length); var resultEnd = UrlPathDecoder.Unescape(begin, end); - var result = begin.GetUtf8String(resultEnd); + var result = begin.GetUtf8String(ref resultEnd); Assert.Equal(raw, result); } } diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/MockConnection.cs b/test/shared/MockConnection.cs similarity index 87% rename from test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/MockConnection.cs rename to test/shared/MockConnection.cs index ee781a3935..e1025c624c 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/MockConnection.cs +++ b/test/shared/MockConnection.cs @@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel; using Microsoft.AspNetCore.Server.Kestrel.Internal; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; -namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers +namespace Microsoft.AspNetCore.Testing { public class MockConnection : Connection, IDisposable { @@ -18,7 +18,10 @@ namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers { ConnectionControl = this; RequestAbortedSource = new CancellationTokenSource(); - ListenerContext = new ListenerContext(new ServiceContext { ServerOptions = options }); + ListenerContext = new ListenerContext(new ServiceContext {ServerOptions = options}) + { + ServerAddress = ServerAddress.FromUrl("http://localhost:5000") + }; } public override void Abort(Exception error = null) diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/SocketInputExtensions.cs b/test/shared/SocketInputExtensions.cs similarity index 94% rename from test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/SocketInputExtensions.cs rename to test/shared/SocketInputExtensions.cs index 998e0552c5..d6dbbb7e88 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/SocketInputExtensions.cs +++ b/test/shared/SocketInputExtensions.cs @@ -4,7 +4,7 @@ using System; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; -namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers +namespace Microsoft.AspNetCore.Testing { public static class SocketInputExtensions {