From 4f7977d440fff5ea4ae60983fb2d20d008252c54 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 3 Oct 2016 07:58:24 +0100 Subject: [PATCH 01/20] Use PowerOfTwoToHighByte for FindFirstEqualByte --- .../Infrastructure/MemoryPoolIterator.cs | 81 +++++++------------ .../MemoryPoolIteratorTests.cs | 24 +----- 2 files changed, 31 insertions(+), 74 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 5f7e908c36..0ab1bf01ad 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -4,12 +4,14 @@ using System; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; using System.Threading; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { public struct MemoryPoolIterator { + private static readonly ulong _powerOfTwoToHighByte = PowerOfTwoToHighByte(); private static readonly int _vectorSpan = Vector.Count; private MemoryPoolBlock _block; @@ -307,7 +309,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(ref byte0Equals); var vectorBytesScanned = firstEqualByteIndex + 1; if (bytesScanned + vectorBytesScanned > limit) @@ -413,7 +415,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(ref byte0Equals); if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) { @@ -512,11 +514,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (!byte0Equals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); + byte0Index = LocateFirstFoundByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector.Zero)) { - byte1Index = FindFirstEqualByte(ref byte1Equals); + byte1Index = LocateFirstFoundByte(ref byte1Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) @@ -656,15 +658,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (!byte0Equals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); + byte0Index = LocateFirstFoundByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector.Zero)) { - byte1Index = FindFirstEqualByte(ref byte1Equals); + byte1Index = LocateFirstFoundByte(ref byte1Equals); } if (!byte2Equals.Equals(Vector.Zero)) { - byte2Index = FindFirstEqualByte(ref byte2Equals); + byte2Index = LocateFirstFoundByte(ref byte2Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) @@ -761,60 +763,30 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } /// - /// Find first byte + /// Locate the first of the found bytes /// /// /// The first index of the result vector - /// byteEquals = 0 - internal static int FindFirstEqualByte(ref Vector byteEquals) + // Force inlining (64 IL bytes, 91 bytes asm) Issue: https://github.com/dotnet/coreclr/issues/7386 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LocateFirstFoundByte(ref Vector byteEquals) { - if (!BitConverter.IsLittleEndian) return FindFirstEqualByteSlow(ref byteEquals); - - // Quasi-tree search var vector64 = Vector.AsVectorInt64(byteEquals); - for (var i = 0; i < Vector.Count; i++) + var i = 0; + long longValue = 0; + for (; i < Vector.Count; i++) { - var longValue = vector64[i]; + longValue = vector64[i]; if (longValue == 0) continue; - - return (i << 3) + - ((longValue & 0x00000000ffffffff) > 0 - ? (longValue & 0x000000000000ffff) > 0 - ? (longValue & 0x00000000000000ff) > 0 ? 0 : 1 - : (longValue & 0x0000000000ff0000) > 0 ? 2 : 3 - : (longValue & 0x0000ffff00000000) > 0 - ? (longValue & 0x000000ff00000000) > 0 ? 4 : 5 - : (longValue & 0x00ff000000000000) > 0 ? 6 : 7); + break; } - throw new InvalidOperationException(); - } - // Internal for testing - internal static int FindFirstEqualByteSlow(ref Vector byteEquals) - { - // Quasi-tree search - var vector64 = Vector.AsVectorInt64(byteEquals); - for (var i = 0; i < Vector.Count; i++) - { - var longValue = vector64[i]; - if (longValue == 0) continue; - - var shift = i << 1; - var offset = shift << 2; - var vector32 = Vector.AsVectorInt32(byteEquals); - if (vector32[shift] != 0) - { - if (byteEquals[offset] != 0) return offset; - if (byteEquals[offset + 1] != 0) return offset + 1; - if (byteEquals[offset + 2] != 0) return offset + 2; - return offset + 3; - } - if (byteEquals[offset + 4] != 0) return offset + 4; - if (byteEquals[offset + 5] != 0) return offset + 5; - if (byteEquals[offset + 6] != 0) return offset + 6; - return offset + 7; - } - throw new InvalidOperationException(); + // Flag least significant power of two bit + var powerOfTwoFlag = (ulong)(longValue & -longValue); + // Shift all powers of two into the high byte and extract + var foundByteIndex = (int)((powerOfTwoFlag * _powerOfTwoToHighByte) >> 61); + // Single LEA instruction with jitted const (using function result) + return i * 8 + foundByteIndex; } /// @@ -1059,5 +1031,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; _index = blockIndex; } + + private static ulong PowerOfTwoToHighByte() + { + return BitConverter.IsLittleEndian ? 0x20406080A0C0E0ul : 0xE0C0A080604020ul; + } } } diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 454f66291f..8ac556ed55 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests for (int i = 0; i < Vector.Count; i++) { Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByte(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(ref vector)); bytes[i] = 0; } @@ -40,27 +40,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests { bytes[i] = 1; Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByte(ref vector)); - bytes[i] = 0; - } - } - - [Fact] - public void TestFindFirstEqualByteSlow() - { - var bytes = Enumerable.Repeat(0xff, Vector.Count).ToArray(); - for (int i = 0; i < Vector.Count; i++) - { - Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByteSlow(ref vector)); - bytes[i] = 0; - } - - for (int i = 0; i < Vector.Count; i++) - { - bytes[i] = 1; - Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByteSlow(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(ref vector)); bytes[i] = 0; } } From 9eb01d1c576320aae04ad46c3b3831774dd2fe91 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 5 Oct 2016 16:23:58 +0100 Subject: [PATCH 02/20] Reduce Vector register pressure by not overlapping --- .../Internal/Infrastructure/MemoryPoolIterator.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 0ab1bf01ad..34f8b44608 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -509,13 +509,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (following >= _vectorSpan) { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); + var byte0Equals = Vector.Equals(data, byte0Vector); if (!byte0Equals.Equals(Vector.Zero)) { byte0Index = LocateFirstFoundByte(ref byte0Equals); } + + var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte1Equals.Equals(Vector.Zero)) { byte1Index = LocateFirstFoundByte(ref byte1Equals); @@ -652,18 +653,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (following >= _vectorSpan) { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); - var byte2Equals = Vector.Equals(data, byte2Vector); + var byte0Equals = Vector.Equals(data, byte0Vector); if (!byte0Equals.Equals(Vector.Zero)) { byte0Index = LocateFirstFoundByte(ref byte0Equals); } + + var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte1Equals.Equals(Vector.Zero)) { byte1Index = LocateFirstFoundByte(ref byte1Equals); } + + var byte2Equals = Vector.Equals(data, byte2Vector); if (!byte2Equals.Equals(Vector.Zero)) { byte2Index = LocateFirstFoundByte(ref byte2Equals); From 972d978d1111054f5caedc1f091059309ecf51cd Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 15 Oct 2016 18:53:57 +0100 Subject: [PATCH 03/20] Swap for vector loop unroll detection --- .../Internal/Infrastructure/MemoryPoolIterator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 34f8b44608..daf2d20f5c 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -775,8 +775,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure internal static int LocateFirstFoundByte(ref Vector byteEquals) { var vector64 = Vector.AsVectorInt64(byteEquals); - var i = 0; long longValue = 0; + var i = 0; for (; i < Vector.Count; i++) { longValue = vector64[i]; From 97d4406614799746c8608d2633a3a54ea9bcb8d9 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 15 Oct 2016 20:05:50 +0100 Subject: [PATCH 04/20] Use Vector.ctor workaround --- .../Internal/Http/Frame.cs | 64 +++++++++---------- .../Internal/Http/MessageBody.cs | 8 +-- .../Infrastructure/MemoryPoolIterator.cs | 49 ++++++++------ .../MemoryPoolBlockTests.cs | 32 ++++------ .../MemoryPoolIteratorTests.cs | 55 ++++++---------- 5 files changed, 97 insertions(+), 111 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs index 307ce7bc49..399976939c 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Net; -using System.Numerics; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -24,6 +23,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { public abstract partial class Frame : IFrameControl { + // byte consts don't have a data type annotation so we pre-cast them + private const byte ByteCR = (byte)'\r'; + private const byte ByteLF = (byte)'\n'; + private const byte ByteColon = (byte)':'; + private const byte ByteSpace = (byte)' '; + private const byte ByteTab = (byte)'\t'; + private const byte ByteQuestionMark = (byte)'?'; + private const byte BytePercentage = (byte)'%'; + private static readonly ArraySegment _endChunkedResponseBytes = CreateAsciiByteArraySegment("0\r\n\r\n"); private static readonly ArraySegment _continueBytes = CreateAsciiByteArraySegment("HTTP/1.1 100 Continue\r\n\r\n"); @@ -35,14 +43,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http private static readonly byte[] _bytesEndHeaders = Encoding.ASCII.GetBytes("\r\n\r\n"); private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: Kestrel"); - private static Vector _vectorCRs = new Vector((byte)'\r'); - private static Vector _vectorLFs = new Vector((byte)'\n'); - private static Vector _vectorColons = new Vector((byte)':'); - private static Vector _vectorSpaces = new Vector((byte)' '); - private static Vector _vectorTabs = new Vector((byte)'\t'); - private static Vector _vectorQuestionMarks = new Vector((byte)'?'); - private static Vector _vectorPercentages = new Vector((byte)'%'); - private readonly object _onStartingSync = new Object(); private readonly object _onCompletedSync = new Object(); @@ -952,7 +952,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http _requestProcessingStatus = RequestProcessingStatus.RequestStarted; int bytesScanned; - if (end.Seek(ref _vectorLFs, out bytesScanned, ServerOptions.Limits.MaxRequestLineSize) == -1) + if (end.Seek(ByteLF, out bytesScanned, ServerOptions.Limits.MaxRequestLineSize) == -1) { if (bytesScanned >= ServerOptions.Limits.MaxRequestLineSize) { @@ -969,7 +969,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var begin = scan; if (!begin.GetKnownMethod(out method)) { - if (scan.Seek(ref _vectorSpaces, ref end) == -1) + if (scan.Seek(ByteSpace, ref end) == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1002,16 +1002,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http scan.Take(); begin = scan; var needDecode = false; - var chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref _vectorPercentages, ref end); + var chFound = scan.Seek(ByteSpace, ByteQuestionMark, BytePercentage, ref end); if (chFound == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); } - else if (chFound == '%') + else if (chFound == BytePercentage) { needDecode = true; - chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref end); + chFound = scan.Seek(ByteSpace, ByteQuestionMark, ref end); if (chFound == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, @@ -1023,10 +1023,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var pathEnd = scan; var queryString = ""; - if (chFound == '?') + if (chFound == ByteQuestionMark) { begin = scan; - if (scan.Seek(ref _vectorSpaces, ref end) == -1) + if (scan.Seek(ByteSpace, ref end) == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1036,7 +1036,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var queryEnd = scan; - if (pathBegin.Peek() == ' ') + if (pathBegin.Peek() == ByteSpace) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1044,7 +1044,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http scan.Take(); begin = scan; - if (scan.Seek(ref _vectorCRs, ref end) == -1) + if (scan.Seek(ByteCR, ref end) == -1) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1067,7 +1067,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } scan.Take(); // consume CR - if (scan.Take() != '\n') + if (scan.Take() != ByteLF) { RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); @@ -1208,7 +1208,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { return false; } - else if (ch == '\r') + else if (ch == ByteCR) { // Check for final CRLF. end.Take(); @@ -1218,7 +1218,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { return false; } - else if (ch == '\n') + else if (ch == ByteLF) { ConnectionControl.CancelTimeout(); consumed = end; @@ -1228,7 +1228,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http // Headers don't end in CRLF line. RejectRequest(RequestRejectionReason.HeadersCorruptedInvalidHeaderSequence); } - else if (ch == ' ' || ch == '\t') + else if (ch == ByteSpace || ch == ByteTab) { RejectRequest(RequestRejectionReason.HeaderLineMustNotStartWithWhitespace); } @@ -1241,7 +1241,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } int bytesScanned; - if (end.Seek(ref _vectorLFs, out bytesScanned, _remainingRequestHeadersBytesAllowed) == -1) + if (end.Seek(ByteLF, out bytesScanned, _remainingRequestHeadersBytesAllowed) == -1) { if (bytesScanned >= _remainingRequestHeadersBytesAllowed) { @@ -1254,7 +1254,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } var beginName = scan; - if (scan.Seek(ref _vectorColons, ref end) == -1) + if (scan.Seek(ByteColon, ref end) == -1) { RejectRequest(RequestRejectionReason.NoColonCharacterFoundInHeaderLine); } @@ -1263,7 +1263,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http scan.Take(); var validateName = beginName; - if (validateName.Seek(ref _vectorSpaces, ref _vectorTabs, ref endName) != -1) + if (validateName.Seek(ByteSpace, ByteTab, ref endName) != -1) { RejectRequest(RequestRejectionReason.WhitespaceIsNotAllowedInHeaderName); } @@ -1271,14 +1271,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var beginValue = scan; ch = scan.Take(); - while (ch == ' ' || ch == '\t') + while (ch == ByteSpace || ch == ByteTab) { beginValue = scan; ch = scan.Take(); } scan = beginValue; - if (scan.Seek(ref _vectorCRs, ref end) == -1) + if (scan.Seek(ByteCR, ref end) == -1) { RejectRequest(RequestRejectionReason.MissingCRInHeaderLine); } @@ -1287,7 +1287,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http ch = scan.Take(); // expecting '\n' end = scan; - if (ch != '\n') + if (ch != ByteLF) { RejectRequest(RequestRejectionReason.HeaderValueMustNotContainCR); } @@ -1297,7 +1297,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { return false; } - else if (next == ' ' || next == '\t') + else if (next == ByteSpace || next == ByteTab) { // From https://tools.ietf.org/html/rfc7230#section-3.2.4: // @@ -1330,15 +1330,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var endValue = scan; do { - ws.Seek(ref _vectorSpaces, ref _vectorTabs, ref _vectorCRs); + ws.Seek(ByteSpace, ByteTab, ByteCR); endValue = ws; ch = ws.Take(); - while (ch == ' ' || ch == '\t') + while (ch == ByteSpace || ch == ByteTab) { ch = ws.Take(); } - } while (ch != '\r'); + } while (ch != ByteCR); var name = beginName.GetArraySegment(endName); var value = beginValue.GetAsciiString(endValue); diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs index 5fb8fa2c3a..77e9b9adcd 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using System.Numerics; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; @@ -390,9 +389,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http /// private class ForChunkedEncoding : MessageBody { - // This causes an InvalidProgramException if made static - // https://github.com/dotnet/corefx/issues/8825 - private Vector _vectorCRs = new Vector((byte)'\r'); + // byte consts don't have a data type annotation so we pre-cast it + private const byte ByteCR = (byte)'\r'; private readonly SocketInput _input; private readonly FrameRequestHeaders _requestHeaders; @@ -613,7 +611,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http // Just drain the data do { - if (scan.Seek(ref _vectorCRs) == -1) + if (scan.Seek(ByteCR) == -1) { // End marker not found yet consumed = scan; diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index daf2d20f5c..3bff457c17 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -236,14 +236,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public int Seek(ref Vector byte0Vector) + public int Seek(byte byte0) { int bytesScanned; - return Seek(ref byte0Vector, out bytesScanned); + return Seek(byte0, out bytesScanned); } public unsafe int Seek( - ref Vector byte0Vector, + byte byte0, out int bytesScanned, int limit = int.MaxValue) { @@ -259,7 +259,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; - var byte0 = byte0Vector[0]; + var byte0Vector = GetVector(byte0); while (true) { @@ -352,7 +352,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } public unsafe int Seek( - ref Vector byte0Vector, + byte byte0, ref MemoryPoolIterator limit) { if (IsDefault) @@ -365,7 +365,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; - var byte0 = byte0Vector[0]; + var byte0Vector = GetVector(byte0); while (true) { @@ -453,15 +453,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public int Seek(ref Vector byte0Vector, ref Vector byte1Vector) + public int Seek(byte byte0, byte byte1) { var limit = new MemoryPoolIterator(); - return Seek(ref byte0Vector, ref byte1Vector, ref limit); + return Seek(byte0, byte1, ref limit); } public unsafe int Seek( - ref Vector byte0Vector, - ref Vector byte1Vector, + byte byte0, + byte byte1, ref MemoryPoolIterator limit) { if (IsDefault) @@ -476,8 +476,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; - var byte0 = byte0Vector[0]; - var byte1 = byte1Vector[0]; + var byte0Vector = GetVector(byte0); + var byte1Vector = GetVector(byte1); while (true) { @@ -595,16 +595,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public int Seek(ref Vector byte0Vector, ref Vector byte1Vector, ref Vector byte2Vector) + public int Seek(byte byte0, byte byte1, byte byte2) { var limit = new MemoryPoolIterator(); - return Seek(ref byte0Vector, ref byte1Vector, ref byte2Vector, ref limit); + return Seek(byte0, byte1, byte2, ref limit); } public unsafe int Seek( - ref Vector byte0Vector, - ref Vector byte1Vector, - ref Vector byte2Vector, + byte byte0, + byte byte1, + byte byte2, ref MemoryPoolIterator limit) { if (IsDefault) @@ -620,9 +620,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; int byte2Index = int.MaxValue; - var byte0 = byte0Vector[0]; - var byte1 = byte1Vector[0]; - var byte2 = byte2Vector[0]; + var byte0Vector = GetVector(byte0); + var byte1Vector = GetVector(byte1); + var byte2Vector = GetVector(byte2); while (true) { @@ -1035,6 +1035,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _index = blockIndex; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector GetVector(byte vectorByte) + { + // Vector .ctor doesn't become an intrinsic due to detection issue + // However this does cause it to become an intrinsic (with additional multiply and reg->reg copy) + // https://github.com/dotnet/coreclr/issues/7459#issuecomment-253965670 + return Vector.AsVectorByte(new Vector(vectorByte * 0x0101010101010101ul)); + } + private static ulong PowerOfTwoToHighByte() { return BitConverter.IsLittleEndian ? 0x20406080A0C0E0ul : 0xE0C0A080604020ul; diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs index a9c5652e4e..4cde6c079a 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolBlockTests.cs @@ -19,35 +19,31 @@ namespace Microsoft.AspNetCore.Server.KestrelTests block.Array[block.End++] = ch; } - var vectorMaxValues = new Vector(byte.MaxValue); - var iterator = block.GetIterator(); foreach (var ch in Enumerable.Range(0, 256).Select(x => (byte)x)) { - var vectorCh = new Vector(ch); - var hit = iterator; - hit.Seek(ref vectorCh); + hit.Seek(ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh); + hit.Seek(byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues); + hit.Seek(byte.MaxValue, ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); } @@ -77,35 +73,31 @@ namespace Microsoft.AspNetCore.Server.KestrelTests block3.Array[block3.End++] = ch; } - var vectorMaxValues = new Vector(byte.MaxValue); - var iterator = block1.GetIterator(); foreach (var ch in Enumerable.Range(0, 256).Select(x => (byte)x)) { - var vectorCh = new Vector(ch); - var hit = iterator; - hit.Seek(ref vectorCh); + hit.Seek(ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh); + hit.Seek(byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); + hit.Seek(ch, byte.MaxValue, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues); + hit.Seek(byte.MaxValue, ch, byte.MaxValue); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(ref vectorMaxValues, ref vectorMaxValues, ref vectorCh); + hit.Seek(byte.MaxValue, byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); } diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 8ac556ed55..c0ff83603b 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -78,21 +78,15 @@ namespace Microsoft.AspNetCore.Server.KestrelTests int found = -1; if (searchFor.Length == 1) { - var search0 = new Vector((byte) searchFor[0]); - found = begin.Seek(ref search0); + found = begin.Seek((byte)searchFor[0]); } else if (searchFor.Length == 2) { - var search0 = new Vector((byte) searchFor[0]); - var search1 = new Vector((byte) searchFor[1]); - found = begin.Seek(ref search0, ref search1); + found = begin.Seek((byte)searchFor[0], (byte)searchFor[1]); } else if (searchFor.Length == 3) { - var search0 = new Vector((byte) searchFor[0]); - var search1 = new Vector((byte) searchFor[1]); - var search2 = new Vector((byte) searchFor[2]); - found = begin.Seek(ref search0, ref search1, ref search2); + found = begin.Seek((byte)searchFor[0], (byte)searchFor[1], (byte)searchFor[2]); } else { @@ -739,7 +733,6 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); block = _pool.Lease(); var chars = input.ToString().ToCharArray().Select(c => (byte)c).ToArray(); @@ -749,7 +742,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests // Act int bytesScanned; - var returnValue = scan.Seek(ref seekVector, out bytesScanned, limit); + var returnValue = scan.Seek((byte)seek, out bytesScanned, limit); // Assert Assert.Equal(expectedBytesScanned, bytesScanned); @@ -779,8 +772,6 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); - var input1 = input.Substring(0, input.Length / 2); block1 = _pool.Lease(); var chars1 = input1.ToCharArray().Select(c => (byte)c).ToArray(); @@ -801,7 +792,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests // Act int bytesScanned; - var returnValue = scan.Seek(ref seekVector, out bytesScanned, limit); + var returnValue = scan.Seek((byte)seek, out bytesScanned, limit); // Assert Assert.Equal(expectedBytesScanned, bytesScanned); @@ -835,9 +826,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); - var limitAtVector = new Vector((byte)limitAt); - var afterSeekVector = new Vector((byte)'B'); + var afterSeek = (byte)'B'; block = _pool.Lease(); var chars = input.ToCharArray().Select(c => (byte)c).ToArray(); @@ -852,13 +841,13 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = scan1; // Act - var endReturnValue = end.Seek(ref limitAtVector); - var returnValue1 = scan1.Seek(ref seekVector, ref end); - var returnValue2_1 = scan2_1.Seek(ref seekVector, ref afterSeekVector, ref end); - var returnValue2_2 = scan2_2.Seek(ref afterSeekVector, ref seekVector, ref end); - var returnValue3_1 = scan3_1.Seek(ref seekVector, ref afterSeekVector, ref afterSeekVector, ref end); - var returnValue3_2 = scan3_2.Seek(ref afterSeekVector, ref seekVector, ref afterSeekVector, ref end); - var returnValue3_3 = scan3_3.Seek(ref afterSeekVector, ref afterSeekVector, ref seekVector, ref end); + var endReturnValue = end.Seek((byte)limitAt); + var returnValue1 = scan1.Seek((byte)seek, ref end); + var returnValue2_1 = scan2_1.Seek((byte)seek, afterSeek, ref end); + var returnValue2_2 = scan2_2.Seek(afterSeek, (byte)seek, ref end); + var returnValue3_1 = scan3_1.Seek((byte)seek, afterSeek, afterSeek, ref end); + var returnValue3_2 = scan3_2.Seek(afterSeek, (byte)seek, afterSeek, ref end); + var returnValue3_3 = scan3_3.Seek(afterSeek, afterSeek, (byte)seek, ref end); // Assert Assert.Equal(input.Contains(limitAt) ? limitAt : -1, endReturnValue); @@ -902,9 +891,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests try { // Arrange - var seekVector = new Vector((byte)seek); - var limitAtVector = new Vector((byte)limitAt); - var afterSeekVector = new Vector((byte)'B'); + var afterSeek = (byte)'B'; var input1 = input.Substring(0, input.Length / 2); block1 = _pool.Lease(); @@ -931,13 +918,13 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = scan1; // Act - var endReturnValue = end.Seek(ref limitAtVector); - var returnValue1 = scan1.Seek(ref seekVector, ref end); - var returnValue2_1 = scan2_1.Seek(ref seekVector, ref afterSeekVector, ref end); - var returnValue2_2 = scan2_2.Seek(ref afterSeekVector, ref seekVector, ref end); - var returnValue3_1 = scan3_1.Seek(ref seekVector, ref afterSeekVector, ref afterSeekVector, ref end); - var returnValue3_2 = scan3_2.Seek(ref afterSeekVector, ref seekVector, ref afterSeekVector, ref end); - var returnValue3_3 = scan3_3.Seek(ref afterSeekVector, ref afterSeekVector, ref seekVector, ref end); + var endReturnValue = end.Seek((byte)limitAt); + var returnValue1 = scan1.Seek((byte)seek, ref end); + var returnValue2_1 = scan2_1.Seek((byte)seek, afterSeek, ref end); + var returnValue2_2 = scan2_2.Seek(afterSeek, (byte)seek, ref end); + var returnValue3_1 = scan3_1.Seek((byte)seek, afterSeek, afterSeek, ref end); + var returnValue3_2 = scan3_2.Seek(afterSeek, (byte)seek, afterSeek, ref end); + var returnValue3_3 = scan3_3.Seek(afterSeek, afterSeek, (byte)seek, ref end); // Assert Assert.Equal(input.Contains(limitAt) ? limitAt : -1, endReturnValue); From 00a63537cf25f225546a0ef3264717ebd224be45 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 16 Oct 2016 21:58:49 +0100 Subject: [PATCH 05/20] And=> xor, powerOfTwoToHighByte to const --- .../Infrastructure/MemoryPoolIterator.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 3bff457c17..83fcc300f0 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -11,7 +11,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { public struct MemoryPoolIterator { - private static readonly ulong _powerOfTwoToHighByte = PowerOfTwoToHighByte(); + private const ulong _xorPowerOfTwoToHighByte = (0x07ul | + 0x06ul << 8 | + 0x05ul << 16 | + 0x04ul << 24 | + 0x03ul << 32 | + 0x02ul << 40 | + 0x01ul << 48 ) + 1; + private static readonly int _vectorSpan = Vector.Count; private MemoryPoolBlock _block; @@ -785,9 +792,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } // Flag least significant power of two bit - var powerOfTwoFlag = (ulong)(longValue & -longValue); + var powerOfTwoFlag = (ulong)(longValue ^ (longValue - 1)); // Shift all powers of two into the high byte and extract - var foundByteIndex = (int)((powerOfTwoFlag * _powerOfTwoToHighByte) >> 61); + var foundByteIndex = (int)((powerOfTwoFlag * _xorPowerOfTwoToHighByte) >> 57); // Single LEA instruction with jitted const (using function result) return i * 8 + foundByteIndex; } @@ -1044,9 +1051,5 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return Vector.AsVectorByte(new Vector(vectorByte * 0x0101010101010101ul)); } - private static ulong PowerOfTwoToHighByte() - { - return BitConverter.IsLittleEndian ? 0x20406080A0C0E0ul : 0xE0C0A080604020ul; - } } } From 28a21fa7a953e79ea150a9e44daea640cb442314 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 16 Oct 2016 22:12:56 +0100 Subject: [PATCH 06/20] Move vectors closer to use --- .../Infrastructure/MemoryPoolIterator.cs | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 83fcc300f0..f8327cebec 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -371,8 +371,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - byte[] array; - var byte0Vector = GetVector(byte0); while (true) { @@ -392,7 +390,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure wasLastBlock = block.Next == null; following = block.End - index; } - array = block.Array; + var array = block.Array; while (following > 0) { // Need unit tests to test Vector path @@ -403,7 +401,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure #endif if (following >= _vectorSpan) { - var byte0Equals = Vector.Equals(new Vector(array, index), byte0Vector); + var byte0Equals = Vector.Equals(new Vector(array, index), GetVector(byte0)); if (byte0Equals.Equals(Vector.Zero)) { @@ -480,11 +478,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; - var byte0Vector = GetVector(byte0); - var byte1Vector = GetVector(byte1); while (true) { @@ -503,7 +498,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure wasLastBlock = block.Next == null; following = block.End - index; } - array = block.Array; + var array = block.Array; while (following > 0) { @@ -517,13 +512,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); + var byte0Equals = Vector.Equals(data, GetVector(byte0)); if (!byte0Equals.Equals(Vector.Zero)) { byte0Index = LocateFirstFoundByte(ref byte0Equals); } - var byte1Equals = Vector.Equals(data, byte1Vector); + var byte1Equals = Vector.Equals(data, GetVector(byte1)); if (!byte1Equals.Equals(Vector.Zero)) { byte1Index = LocateFirstFoundByte(ref byte1Equals); @@ -623,13 +618,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - byte[] array; int byte0Index = int.MaxValue; int byte1Index = int.MaxValue; int byte2Index = int.MaxValue; - var byte0Vector = GetVector(byte0); - var byte1Vector = GetVector(byte1); - var byte2Vector = GetVector(byte2); while (true) { @@ -648,7 +639,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure wasLastBlock = block.Next == null; following = block.End - index; } - array = block.Array; + var array = block.Array; while (following > 0) { // Need unit tests to test Vector path @@ -661,19 +652,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); + var byte0Equals = Vector.Equals(data, GetVector(byte0)); if (!byte0Equals.Equals(Vector.Zero)) { byte0Index = LocateFirstFoundByte(ref byte0Equals); } - var byte1Equals = Vector.Equals(data, byte1Vector); + var byte1Equals = Vector.Equals(data, GetVector(byte1)); if (!byte1Equals.Equals(Vector.Zero)) { byte1Index = LocateFirstFoundByte(ref byte1Equals); } - var byte2Equals = Vector.Equals(data, byte2Vector); + var byte2Equals = Vector.Equals(data, GetVector(byte2)); if (!byte2Equals.Equals(Vector.Zero)) { byte2Index = LocateFirstFoundByte(ref byte2Equals); From 8bcbfb9971b208e966b9b8f10010ad287acd4f60 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 9 Nov 2016 11:40:31 -0800 Subject: [PATCH 07/20] Only LocateFirstFoundByte once --- .../Infrastructure/MemoryPoolIterator.cs | 111 +++++------------- .../MemoryPoolIteratorTests.cs | 4 +- 2 files changed, 33 insertions(+), 82 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index f8327cebec..67856d530b 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -316,7 +316,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = LocateFirstFoundByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(byte0Equals); var vectorBytesScanned = firstEqualByteIndex + 1; if (bytesScanned + vectorBytesScanned > limit) @@ -420,7 +420,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = LocateFirstFoundByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(byte0Equals); if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) { @@ -478,8 +478,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - int byte0Index = int.MaxValue; - int byte1Index = int.MaxValue; + int byteIndex = int.MaxValue; while (true) { @@ -512,19 +511,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, GetVector(byte0)); - if (!byte0Equals.Equals(Vector.Zero)) + var byteEquals = Vector.Equals(data, GetVector(byte0)); + byteEquals = Vector.ConditionalSelect(byteEquals, byteEquals, Vector.Equals(data, GetVector(byte1))); + + if (!byteEquals.Equals(Vector.Zero)) { - byte0Index = LocateFirstFoundByte(ref byte0Equals); + byteIndex = LocateFirstFoundByte(byteEquals); } - var byte1Equals = Vector.Equals(data, GetVector(byte1)); - if (!byte1Equals.Equals(Vector.Zero)) - { - byte1Index = LocateFirstFoundByte(ref byte1Equals); - } - - if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) + if (byteIndex == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; @@ -542,21 +537,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - if (byte0Index < byte1Index) - { - _index = index + byte0Index; - - if (block == limit.Block && _index > limit.Index) - { - // Ensure iterator is left at limit position - _index = limit.Index; - return -1; - } - - return byte0; - } - - _index = index + byte1Index; + _index = index + byteIndex; if (block == limit.Block && _index > limit.Index) { @@ -565,7 +546,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } - return byte1; + _index = index + byteIndex; + + if (block == limit.Block && _index > limit.Index) + { + // Ensure iterator is left at limit position + _index = limit.Index; + return -1; + } + + return block.Array[index + byteIndex]; } // Need unit tests to test Vector path #if !DEBUG @@ -618,9 +608,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; - int byte0Index = int.MaxValue; - int byte1Index = int.MaxValue; - int byte2Index = int.MaxValue; + int byteIndex = int.MaxValue; while (true) { @@ -652,25 +640,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, GetVector(byte0)); - if (!byte0Equals.Equals(Vector.Zero)) + var byteEquals = Vector.Equals(data, GetVector(byte0)); + byteEquals = Vector.ConditionalSelect(byteEquals, byteEquals, Vector.Equals(data, GetVector(byte1))); + byteEquals = Vector.ConditionalSelect(byteEquals, byteEquals, Vector.Equals(data, GetVector(byte2))); + + if (!byteEquals.Equals(Vector.Zero)) { - byte0Index = LocateFirstFoundByte(ref byte0Equals); + byteIndex = LocateFirstFoundByte(byteEquals); } - var byte1Equals = Vector.Equals(data, GetVector(byte1)); - if (!byte1Equals.Equals(Vector.Zero)) - { - byte1Index = LocateFirstFoundByte(ref byte1Equals); - } - - var byte2Equals = Vector.Equals(data, GetVector(byte2)); - if (!byte2Equals.Equals(Vector.Zero)) - { - byte2Index = LocateFirstFoundByte(ref byte2Equals); - } - - if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) + if (byteIndex == int.MaxValue) { following -= _vectorSpan; index += _vectorSpan; @@ -688,35 +667,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - int toReturn, toMove; - if (byte0Index < byte1Index) - { - if (byte0Index < byte2Index) - { - toReturn = byte0; - toMove = byte0Index; - } - else - { - toReturn = byte2; - toMove = byte2Index; - } - } - else - { - if (byte1Index < byte2Index) - { - toReturn = byte1; - toMove = byte1Index; - } - else - { - toReturn = byte2; - toMove = byte2Index; - } - } - - _index = index + toMove; + _index = index + byteIndex; if (block == limit.Block && _index > limit.Index) { @@ -725,7 +676,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } - return toReturn; + return block.Array[index + byteIndex]; } // Need unit tests to test Vector path #if !DEBUG @@ -770,7 +721,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// The first index of the result vector // Force inlining (64 IL bytes, 91 bytes asm) Issue: https://github.com/dotnet/coreclr/issues/7386 [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int LocateFirstFoundByte(ref Vector byteEquals) + internal static int LocateFirstFoundByte(Vector byteEquals) { var vector64 = Vector.AsVectorInt64(byteEquals); long longValue = 0; diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index c0ff83603b..156832ca0f 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests for (int i = 0; i < Vector.Count; i++) { Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(vector)); bytes[i] = 0; } @@ -40,7 +40,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests { bytes[i] = 1; Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(vector)); bytes[i] = 0; } } From 53f361160e55ba3a7c2fea3fed7ea11a792d5184 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 9 Nov 2016 09:57:59 -0800 Subject: [PATCH 08/20] MemoryPoolIterator fast paths --- .../Infrastructure/MemoryPoolIterator.cs | 72 +++++++++++++++---- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 67856d530b..f41a6542f0 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -39,6 +39,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public bool IsEnd { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { if (_block == null) @@ -51,24 +52,30 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } else { - var block = _block.Next; - while (block != null) - { - if (block.Start < block.End) - { - return false; // subsequent block has data - IsEnd is false - } - block = block.Next; - } - return true; + return IsEndMultiBlock(); } } } + private bool IsEndMultiBlock() + { + var block = _block.Next; + while (block != null) + { + if (block.Start < block.End) + { + return false; // subsequent block has data - IsEnd is false + } + block = block.Next; + } + return true; + } + public MemoryPoolBlock Block => _block; public int Index => _index; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Take() { var block = _block; @@ -78,7 +85,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } var index = _index; - var wasLastBlock = block.Next == null; if (index < block.End) { @@ -86,6 +92,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return block.Array[index]; } + return TakeMultiBlock(block, index); + } + + private int TakeMultiBlock(MemoryPoolBlock block, int index) + { + var wasLastBlock = block.Next == null; do { if (wasLastBlock) @@ -109,6 +121,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } while (true); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Skip(int bytesToSkip) { if (_block == null) @@ -116,7 +129,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return; } - var wasLastBlock = _block.Next == null; var following = _block.End - _index; if (following >= bytesToSkip) @@ -125,6 +137,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return; } + SkipMultiBlock(bytesToSkip, following); + } + + private void SkipMultiBlock(int bytesToSkip, int following) + { + var wasLastBlock = _block.Next == null; var block = _block; var index = _index; while (true) @@ -152,6 +170,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Peek() { var block = _block; @@ -160,7 +179,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } - var wasLastBlock = _block.Next == null; var index = _index; if (index < block.End) @@ -168,6 +186,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return block.Array[index]; } + return PeekMultiBlock(block, index); + } + + private static int PeekMultiBlock(MemoryPoolBlock block, int index) + { + var wasLastBlock = block.Next == null; do { if (wasLastBlock) @@ -190,6 +214,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } // NOTE: Little-endian only! + [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe bool TryPeekLong(out ulong longValue) { longValue = 0; @@ -199,7 +224,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return false; } - var wasLastBlock = _block.Next == null; var blockBytes = _block.End - _index; if (blockBytes >= sizeof(ulong)) @@ -207,7 +231,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure longValue = *(ulong*)(_block.DataFixedPtr + _index); return true; } - else if (wasLastBlock) + + return TryPeekLongMultiBlock(ref longValue, blockBytes); + } + + private unsafe bool TryPeekLongMultiBlock(ref ulong longValue, int blockBytes) + { + var wasLastBlock = _block.Next == null; + if (wasLastBlock) { return false; } @@ -778,6 +809,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetLength(MemoryPoolIterator end) { if (IsDefault || end.IsDefault) @@ -785,6 +817,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return -1; } + if (_block == end._block) + { + return end._index - _index; + } + + return GetLengthMultiBlock(end); + } + + public int GetLengthMultiBlock(MemoryPoolIterator end) + { var block = _block; var index = _index; var length = 0; From 90c7be1fc0c6566a37f487dc165193d3aa2cb961 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 15 Nov 2016 00:42:04 +0000 Subject: [PATCH 09/20] Add Request Parsing benchmark --- .gitignore | 2 + KestrelHttpServer.sln | 9 + global.json | 2 +- ...spNetCore.Server.Kestrel.Performance.xproj | 22 ++ .../Program.cs | 48 +++++ .../Readme.md | 11 + .../RequestParsing.cs | 200 ++++++++++++++++++ .../columns/RpsColumn.cs | 31 +++ .../configs/CoreConfig.cs | 31 +++ .../configs/DefaultConfig.cs | 42 ++++ .../helpers/MockApplicationErrorLogger.cs | 51 +++++ .../helpers/MockConnectionControl.cs | 34 +++ .../helpers/MockKestrelTrace.cs | 36 ++++ .../helpers/SocketInputExtensions.cs | 37 ++++ .../project.json | 29 +++ 15 files changed, 584 insertions(+), 1 deletion(-) create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs create mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json diff --git a/.gitignore b/.gitignore index 6acc284439..af0898e29a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ runtimes/ .build/ .testPublish/ launchSettings.json +BenchmarkDotNet.Artifacts/ +BDN.Generated/ diff --git a/KestrelHttpServer.sln b/KestrelHttpServer.sln index 1a5cda8a81..e6dca6eac9 100644 --- a/KestrelHttpServer.sln +++ b/KestrelHttpServer.sln @@ -45,6 +45,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{0EF2AC test\shared\TestServiceContext.cs = test\shared\TestServiceContext.cs EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "perf", "perf", "{21B17FBB-5A58-42A8-8473-43160509A9FF}" +EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.AspNetCore.Server.Kestrel.Performance", "perf\Microsoft.AspNetCore.Server.Kestrel.Performance\Microsoft.AspNetCore.Server.Kestrel.Performance.xproj", "{70567566-524C-4B67-9B59-E5C206D6C2EB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -79,6 +83,10 @@ Global {9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Debug|Any CPU.Build.0 = Debug|Any CPU {9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Release|Any CPU.ActiveCfg = Release|Any CPU {9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Release|Any CPU.Build.0 = Release|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {70567566-524C-4B67-9B59-E5C206D6C2EB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -92,5 +100,6 @@ Global {5F64B3C3-0C2E-431A-B820-A81BBFC863DA} = {2D5D5227-4DBD-499A-96B1-76A36B03B750} {9559A5F1-080C-4909-B6CF-7E4B3DC55748} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} {0EF2ACDF-012F-4472-A13A-4272419E2903} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} + {70567566-524C-4B67-9B59-E5C206D6C2EB} = {21B17FBB-5A58-42A8-8473-43160509A9FF} EndGlobalSection EndGlobal diff --git a/global.json b/global.json index 983ba0401e..c0415c19af 100644 --- a/global.json +++ b/global.json @@ -1,3 +1,3 @@ { - "projects": ["src"] + "projects": ["src", "perf"] } diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj new file mode 100644 index 0000000000..3bbc5601e0 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj @@ -0,0 +1,22 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 70567566-524c-4b67-9b59-e5c206d6c2eb + Microsoft.AspNetCore.Server.Kestrel.Performance + .\obj + .\bin\ + v4.6.2 + + + 2.0 + + + + + + \ No newline at end of file diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs new file mode 100644 index 0000000000..db29b584db --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using BenchmarkDotNet.Running; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class Program + { + public static void Main(string[] args) + { + var options = (uint[])Enum.GetValues(typeof(BenchmarkType)); + BenchmarkType type; + if (args.Length != 1 || !Enum.TryParse(args[0], out type)) + { + Console.WriteLine($"Please add benchmark to run as parameter:"); + for (var i = 0; i < options.Length; i++) + { + Console.WriteLine($" {((BenchmarkType)options[i]).ToString()}"); + } + + return; + } + + RunSelectedBenchmarks(type); + } + + private static void RunSelectedBenchmarks(BenchmarkType type) + { + if (type.HasFlag(BenchmarkType.RequestParsing)) + { + BenchmarkRunner.Run(); + } + } + } + + [Flags] + public enum BenchmarkType : uint + { + RequestParsing = 1, + // add new ones in powers of two - e.g. 2,4,8,16... + + All = uint.MaxValue + } + + +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md new file mode 100644 index 0000000000..4088c38007 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md @@ -0,0 +1,11 @@ +Compile the solution in Release mode (so Kestrel is available in release) + +To run a specific benchmark add it as parameter +``` +dotnet run -c Release RequestParsing +``` +To run all use `All` as parameter +``` +dotnet run -c Release All +``` +Using no parameter will list all available benchmarks \ No newline at end of file diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs new file mode 100644 index 0000000000..b1bdbe16f9 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs @@ -0,0 +1,200 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Attributes; +using Microsoft.AspNetCore.Server.Kestrel.Internal; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure; +using RequestLineStatus = Microsoft.AspNetCore.Server.Kestrel.Internal.Http.Frame.RequestLineStatus; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + [Config(typeof(CoreConfig))] + public class RequestParsing + { + private const int InnerLoopCount = 512; + private const int Pipelining = 16; + + private const string plaintextRequest = "GET /plaintext HTTP/1.1\r\nHost: www.example.com\r\n\r\n"; + + private const string liveaspnetRequest = "GET https://live.asp.net/ HTTP/1.1\r\n" + + "Host: live.asp.net\r\n" + + "Connection: keep-alive\r\n" + + "Upgrade-Insecure-Requests: 1\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" + + "DNT: 1\r\n" + + "Accept-Encoding: gzip, deflate, sdch, br\r\n" + + "Accept-Language: en-US,en;q=0.8\r\n" + + "Cookie: __unam=7a67379-1s65dc575c4-6d778abe-1; omniID=9519gfde_3347_4762_8762_df51458c8ec2\r\n\r\n"; + + private const string unicodeRequest = + "GET http://stackoverflow.com/questions/40148683/why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric HTTP/1.1\r\n" + + "Accept: text/html, application/xhtml+xml, image/jxr, */*\r\n" + + "Accept-Language: en-US,en-GB;q=0.7,en;q=0.3\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.14965\r\n" + + "Accept-Encoding: gzip, deflate\r\n" + + "Host: stackoverflow.com\r\n" + + "Connection: Keep-Alive\r\n" + + "Cache-Control: max-age=0\r\n" + + "Upgrade-Insecure-Requests: 1\r\n" + + "DNT: 1\r\n" + + "Referer: http://stackoverflow.com/?tab=month\r\n" + + "Pragma: no-cache\r\n" + + "Cookie: prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric\r\n\r\n"; + + private static readonly byte[] _plaintextPipelinedRequests = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(plaintextRequest, Pipelining))); + private static readonly byte[] _plaintextRequest = Encoding.ASCII.GetBytes(plaintextRequest); + + private static readonly byte[] _liveaspnentPipelinedRequests = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(liveaspnetRequest, Pipelining))); + private static readonly byte[] _liveaspnentRequest = Encoding.ASCII.GetBytes(liveaspnetRequest); + + private static readonly byte[] _unicodePipelinedRequests = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(unicodeRequest, Pipelining))); + private static readonly byte[] _unicodeRequest = Encoding.ASCII.GetBytes(unicodeRequest); + + private KestrelTrace Trace; + private LoggingThreadPool ThreadPool; + private MemoryPool MemoryPool; + private SocketInput SocketInput; + private Frame Frame; + + [Benchmark(Baseline = true, OperationsPerInvoke = InnerLoopCount)] + public void ParsePlaintext() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_plaintextRequest); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount * Pipelining)] + public void ParsePipelinedPlaintext() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_plaintextPipelinedRequests); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount)] + public void ParseLiveAspNet() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_liveaspnentRequest); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount * Pipelining)] + public void ParsePipelinedLiveAspNet() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_liveaspnentPipelinedRequests); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount)] + public void ParseUnicode() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_unicodeRequest); + + ParseData(); + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount * Pipelining)] + public void ParseUnicodePipelined() + { + for (var i = 0; i < InnerLoopCount; i++) + { + InsertData(_unicodePipelinedRequests); + + ParseData(); + } + } + + private void InsertData(byte[] dataBytes) + { + SocketInput.IncomingData(dataBytes, 0, dataBytes.Length); + } + + private void ParseData() + { + while (SocketInput.GetAwaiter().IsCompleted) + { + Frame.Reset(); + + if (Frame.TakeStartLine(SocketInput) != RequestLineStatus.Done) + { + ThrowInvalidStartLine(); + } + + Frame.InitializeHeaders(); + + if (!Frame.TakeMessageHeaders(SocketInput, (FrameRequestHeaders) Frame.RequestHeaders)) + { + ThrowInvalidMessageHeaders(); + } + } + } + + private void ThrowInvalidStartLine() + { + throw new InvalidOperationException("Invalid StartLine"); + } + + private void ThrowInvalidMessageHeaders() + { + throw new InvalidOperationException("Invalid MessageHeaders"); + } + + [Setup] + public void Setup() + { + Trace = new KestrelTrace(new MockKestrelTrace()); + ThreadPool = new LoggingThreadPool(Trace); + MemoryPool = new MemoryPool(); + SocketInput = new SocketInput(MemoryPool, ThreadPool); + + var serviceContext = new ServiceContext + { + DateHeaderValueManager = new DateHeaderValueManager(), + ServerOptions = new KestrelServerOptions(), + Log = Trace + }; + var listenerContext = new ListenerContext(serviceContext) + { + ServerAddress = ServerAddress.FromUrl("http://localhost:5000") + }; + var connectionContext = new ConnectionContext(listenerContext) + { + ConnectionControl = new MockConnectionControl(), + SocketInput = SocketInput + }; + + Frame = new Frame(application: null, context: connectionContext); + } + + [Cleanup] + public void Cleanup() + { + SocketInput.IncomingFin(); + SocketInput.Dispose(); + MemoryPool.Dispose(); + } + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs new file mode 100644 index 0000000000..12e9969ed2 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class RpsColumn : IColumn + { + private static int NanosPerSecond = 1000 * 1000 * 1000; + + public string GetValue(Summary summary, Benchmark benchmark) + { + var totalNanos = summary.Reports.First(r => r.Benchmark == benchmark).ResultStatistics.Mean; + // Make sure we don't divide by zero!! + return Math.Abs(totalNanos) > 0.0 ? (NanosPerSecond / totalNanos).ToString("N2") : "N/A"; + } + + public bool IsDefault(Summary summary, Benchmark benchmark) => false; + public bool IsAvailable(Summary summary) => true; + public string Id => "RPS-Column"; + public string ColumnName => "RPS"; + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Custom; + public int PriorityInCategory => 1; + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs new file mode 100644 index 0000000000..eff6663fec --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Validators; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class CoreConfig : ManualConfig + { + public CoreConfig() + { + Add(JitOptimizationsValidator.FailOnError); + Add(new RpsColumn()); + + Add(Job.Default. + With(Platform.X64). + With(Jit.RyuJit). + With(BenchmarkDotNet.Environments.Runtime.Core). + WithRemoveOutliers(true). + With(new GcMode() { Server = true }). + With(RunStrategy.Throughput). + WithLaunchCount(3). + WithWarmupCount(5). + WithTargetCount(10)); + } + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs new file mode 100644 index 0000000000..01f30373c4 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs @@ -0,0 +1,42 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Validators; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class DefaultConfig : ManualConfig + { + public DefaultConfig() + { + Add(JitOptimizationsValidator.FailOnError); + Add(new RpsColumn()); + + Add(Job.Default. + With(Platform.X64). + With(Jit.RyuJit). + With(BenchmarkDotNet.Environments.Runtime.Clr). + WithRemoveOutliers(true). + With(new GcMode() { Server = true }). + With(RunStrategy.Throughput). + WithLaunchCount(3). + WithWarmupCount(5). + WithTargetCount(10)); + + Add(Job.Default. + With(Platform.X64). + With(Jit.RyuJit). + With(BenchmarkDotNet.Environments.Runtime.Core). + WithRemoveOutliers(true). + With(new GcMode() { Server = true }). + With(RunStrategy.Throughput). + WithLaunchCount(3). + WithWarmupCount(5). + WithTargetCount(10)); + } + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs new file mode 100644 index 0000000000..c126eba79c --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Server.Kestrel.Internal; +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class MockApplicationErrorLogger : ILogger + { + // Application errors are logged using 13 as the eventId. + private const int ApplicationErrorEventId = 13; + + public List Messages { get; } = new List(); + + public int TotalErrorsLogged => Messages.Count(message => message.LogLevel == LogLevel.Error); + + public int CriticalErrorsLogged => Messages.Count(message => message.LogLevel == LogLevel.Critical); + + public int ApplicationErrorsLogged => Messages.Count(message => message.EventId.Id == ApplicationErrorEventId); + + public IDisposable BeginScope(TState state) + { + return new Disposable(() => { }); + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { +#if false + Console.WriteLine($"Log {logLevel}[{eventId}]: {formatter(state, exception)} {exception?.Message}"); +#endif + + Messages.Add(new LogMessage { LogLevel = logLevel, EventId = eventId, Exception = exception }); + } + + public class LogMessage + { + public LogLevel LogLevel { get; set; } + public EventId EventId { get; set; } + public Exception Exception { get; set; } + } + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs new file mode 100644 index 0000000000..b3587d81db --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class MockConnectionControl : IConnectionControl + { + public void CancelTimeout() + { + } + + public void End(ProduceEndType endType) + { + } + + public void Pause() + { + } + + public void ResetTimeout(long milliseconds, TimeoutAction timeoutAction) + { + } + + public void Resume() + { + } + + public void SetTimeout(long milliseconds, TimeoutAction timeoutAction) + { + } + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs new file mode 100644 index 0000000000..9e035010d3 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Server.Kestrel.Internal; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class MockKestrelTrace : KestrelTrace + { + public MockKestrelTrace() : this(new MockApplicationErrorLogger()) + { + } + + public MockKestrelTrace(MockApplicationErrorLogger testLogger) : base(testLogger) + { + Logger = testLogger; + } + + public MockApplicationErrorLogger Logger { get; private set; } + + public override void ConnectionRead(string connectionId, int count) + { + //_logger.LogDebug(1, @"Connection id ""{ConnectionId}"" recv {count} bytes.", connectionId, count); + } + + public override void ConnectionWrite(string connectionId, int count) + { + //_logger.LogDebug(1, @"Connection id ""{ConnectionId}"" send {count} bytes.", connectionId, count); + } + + public override void ConnectionWriteCallback(string connectionId, int status) + { + //_logger.LogDebug(1, @"Connection id ""{ConnectionId}"" send finished with status {status}.", connectionId, status); + } + } +} \ No newline at end of file diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs new file mode 100644 index 0000000000..af26c2994c --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public static class SocketInputExtensions + { + public static void IncomingData(this SocketInput input, byte[] buffer, int offset, int count) + { + var bufferIndex = offset; + var remaining = count; + + while (remaining > 0) + { + var block = input.IncomingStart(); + + var bytesLeftInBlock = block.Data.Offset + block.Data.Count - block.End; + var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock; + + Buffer.BlockCopy(buffer, bufferIndex, block.Array, block.End, bytesToCopy); + + bufferIndex += bytesToCopy; + remaining -= bytesToCopy; + + input.IncomingComplete(bytesToCopy, null); + } + } + + public static void IncomingFin(this SocketInput input) + { + input.IncomingComplete(0, null); + } + } +} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json new file mode 100644 index 0000000000..38bb58ec76 --- /dev/null +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json @@ -0,0 +1,29 @@ +{ + "version": "1.0.0-*", + "buildOptions": { + "emitEntryPoint": true + }, + + "dependencies": { + "BenchmarkDotNet": "0.10.0", + "Microsoft.AspNetCore.Server.Kestrel": "1.2.0-*" + }, + + "frameworks": { + "net46": {}, + "netcoreapp1.0": { + "dependencies": { + "Microsoft.NETCore.App": { + "version": "1.0.1-*", + "type": "platform" + } + } + } + }, + + "runtimeOptions": { + "configProperties": { + "System.GC.Server": true + } + } +} \ No newline at end of file From 12e2f3057756069f0ff4431fb530a5c259767efe Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 15 Nov 2016 12:01:13 +0000 Subject: [PATCH 10/20] MemoryPoolIterator byref structs --- .../Internal/Http/Frame.cs | 14 +- .../Infrastructure/MemoryPoolIterator.cs | 161 ++++++++++++++++++ .../MemoryPoolIteratorExtensions.cs | 155 +---------------- .../AsciiDecoding.cs | 8 +- .../UrlPathDecoder.cs | 8 +- 5 files changed, 179 insertions(+), 167 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs index 399976939c..59891a7355 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs @@ -975,7 +975,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); } - method = begin.GetAsciiString(scan); + method = begin.GetAsciiString(ref scan); if (method == null) { @@ -1031,7 +1031,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http RejectRequest(RequestRejectionReason.InvalidRequestLine, Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty); } - queryString = begin.GetAsciiString(scan); + queryString = begin.GetAsciiString(ref scan); } var queryEnd = scan; @@ -1081,16 +1081,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http if (needDecode) { // Read raw target before mutating memory. - rawTarget = pathBegin.GetAsciiString(queryEnd); + rawTarget = pathBegin.GetAsciiString(ref queryEnd); // URI was encoded, unescape and then parse as utf8 pathEnd = UrlPathDecoder.Unescape(pathBegin, pathEnd); - requestUrlPath = pathBegin.GetUtf8String(pathEnd); + requestUrlPath = pathBegin.GetUtf8String(ref pathEnd); } else { // URI wasn't encoded, parse as ASCII - requestUrlPath = pathBegin.GetAsciiString(pathEnd); + requestUrlPath = pathBegin.GetAsciiString(ref pathEnd); if (queryString.Length == 0) { @@ -1100,7 +1100,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } else { - rawTarget = pathBegin.GetAsciiString(queryEnd); + rawTarget = pathBegin.GetAsciiString(ref queryEnd); } } @@ -1341,7 +1341,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } while (ch != ByteCR); var name = beginName.GetArraySegment(endName); - var value = beginValue.GetAsciiString(endValue); + var value = beginValue.GetAsciiString(ref endValue); consumed = scan; requestHeaders.Append(name.Array, name.Offset, name.Count, value); diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index f41a6542f0..78f782d39e 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -5,7 +5,9 @@ using System; using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { @@ -19,6 +21,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure 0x02ul << 40 | 0x01ul << 48 ) + 1; + private static readonly Encoding _utf8 = Encoding.UTF8; private static readonly int _vectorSpan = Vector.Count; private MemoryPoolBlock _block; @@ -1026,6 +1029,164 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _index = blockIndex; } + public unsafe string GetAsciiString(ref MemoryPoolIterator end) + { + if (IsDefault || end.IsDefault) + { + return null; + } + + var length = GetLength(end); + + if (length == 0) + { + return null; + } + + var inputOffset = Index; + var block = Block; + + var asciiString = new string('\0', length); + + fixed (char* outputStart = asciiString) + { + var output = outputStart; + var remaining = length; + + var endBlock = end.Block; + var endIndex = end.Index; + + var outputOffset = 0; + while (true) + { + int following = (block != endBlock ? block.End : endIndex) - inputOffset; + + if (following > 0) + { + if (!AsciiUtilities.TryGetAsciiString(block.DataFixedPtr + inputOffset, output + outputOffset, following)) + { + throw BadHttpRequestException.GetException(RequestRejectionReason.NonAsciiOrNullCharactersInInputString); + } + + outputOffset += following; + remaining -= following; + } + + if (remaining == 0) + { + break; + } + + block = block.Next; + inputOffset = block.Start; + } + } + + return asciiString; + } + + public string GetUtf8String(ref MemoryPoolIterator end) + { + if (IsDefault || end.IsDefault) + { + return default(string); + } + if (end.Block == Block) + { + return _utf8.GetString(Block.Array, Index, end.Index - Index); + } + + var decoder = _utf8.GetDecoder(); + + var length = GetLength(end); + var charLength = length; + // Worse case is 1 byte = 1 char + var chars = new char[charLength]; + var charIndex = 0; + + var block = Block; + var index = Index; + var remaining = length; + while (true) + { + int bytesUsed; + int charsUsed; + bool completed; + var following = block.End - index; + if (remaining <= following) + { + decoder.Convert( + block.Array, + index, + remaining, + chars, + charIndex, + charLength - charIndex, + true, + out bytesUsed, + out charsUsed, + out completed); + return new string(chars, 0, charIndex + charsUsed); + } + else if (block.Next == null) + { + decoder.Convert( + block.Array, + index, + following, + chars, + charIndex, + charLength - charIndex, + true, + out bytesUsed, + out charsUsed, + out completed); + return new string(chars, 0, charIndex + charsUsed); + } + else + { + decoder.Convert( + block.Array, + index, + following, + chars, + charIndex, + charLength - charIndex, + false, + out bytesUsed, + out charsUsed, + out completed); + charIndex += charsUsed; + remaining -= following; + block = block.Next; + index = block.Start; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ArraySegment GetArraySegment(MemoryPoolIterator end) + { + if (IsDefault || end.IsDefault) + { + return default(ArraySegment); + } + if (end.Block == Block) + { + return new ArraySegment(Block.Array, Index, end.Index - Index); + } + + return GetArraySegmentMultiBlock(ref end); + } + + private ArraySegment GetArraySegmentMultiBlock(ref MemoryPoolIterator end) + { + var length = GetLength(end); + var array = new byte[length]; + CopyTo(array, 0, length, out length); + return new ArraySegment(array, 0, length); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector GetVector(byte vectorByte) { diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs index d59f689662..0b7e988af3 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Text; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; @@ -71,62 +72,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } - public unsafe static string GetAsciiString(this MemoryPoolIterator start, MemoryPoolIterator end) - { - if (start.IsDefault || end.IsDefault) - { - return null; - } - - var length = start.GetLength(end); - - if (length == 0) - { - return null; - } - - var inputOffset = start.Index; - var block = start.Block; - - var asciiString = new string('\0', length); - - fixed (char* outputStart = asciiString) - { - var output = outputStart; - var remaining = length; - - var endBlock = end.Block; - var endIndex = end.Index; - - var outputOffset = 0; - while (true) - { - int following = (block != endBlock ? block.End : endIndex) - inputOffset; - - if (following > 0) - { - if (!AsciiUtilities.TryGetAsciiString(block.DataFixedPtr + inputOffset, output + outputOffset, following)) - { - throw BadHttpRequestException.GetException(RequestRejectionReason.NonAsciiOrNullCharactersInInputString); - } - - outputOffset += following; - remaining -= following; - } - - if (remaining == 0) - { - break; - } - - block = block.Next; - inputOffset = block.Start; - } - } - - return asciiString; - } - public static string GetAsciiStringEscaped(this MemoryPoolIterator start, MemoryPoolIterator end, int maxChars) { var sb = new StringBuilder(); @@ -147,102 +92,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return sb.ToString(); } - public static string GetUtf8String(this MemoryPoolIterator start, MemoryPoolIterator end) - { - if (start.IsDefault || end.IsDefault) - { - return default(string); - } - if (end.Block == start.Block) - { - return _utf8.GetString(start.Block.Array, start.Index, end.Index - start.Index); - } - - var decoder = _utf8.GetDecoder(); - - var length = start.GetLength(end); - var charLength = length; - // Worse case is 1 byte = 1 char - var chars = new char[charLength]; - var charIndex = 0; - - var block = start.Block; - var index = start.Index; - var remaining = length; - while (true) - { - int bytesUsed; - int charsUsed; - bool completed; - var following = block.End - index; - if (remaining <= following) - { - decoder.Convert( - block.Array, - index, - remaining, - chars, - charIndex, - charLength - charIndex, - true, - out bytesUsed, - out charsUsed, - out completed); - return new string(chars, 0, charIndex + charsUsed); - } - else if (block.Next == null) - { - decoder.Convert( - block.Array, - index, - following, - chars, - charIndex, - charLength - charIndex, - true, - out bytesUsed, - out charsUsed, - out completed); - return new string(chars, 0, charIndex + charsUsed); - } - else - { - decoder.Convert( - block.Array, - index, - following, - chars, - charIndex, - charLength - charIndex, - false, - out bytesUsed, - out charsUsed, - out completed); - charIndex += charsUsed; - remaining -= following; - block = block.Next; - index = block.Start; - } - } - } - - public static ArraySegment GetArraySegment(this MemoryPoolIterator start, MemoryPoolIterator end) - { - if (start.IsDefault || end.IsDefault) - { - return default(ArraySegment); - } - if (end.Block == start.Block) - { - return new ArraySegment(start.Block.Array, start.Index, end.Index - start.Index); - } - - var length = start.GetLength(end); - var array = new byte[length]; - start.CopyTo(array, 0, length, out length); - return new ArraySegment(array, 0, length); - } - public static ArraySegment PeekArraySegment(this MemoryPoolIterator iter) { if (iter.IsDefault || iter.IsEnd) @@ -283,6 +132,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// The iterator from which to start the known string lookup. /// A reference to a pre-allocated known string, if the input matches any. /// true if the input matches a known string, false otherwise. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool GetKnownMethod(this MemoryPoolIterator begin, out string knownMethod) { knownMethod = null; @@ -323,6 +173,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// The iterator from which to start the known string lookup. /// A reference to a pre-allocated known string, if the input matches any. /// true if the input matches a known string, false otherwise. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool GetKnownVersion(this MemoryPoolIterator begin, out string knownVersion) { knownVersion = null; diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs index 87990788c6..220ae3d80b 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/AsciiDecoding.cs @@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem.GetIterator(); var end = GetIterator(begin, byteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(ref end); Assert.Equal(s.Length, byteRange.Length); @@ -58,7 +58,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem.GetIterator(); var end = GetIterator(begin, byteRange.Length); - Assert.Throws(() => begin.GetAsciiString(end)); + Assert.Throws(() => begin.GetAsciiString(ref end)); pool.Return(mem); } @@ -94,7 +94,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem0.GetIterator(); var end = GetIterator(begin, expectedByteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(ref end); Assert.Equal(s.Length, expectedByteRange.Length); @@ -135,7 +135,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var begin = mem0.GetIterator(); var end = GetIterator(begin, expectedByteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(ref end); Assert.Equal(expectedByteRange.Length, s.Length); diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs index c33c8e5168..0acfc27fe8 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/UrlPathDecoder.cs @@ -167,7 +167,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, rawLength); var end2 = UrlPathDecoder.Unescape(begin, end); - var result = begin.GetUtf8String(end2); + var result = begin.GetUtf8String(ref end2); Assert.Equal(expectLength, result.Length); Assert.Equal(expect, result); @@ -201,7 +201,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, raw.Length); var result = UrlPathDecoder.Unescape(begin, end); - Assert.Equal(expect, begin.GetUtf8String(result)); + Assert.Equal(expect, begin.GetUtf8String(ref result)); } private void PositiveAssert(MemoryPoolBlock mem, string raw) @@ -210,7 +210,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, raw.Length); var result = UrlPathDecoder.Unescape(begin, end); - Assert.NotEqual(raw.Length, begin.GetUtf8String(result).Length); + Assert.NotEqual(raw.Length, begin.GetUtf8String(ref result).Length); } private void NegativeAssert(MemoryPoolBlock mem, string raw) @@ -219,7 +219,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = GetIterator(begin, raw.Length); var resultEnd = UrlPathDecoder.Unescape(begin, end); - var result = begin.GetUtf8String(resultEnd); + var result = begin.GetUtf8String(ref resultEnd); Assert.Equal(raw, result); } } From 4cdcca212ed700bf3643f36dccde302f90c84f05 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 16 Nov 2016 04:56:45 +0000 Subject: [PATCH 11/20] Don't inline slow paths --- .../Infrastructure/MemoryPoolIterator.cs | 70 ++++++++++++++----- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 78f782d39e..f562c9ae02 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -60,6 +60,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } } + [MethodImpl(MethodImplOptions.NoInlining)] private bool IsEndMultiBlock() { var block = _block.Next; @@ -95,14 +96,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return block.Array[index]; } - return TakeMultiBlock(block, index); + return TakeMultiBlock(); } - private int TakeMultiBlock(MemoryPoolBlock block, int index) + [MethodImpl(MethodImplOptions.NoInlining)] + private int TakeMultiBlock() { + var block = _block; var wasLastBlock = block.Next == null; do { + int index; if (wasLastBlock) { return -1; @@ -143,13 +147,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure SkipMultiBlock(bytesToSkip, following); } + [MethodImpl(MethodImplOptions.NoInlining)] private void SkipMultiBlock(int bytesToSkip, int following) { var wasLastBlock = _block.Next == null; var block = _block; - var index = _index; while (true) { + int index; if (wasLastBlock) { throw new InvalidOperationException("Attempted to skip more bytes than available."); @@ -189,14 +194,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return block.Array[index]; } - return PeekMultiBlock(block, index); + return PeekMultiBlock(); } - private static int PeekMultiBlock(MemoryPoolBlock block, int index) + [MethodImpl(MethodImplOptions.NoInlining)] + private int PeekMultiBlock() { + var block = _block; var wasLastBlock = block.Next == null; do { + int index; if (wasLastBlock) { return -1; @@ -238,6 +246,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return TryPeekLongMultiBlock(ref longValue, blockBytes); } + [MethodImpl(MethodImplOptions.NoInlining)] private unsafe bool TryPeekLongMultiBlock(ref ulong longValue, int blockBytes) { var wasLastBlock = _block.Next == null; @@ -780,6 +789,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure /// /// The byte to be saved. /// true if the operation successes. false if can't find available space. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool Put(byte data) { if (_block == null) @@ -789,18 +799,26 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var block = _block; var index = _index; - while (true) - { - var wasLastBlock = block.Next == null; - if (index < block.End) - { - _block = block; - _index = index + 1; - block.Array[index] = data; - return true; - } - else if (wasLastBlock) + if (index < block.End) + { + _index = index + 1; + block.Array[index] = data; + return true; + } + + return PutMultiBlock(data); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private bool PutMultiBlock(byte data) + { + var block = _block; + var wasLastBlock = block.Next == null; + do + { + int index; + if (wasLastBlock) { return false; } @@ -809,7 +827,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure block = block.Next; index = block.Start; } - } + + wasLastBlock = block.Next == null; + + if (index < block.End) + { + _block = block; + _index = index + 1; + block.Array[index] = data; + break; + } + } while (true); + + return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -825,10 +855,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return end._index - _index; } - return GetLengthMultiBlock(end); + return GetLengthMultiBlock(ref end); } - public int GetLengthMultiBlock(MemoryPoolIterator end) + [MethodImpl(MethodImplOptions.NoInlining)] + public int GetLengthMultiBlock(ref MemoryPoolIterator end) { var block = _block; var index = _index; @@ -1179,6 +1210,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return GetArraySegmentMultiBlock(ref end); } + [MethodImpl(MethodImplOptions.NoInlining)] private ArraySegment GetArraySegmentMultiBlock(ref MemoryPoolIterator end) { var length = GetLength(end); From 7ab44423920eb57f48ef1aff09d8dd7f8b8e1270 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Fri, 18 Nov 2016 23:30:12 +0000 Subject: [PATCH 12/20] Feedback + Cleanup --- .../Program.cs | 2 -- .../configs/CoreConfig.cs | 4 ---- .../configs/DefaultConfig.cs | 4 ---- .../helpers/MockKestrelTrace.cs | 3 --- .../Internal/Infrastructure/Constants.cs | 3 +++ .../Internal/Infrastructure/MemoryPoolIterator.cs | 5 ++--- .../Internal/Infrastructure/MemoryPoolIteratorExtensions.cs | 3 --- 7 files changed, 5 insertions(+), 19 deletions(-) diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs index db29b584db..688771f622 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs @@ -43,6 +43,4 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance All = uint.MaxValue } - - } diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs index eff6663fec..72056a28b4 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs @@ -3,7 +3,6 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Validators; @@ -17,10 +16,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance Add(new RpsColumn()); Add(Job.Default. - With(Platform.X64). - With(Jit.RyuJit). With(BenchmarkDotNet.Environments.Runtime.Core). - WithRemoveOutliers(true). With(new GcMode() { Server = true }). With(RunStrategy.Throughput). WithLaunchCount(3). diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs index 01f30373c4..6b2670302c 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs @@ -20,7 +20,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance With(Platform.X64). With(Jit.RyuJit). With(BenchmarkDotNet.Environments.Runtime.Clr). - WithRemoveOutliers(true). With(new GcMode() { Server = true }). With(RunStrategy.Throughput). WithLaunchCount(3). @@ -28,10 +27,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance WithTargetCount(10)); Add(Job.Default. - With(Platform.X64). - With(Jit.RyuJit). With(BenchmarkDotNet.Environments.Runtime.Core). - WithRemoveOutliers(true). With(new GcMode() { Server = true }). With(RunStrategy.Throughput). WithLaunchCount(3). diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs index 9e035010d3..5d5bf38201 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs +++ b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs @@ -20,17 +20,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance public override void ConnectionRead(string connectionId, int count) { - //_logger.LogDebug(1, @"Connection id ""{ConnectionId}"" recv {count} bytes.", connectionId, count); } public override void ConnectionWrite(string connectionId, int count) { - //_logger.LogDebug(1, @"Connection id ""{ConnectionId}"" send {count} bytes.", connectionId, count); } public override void ConnectionWriteCallback(string connectionId, int status) { - //_logger.LogDebug(1, @"Connection id ""{ConnectionId}"" send finished with status {status}.", connectionId, status); } } } \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs index 77968bca17..32267912e0 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Runtime.InteropServices; +using System.Text; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { @@ -13,6 +14,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public static readonly int? ECONNRESET = GetECONNRESET(); public static readonly int? EADDRINUSE = GetEADDRINUSE(); + public static readonly Encoding UTF8 = Encoding.UTF8; + /// /// Prefix of host name used to specify Unix sockets in the configuration. /// diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index f562c9ae02..9f18b75465 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -21,7 +21,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure 0x02ul << 40 | 0x01ul << 48 ) + 1; - private static readonly Encoding _utf8 = Encoding.UTF8; private static readonly int _vectorSpan = Vector.Count; private MemoryPoolBlock _block; @@ -1124,10 +1123,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } if (end.Block == Block) { - return _utf8.GetString(Block.Array, Index, end.Index - Index); + return Constants.UTF8.GetString(Block.Array, Index, end.Index - Index); } - var decoder = _utf8.GetDecoder(); + var decoder = Constants.UTF8.GetDecoder(); var length = GetLength(end); var charLength = length; diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs index 0b7e988af3..1839d3309f 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs @@ -6,14 +6,11 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Text; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { public static class MemoryPoolIteratorExtensions { - private static readonly Encoding _utf8 = Encoding.UTF8; - public const string Http10Version = "HTTP/1.0"; public const string Http11Version = "HTTP/1.1"; From 5041d6c2918ff6d9e4b4bc3aab8a6e0796c32fa8 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 04:32:42 +0000 Subject: [PATCH 13/20] wasLastBlock adjustments --- .../Internal/Infrastructure/Constants.cs | 2 - .../Infrastructure/MemoryPoolIterator.cs | 197 +++++++++--------- 2 files changed, 103 insertions(+), 96 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs index 32267912e0..3acb659218 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs @@ -14,8 +14,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public static readonly int? ECONNRESET = GetECONNRESET(); public static readonly int? EADDRINUSE = GetEADDRINUSE(); - public static readonly Encoding UTF8 = Encoding.UTF8; - /// /// Prefix of host name used to specify Unix sockets in the configuration. /// diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 9f18b75465..7c7e0865e8 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -44,14 +44,19 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - if (_block == null) + var block = _block; + if (block == null) { return true; } - else if (_index < _block.End) + else if (_index < block.End) { return false; } + else if (block.Next == null) + { + return true; + } else { return IsEndMultiBlock(); @@ -63,14 +68,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure private bool IsEndMultiBlock() { var block = _block.Next; - while (block != null) + do { if (block.Start < block.End) { return false; // subsequent block has data - IsEnd is false } block = block.Next; - } + } while (block != null); + return true; } @@ -88,6 +94,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } var index = _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { @@ -95,28 +103,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return block.Array[index]; } - return TakeMultiBlock(); + return wasLastBlock ? -1 : TakeMultiBlock(); } [MethodImpl(MethodImplOptions.NoInlining)] private int TakeMultiBlock() { var block = _block; - var wasLastBlock = block.Next == null; do { - int index; - if (wasLastBlock) - { - return -1; - } - else - { - block = block.Next; - index = block.Start; - } + block = block.Next; + var index = block.Start; - wasLastBlock = block.Next == null; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { @@ -124,18 +124,26 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _index = index + 1; return block.Array[index]; } + + if (wasLastBlock) + { + return -1; + } } while (true); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Skip(int bytesToSkip) { - if (_block == null) + var block = _block; + if (block == null) { return; } - var following = _block.End - _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; + var following = block.End - _index; if (following >= bytesToSkip) { @@ -143,29 +151,26 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return; } + if (wasLastBlock) + { + ThrowInvalidOperationException_SkipMoreThanAvailable(); + } + SkipMultiBlock(bytesToSkip, following); } [MethodImpl(MethodImplOptions.NoInlining)] private void SkipMultiBlock(int bytesToSkip, int following) { - var wasLastBlock = _block.Next == null; var block = _block; - while (true) + do { - int index; - if (wasLastBlock) - { - throw new InvalidOperationException("Attempted to skip more bytes than available."); - } - else - { - bytesToSkip -= following; - block = block.Next; - index = block.Start; - } + bytesToSkip -= following; + block = block.Next; + var index = block.Start; - wasLastBlock = block.Next == null; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; following = block.End - index; if (following >= bytesToSkip) @@ -174,7 +179,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _index = index + bytesToSkip; return; } - } + + if (wasLastBlock) + { + ThrowInvalidOperationException_SkipMoreThanAvailable(); + } + } while (true); + } + + private static void ThrowInvalidOperationException_SkipMoreThanAvailable() + { + throw new InvalidOperationException("Attempted to skip more bytes than available."); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -188,38 +203,36 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var index = _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { return block.Array[index]; } - return PeekMultiBlock(); + return wasLastBlock ? -1 : PeekMultiBlock(); } [MethodImpl(MethodImplOptions.NoInlining)] private int PeekMultiBlock() { var block = _block; - var wasLastBlock = block.Next == null; do { - int index; - if (wasLastBlock) - { - return -1; - } - else - { - block = block.Next; - index = block.Start; - } + block = block.Next; + var index = block.Start; - wasLastBlock = block.Next == null; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { return block.Array[index]; } + if (wasLastBlock) + { + return -1; + } } while (true); } @@ -229,60 +242,57 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { longValue = 0; - if (_block == null) + var block = _block; + if (block == null) { return false; } - var blockBytes = _block.End - _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; + var blockBytes = block.End - _index; if (blockBytes >= sizeof(ulong)) { - longValue = *(ulong*)(_block.DataFixedPtr + _index); + longValue = *(ulong*)(block.DataFixedPtr + _index); return true; } - return TryPeekLongMultiBlock(ref longValue, blockBytes); + // wasLastBlock ? false : TryPeekLongMultiBlock(ref longValue, blockBytes); + return !wasLastBlock && TryPeekLongMultiBlock(ref longValue, blockBytes); } [MethodImpl(MethodImplOptions.NoInlining)] private unsafe bool TryPeekLongMultiBlock(ref ulong longValue, int blockBytes) { - var wasLastBlock = _block.Next == null; - if (wasLastBlock) + // Each block will be filled with at least 2048 bytes before the Next pointer is set, so a long + // will cross at most one block boundary assuming there are at least 8 bytes following the iterator. + var nextBytes = sizeof(ulong) - blockBytes; + + var block = _block; + if (block.Next.End - block.Next.Start < nextBytes) { return false; } + + var nextLong = *(ulong*)(block.Next.DataFixedPtr + block.Next.Start); + + if (blockBytes == 0) + { + // This case can not fall through to the else block since that would cause a 64-bit right shift + // on blockLong which is equivalent to no shift at all instead of shifting in all zeros. + // https://msdn.microsoft.com/en-us/library/xt18et0d.aspx + longValue = nextLong; + } else { - // Each block will be filled with at least 2048 bytes before the Next pointer is set, so a long - // will cross at most one block boundary assuming there are at least 8 bytes following the iterator. - var nextBytes = sizeof(ulong) - blockBytes; + var blockLong = *(ulong*)(block.DataFixedPtr + block.End - sizeof(ulong)); - if (_block.Next.End - _block.Next.Start < nextBytes) - { - return false; - } - - var nextLong = *(ulong*)(_block.Next.DataFixedPtr + _block.Next.Start); - - if (blockBytes == 0) - { - // This case can not fall through to the else block since that would cause a 64-bit right shift - // on blockLong which is equivalent to no shift at all instead of shifting in all zeros. - // https://msdn.microsoft.com/en-us/library/xt18et0d.aspx - longValue = nextLong; - } - else - { - var blockLong = *(ulong*)(_block.DataFixedPtr + _block.End - sizeof(ulong)); - - // Ensure that the right shift has a ulong operand so a logical shift is performed. - longValue = (blockLong >> nextBytes * 8) | (nextLong << blockBytes * 8); - } - - return true; + // Ensure that the right shift has a ulong operand so a logical shift is performed. + longValue = (blockLong >> nextBytes * 8) | (nextLong << blockBytes * 8); } + + return true; } public int Seek(byte byte0) @@ -799,6 +809,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var block = _block; var index = _index; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { _index = index + 1; @@ -806,28 +818,21 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return true; } - return PutMultiBlock(data); + // wasLastBlock ? false : PutMultiBlock(data); + return !wasLastBlock && PutMultiBlock(data); } [MethodImpl(MethodImplOptions.NoInlining)] private bool PutMultiBlock(byte data) { var block = _block; - var wasLastBlock = block.Next == null; do { - int index; - if (wasLastBlock) - { - return false; - } - else - { - block = block.Next; - index = block.Start; - } + block = block.Next; + var index = block.Start; - wasLastBlock = block.Next == null; + // Always set wasLastBlock before checking .End to avoid race which may cause data loss + var wasLastBlock = block.Next == null; if (index < block.End) { @@ -836,6 +841,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure block.Array[index] = data; break; } + if (wasLastBlock) + { + return false; + } } while (true); return true; @@ -1123,10 +1132,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } if (end.Block == Block) { - return Constants.UTF8.GetString(Block.Array, Index, end.Index - Index); + return Encoding.UTF8.GetString(Block.Array, Index, end.Index - Index); } - var decoder = Constants.UTF8.GetDecoder(); + var decoder = Encoding.UTF8.GetDecoder(); var length = GetLength(end); var charLength = length; From 8ce47fb8b64ace860e0d0167eba89dd45f0dcad5 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 04:33:08 +0000 Subject: [PATCH 14/20] Move perf, share code --- KestrelHttpServer.sln | 9 ++-- global.json | 2 +- .../configs/DefaultConfig.cs | 38 -------------- .../helpers/MockApplicationErrorLogger.cs | 51 ------------------- .../helpers/MockConnectionControl.cs | 34 ------------- .../helpers/MockKestrelTrace.cs | 33 ------------ .../helpers/SocketInputExtensions.cs | 37 -------------- .../project.json | 29 ----------- .../Properties/AssemblyInfo.cs | 1 + ...spNetCore.Server.Kestrel.Performance.xproj | 0 .../Program.cs | 4 ++ .../Readme.md | 4 +- .../RequestParsing.cs | 21 ++------ .../columns/RpsColumn.cs | 0 .../configs/CoreConfig.cs | 1 + .../global.json | 3 ++ .../project.json | 42 +++++++++++++++ .../TestHelpers => shared}/MockConnection.cs | 5 +- .../SocketInputExtensions.cs | 0 19 files changed, 68 insertions(+), 246 deletions(-) delete mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs delete mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs delete mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs delete mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs delete mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs delete mode 100644 perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json rename {perf => test}/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj (100%) rename {perf => test}/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs (90%) rename {perf => test}/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md (60%) rename {perf => test}/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs (90%) rename {perf => test}/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs (100%) rename {perf => test}/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs (95%) create mode 100644 test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json create mode 100644 test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json rename test/{Microsoft.AspNetCore.Server.KestrelTests/TestHelpers => shared}/MockConnection.cs (90%) rename test/{Microsoft.AspNetCore.Server.KestrelTests/TestHelpers => shared}/SocketInputExtensions.cs (100%) diff --git a/KestrelHttpServer.sln b/KestrelHttpServer.sln index e6dca6eac9..d78ceec5ad 100644 --- a/KestrelHttpServer.sln +++ b/KestrelHttpServer.sln @@ -37,7 +37,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{0EF2AC test\shared\DummyApplication.cs = test\shared\DummyApplication.cs test\shared\HttpClientSlim.cs = test\shared\HttpClientSlim.cs test\shared\LifetimeNotImplemented.cs = test\shared\LifetimeNotImplemented.cs + test\shared\MockConnection.cs = test\shared\MockConnection.cs + test\shared\MockFrameControl.cs = test\shared\MockFrameControl.cs test\shared\MockSystemClock.cs = test\shared\MockSystemClock.cs + test\shared\SocketInputExtensions.cs = test\shared\SocketInputExtensions.cs test\shared\TestApplicationErrorLogger.cs = test\shared\TestApplicationErrorLogger.cs test\shared\TestConnection.cs = test\shared\TestConnection.cs test\shared\TestKestrelTrace.cs = test\shared\TestKestrelTrace.cs @@ -45,9 +48,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{0EF2AC test\shared\TestServiceContext.cs = test\shared\TestServiceContext.cs EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "perf", "perf", "{21B17FBB-5A58-42A8-8473-43160509A9FF}" -EndProject -Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.AspNetCore.Server.Kestrel.Performance", "perf\Microsoft.AspNetCore.Server.Kestrel.Performance\Microsoft.AspNetCore.Server.Kestrel.Performance.xproj", "{70567566-524C-4B67-9B59-E5C206D6C2EB}" +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.AspNetCore.Server.Kestrel.Performance", "test\Microsoft.AspNetCore.Server.Kestrel.Performance\Microsoft.AspNetCore.Server.Kestrel.Performance.xproj", "{70567566-524C-4B67-9B59-E5C206D6C2EB}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -100,6 +101,6 @@ Global {5F64B3C3-0C2E-431A-B820-A81BBFC863DA} = {2D5D5227-4DBD-499A-96B1-76A36B03B750} {9559A5F1-080C-4909-B6CF-7E4B3DC55748} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} {0EF2ACDF-012F-4472-A13A-4272419E2903} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} - {70567566-524C-4B67-9B59-E5C206D6C2EB} = {21B17FBB-5A58-42A8-8473-43160509A9FF} + {70567566-524C-4B67-9B59-E5C206D6C2EB} = {D3273454-EA07-41D2-BF0B-FCC3675C2483} EndGlobalSection EndGlobal diff --git a/global.json b/global.json index c0415c19af..d9b4ed63ae 100644 --- a/global.json +++ b/global.json @@ -1,3 +1,3 @@ { - "projects": ["src", "perf"] + "projects": [ "src" ] } diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs deleted file mode 100644 index 6b2670302c..0000000000 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/DefaultConfig.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Validators; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - public class DefaultConfig : ManualConfig - { - public DefaultConfig() - { - Add(JitOptimizationsValidator.FailOnError); - Add(new RpsColumn()); - - Add(Job.Default. - With(Platform.X64). - With(Jit.RyuJit). - With(BenchmarkDotNet.Environments.Runtime.Clr). - With(new GcMode() { Server = true }). - With(RunStrategy.Throughput). - WithLaunchCount(3). - WithWarmupCount(5). - WithTargetCount(10)); - - Add(Job.Default. - With(BenchmarkDotNet.Environments.Runtime.Core). - With(new GcMode() { Server = true }). - With(RunStrategy.Throughput). - WithLaunchCount(3). - WithWarmupCount(5). - WithTargetCount(10)); - } - } -} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs deleted file mode 100644 index c126eba79c..0000000000 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockApplicationErrorLogger.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.AspNetCore.Server.Kestrel.Internal; -using Microsoft.Extensions.Logging; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - public class MockApplicationErrorLogger : ILogger - { - // Application errors are logged using 13 as the eventId. - private const int ApplicationErrorEventId = 13; - - public List Messages { get; } = new List(); - - public int TotalErrorsLogged => Messages.Count(message => message.LogLevel == LogLevel.Error); - - public int CriticalErrorsLogged => Messages.Count(message => message.LogLevel == LogLevel.Critical); - - public int ApplicationErrorsLogged => Messages.Count(message => message.EventId.Id == ApplicationErrorEventId); - - public IDisposable BeginScope(TState state) - { - return new Disposable(() => { }); - } - - public bool IsEnabled(LogLevel logLevel) - { - return true; - } - - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) - { -#if false - Console.WriteLine($"Log {logLevel}[{eventId}]: {formatter(state, exception)} {exception?.Message}"); -#endif - - Messages.Add(new LogMessage { LogLevel = logLevel, EventId = eventId, Exception = exception }); - } - - public class LogMessage - { - public LogLevel LogLevel { get; set; } - public EventId EventId { get; set; } - public Exception Exception { get; set; } - } - } -} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs deleted file mode 100644 index b3587d81db..0000000000 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockConnectionControl.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - public class MockConnectionControl : IConnectionControl - { - public void CancelTimeout() - { - } - - public void End(ProduceEndType endType) - { - } - - public void Pause() - { - } - - public void ResetTimeout(long milliseconds, TimeoutAction timeoutAction) - { - } - - public void Resume() - { - } - - public void SetTimeout(long milliseconds, TimeoutAction timeoutAction) - { - } - } -} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs deleted file mode 100644 index 5d5bf38201..0000000000 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/MockKestrelTrace.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using Microsoft.AspNetCore.Server.Kestrel.Internal; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - public class MockKestrelTrace : KestrelTrace - { - public MockKestrelTrace() : this(new MockApplicationErrorLogger()) - { - } - - public MockKestrelTrace(MockApplicationErrorLogger testLogger) : base(testLogger) - { - Logger = testLogger; - } - - public MockApplicationErrorLogger Logger { get; private set; } - - public override void ConnectionRead(string connectionId, int count) - { - } - - public override void ConnectionWrite(string connectionId, int count) - { - } - - public override void ConnectionWriteCallback(string connectionId, int status) - { - } - } -} \ No newline at end of file diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs deleted file mode 100644 index af26c2994c..0000000000 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/helpers/SocketInputExtensions.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - public static class SocketInputExtensions - { - public static void IncomingData(this SocketInput input, byte[] buffer, int offset, int count) - { - var bufferIndex = offset; - var remaining = count; - - while (remaining > 0) - { - var block = input.IncomingStart(); - - var bytesLeftInBlock = block.Data.Offset + block.Data.Count - block.End; - var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock; - - Buffer.BlockCopy(buffer, bufferIndex, block.Array, block.End, bytesToCopy); - - bufferIndex += bytesToCopy; - remaining -= bytesToCopy; - - input.IncomingComplete(bytesToCopy, null); - } - } - - public static void IncomingFin(this SocketInput input) - { - input.IncomingComplete(0, null); - } - } -} diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json b/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json deleted file mode 100644 index 38bb58ec76..0000000000 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "version": "1.0.0-*", - "buildOptions": { - "emitEntryPoint": true - }, - - "dependencies": { - "BenchmarkDotNet": "0.10.0", - "Microsoft.AspNetCore.Server.Kestrel": "1.2.0-*" - }, - - "frameworks": { - "net46": {}, - "netcoreapp1.0": { - "dependencies": { - "Microsoft.NETCore.App": { - "version": "1.0.1-*", - "type": "platform" - } - } - } - }, - - "runtimeOptions": { - "configProperties": { - "System.GC.Server": true - } - } -} \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs index 8989a4b649..86b8eff609 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Properties/AssemblyInfo.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.AspNetCore.Server.Kestrel.FunctionalTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] [assembly: InternalsVisibleTo("Microsoft.AspNetCore.Server.KestrelTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] +[assembly: InternalsVisibleTo("Microsoft.AspNetCore.Server.Kestrel.Performance, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] [assembly: AssemblyMetadata("Serviceable", "True")] [assembly: NeutralResourcesLanguage("en-us")] [assembly: AssemblyCompany("Microsoft Corporation.")] diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj similarity index 100% rename from perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/Microsoft.AspNetCore.Server.Kestrel.Performance.xproj diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs similarity index 90% rename from perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs index 688771f622..b627755d76 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs @@ -2,7 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Properties; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md similarity index 60% rename from perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md index 4088c38007..b98f36ff5c 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Readme.md @@ -2,10 +2,10 @@ To run a specific benchmark add it as parameter ``` -dotnet run -c Release RequestParsing +dotnet run RequestParsing ``` To run all use `All` as parameter ``` -dotnet run -c Release All +dotnet run All ``` Using no parameter will list all available benchmarks \ No newline at end of file diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs similarity index 90% rename from perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs index b1bdbe16f9..7388c547a2 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs @@ -8,6 +8,8 @@ using BenchmarkDotNet.Attributes; using Microsoft.AspNetCore.Server.Kestrel.Internal; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure; +using Microsoft.AspNetCore.Server.KestrelTests.TestHelpers; +using Microsoft.AspNetCore.Testing; using RequestLineStatus = Microsoft.AspNetCore.Server.Kestrel.Internal.Http.Frame.RequestLineStatus; namespace Microsoft.AspNetCore.Server.Kestrel.Performance @@ -165,26 +167,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance [Setup] public void Setup() { - Trace = new KestrelTrace(new MockKestrelTrace()); + Trace = new KestrelTrace(new TestKestrelTrace()); ThreadPool = new LoggingThreadPool(Trace); MemoryPool = new MemoryPool(); SocketInput = new SocketInput(MemoryPool, ThreadPool); - var serviceContext = new ServiceContext - { - DateHeaderValueManager = new DateHeaderValueManager(), - ServerOptions = new KestrelServerOptions(), - Log = Trace - }; - var listenerContext = new ListenerContext(serviceContext) - { - ServerAddress = ServerAddress.FromUrl("http://localhost:5000") - }; - var connectionContext = new ConnectionContext(listenerContext) - { - ConnectionControl = new MockConnectionControl(), - SocketInput = SocketInput - }; + var connectionContext = new MockConnection(new KestrelServerOptions()); + connectionContext.SocketInput = SocketInput; Frame = new Frame(application: null, context: connectionContext); } diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs similarity index 100% rename from perf/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/columns/RpsColumn.cs diff --git a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs similarity index 95% rename from perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs index 72056a28b4..efdf192205 100644 --- a/perf/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/configs/CoreConfig.cs @@ -17,6 +17,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance Add(Job.Default. With(BenchmarkDotNet.Environments.Runtime.Core). + WithRemoveOutliers(false). With(new GcMode() { Server = true }). With(RunStrategy.Throughput). WithLaunchCount(3). diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json new file mode 100644 index 0000000000..33f0f54e92 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/global.json @@ -0,0 +1,3 @@ +{ + "projects": [ "..\\" ] +} diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json new file mode 100644 index 0000000000..a74be5fefb --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/project.json @@ -0,0 +1,42 @@ +{ + "version": "1.0.0-*", + "dependencies": { + "BenchmarkDotNet": "0.10.0", + "Microsoft.AspNetCore.Server.Kestrel": "1.2.0-*" + }, + "frameworks": { + "netcoreapp1.0": { + "dependencies": { + "Microsoft.NETCore.App": { + "version": "1.0.1-*", + "type": "platform" + } + } + } + }, + "buildOptions": { + "emitEntryPoint": true, + "compile": { + "include": [ + "../shared/SocketInputExtensions.cs", + "../shared/TestKestrelTrace.cs", + "../shared/TestApplicationErrorLogger.cs", + "../shared/MockConnection.cs" + ] + }, + "keyFile": "../../tools/Key.snk", + "copyToOutput": { + "include": "TestResources/testCert.pfx" + } + }, + "runtimeOptions": { + "configProperties": { + "System.GC.Server": true + } + }, + "publishOptions": { + "include": [ + "TestResources/testCert.pfx" + ] + } +} \ No newline at end of file diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/MockConnection.cs b/test/shared/MockConnection.cs similarity index 90% rename from test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/MockConnection.cs rename to test/shared/MockConnection.cs index ee781a3935..a782f51ffd 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/MockConnection.cs +++ b/test/shared/MockConnection.cs @@ -18,7 +18,10 @@ namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers { ConnectionControl = this; RequestAbortedSource = new CancellationTokenSource(); - ListenerContext = new ListenerContext(new ServiceContext { ServerOptions = options }); + ListenerContext = new ListenerContext(new ServiceContext {ServerOptions = options}) + { + ServerAddress = ServerAddress.FromUrl("http://localhost:5000") + }; } public override void Abort(Exception error = null) diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/SocketInputExtensions.cs b/test/shared/SocketInputExtensions.cs similarity index 100% rename from test/Microsoft.AspNetCore.Server.KestrelTests/TestHelpers/SocketInputExtensions.cs rename to test/shared/SocketInputExtensions.cs From 2011a27bde7f2363b231f607c33355a84a3e9ab2 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 05:10:16 +0000 Subject: [PATCH 15/20] Use ternary op rather than lazy and --- .../Internal/Infrastructure/MemoryPoolIterator.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 7c7e0865e8..97b71e3a2d 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -258,8 +258,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return true; } - // wasLastBlock ? false : TryPeekLongMultiBlock(ref longValue, blockBytes); - return !wasLastBlock && TryPeekLongMultiBlock(ref longValue, blockBytes); + return wasLastBlock ? false : TryPeekLongMultiBlock(ref longValue, blockBytes); } [MethodImpl(MethodImplOptions.NoInlining)] @@ -818,8 +817,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return true; } - // wasLastBlock ? false : PutMultiBlock(data); - return !wasLastBlock && PutMultiBlock(data); + return wasLastBlock ? false : PutMultiBlock(data); } [MethodImpl(MethodImplOptions.NoInlining)] From ef5ad3deeaaec690cf0ee315e11d75b35f220ec7 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 12:37:11 +0000 Subject: [PATCH 16/20] defaullt(MemoryPoolIterator) test coverage --- .../Infrastructure/MemoryPoolIterator.cs | 81 ++++++++++--------- .../MemoryPoolIteratorTests.cs | 28 +++++++ 2 files changed, 70 insertions(+), 39 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 97b71e3a2d..4cceb50bf7 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -136,9 +136,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public void Skip(int bytesToSkip) { var block = _block; - if (block == null) + if (block == null && bytesToSkip > 0) { - return; + ThrowInvalidOperationException_SkipMoreThanAvailable(); } // Always set wasLastBlock before checking .End to avoid race which may cause data loss @@ -307,12 +307,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { bytesScanned = 0; - if (IsDefault || limit <= 0) + var block = _block; + if (block == null || limit <= 0) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; @@ -413,12 +413,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure byte byte0, ref MemoryPoolIterator limit) { - if (IsDefault) + var block = _block; + if (block == null) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; @@ -520,12 +520,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure byte byte1, ref MemoryPoolIterator limit) { - if (IsDefault) + var block = _block; + if (block == null) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; @@ -650,12 +650,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure byte byte2, ref MemoryPoolIterator limit) { - if (IsDefault) + var block = _block; + if (block == null) { return -1; } - var block = _block; var index = _index; var wasLastBlock = block.Next == null; var following = block.End - index; @@ -800,12 +800,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool Put(byte data) { - if (_block == null) + var block = _block; + if (block == null) { return false; } - var block = _block; var index = _index; // Always set wasLastBlock before checking .End to avoid race which may cause data loss @@ -851,12 +851,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetLength(MemoryPoolIterator end) { - if (IsDefault || end.IsDefault) + var block = _block; + if (block == null || end.IsDefault) { return -1; } - if (_block == end._block) + if (block == end._block) { return end._index - _index; } @@ -894,13 +895,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public MemoryPoolIterator CopyTo(byte[] array, int offset, int count, out int actual) { - if (IsDefault) + var block = _block; + if (block == null) { actual = 0; return this; } - var block = _block; var index = _index; var remaining = count; while (true) @@ -955,17 +956,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public void CopyFrom(byte[] data, int offset, int count) { - if (IsDefault) + var block = _block; + if (block == null) { return; } - Debug.Assert(_block != null); - Debug.Assert(_block.Next == null); - Debug.Assert(_block.End == _index); + Debug.Assert(block.Next == null); + Debug.Assert(block.End == _index); - var pool = _block.Pool; - var block = _block; + var pool = block.Pool; var blockIndex = _index; var bufferIndex = offset; @@ -1002,17 +1002,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public unsafe void CopyFromAscii(string data) { - if (IsDefault) + var block = _block; + if (block == null) { return; } - Debug.Assert(_block != null); - Debug.Assert(_block.Next == null); - Debug.Assert(_block.End == _index); + Debug.Assert(block.Next == null); + Debug.Assert(block.End == _index); - var pool = _block.Pool; - var block = _block; + var pool = block.Pool; var blockIndex = _index; var length = data.Length; @@ -1068,7 +1067,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public unsafe string GetAsciiString(ref MemoryPoolIterator end) { - if (IsDefault || end.IsDefault) + var block = _block; + if (block == null || end.IsDefault) { return null; } @@ -1080,8 +1080,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return null; } - var inputOffset = Index; - var block = Block; + var inputOffset = _index; var asciiString = new string('\0', length); @@ -1124,13 +1123,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure public string GetUtf8String(ref MemoryPoolIterator end) { - if (IsDefault || end.IsDefault) + var block = _block; + if (block == null || end.IsDefault) { return default(string); } - if (end.Block == Block) + + var index = _index; + if (end.Block == block) { - return Encoding.UTF8.GetString(Block.Array, Index, end.Index - Index); + return Encoding.UTF8.GetString(block.Array, index, end.Index - index); } var decoder = Encoding.UTF8.GetDecoder(); @@ -1141,8 +1143,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var chars = new char[charLength]; var charIndex = 0; - var block = Block; - var index = Index; var remaining = length; while (true) { @@ -1204,13 +1204,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure [MethodImpl(MethodImplOptions.AggressiveInlining)] public ArraySegment GetArraySegment(MemoryPoolIterator end) { - if (IsDefault || end.IsDefault) + var block = _block; + if (block == null || end.IsDefault) { return default(ArraySegment); } - if (end.Block == Block) + + var index = _index; + if (end.Block == block) { - return new ArraySegment(Block.Array, Index, end.Index - Index); + return new ArraySegment(block.Array, index, end.Index - index); } return GetArraySegmentMultiBlock(ref end); diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 156832ca0f..f3ec36afe2 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -966,6 +966,34 @@ namespace Microsoft.AspNetCore.Server.KestrelTests } } + [Fact] + public void EmptyIteratorBehaviourIsValid() + { + const byte byteCr = (byte) '\n'; + ulong longValue; + var end = default(MemoryPoolIterator); + + Assert.False(default(MemoryPoolIterator).TryPeekLong(out longValue)); + Assert.False(default(MemoryPoolIterator).Put(byteCr)); + Assert.Null(default(MemoryPoolIterator).GetAsciiString(ref end)); + Assert.Null(default(MemoryPoolIterator).GetUtf8String(ref end)); + // Assert.Equal doesn't work for default(ArraySegments) + Assert.True(default(MemoryPoolIterator).GetArraySegment(end).Equals(default(ArraySegment))); + Assert.True(default(MemoryPoolIterator).IsDefault); + Assert.True(default(MemoryPoolIterator).IsEnd); + Assert.Equal(default(MemoryPoolIterator).Take(), -1); + Assert.Equal(default(MemoryPoolIterator).Peek(), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr, ref end), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr, byteCr), -1); + Assert.Equal(default(MemoryPoolIterator).Seek(byteCr, byteCr, byteCr), -1); + + default(MemoryPoolIterator).CopyFrom(default(ArraySegment)); + default(MemoryPoolIterator).CopyFromAscii(""); + Assert.Equal(default(MemoryPoolIterator).GetLength(end), -1); + Assert.ThrowsAny(() => default(MemoryPoolIterator).Skip(1)); + } + [Theory] [InlineData("a", "a", 1)] [InlineData("ab", "a...", 1)] From dc90dd164967848abe7991f99181318429afdabf Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 13:21:37 +0000 Subject: [PATCH 17/20] Add GetArraySegment test --- .../MemoryPoolIteratorTests.cs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index f3ec36afe2..07a8bfa666 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -994,6 +994,65 @@ namespace Microsoft.AspNetCore.Server.KestrelTests Assert.ThrowsAny(() => default(MemoryPoolIterator).Skip(1)); } + [Fact] + public void TestGetArraySegment() + { + MemoryPoolBlock block0 = null; + MemoryPoolBlock block1 = null; + + var byteRange = Enumerable.Range(1, 127).Select(x => (byte)x).ToArray(); + try + { + // Arrange + block0 = _pool.Lease(); + block1 = _pool.Lease(); + + block0.GetIterator().CopyFrom(byteRange); + block1.GetIterator().CopyFrom(byteRange); + + block0.Next = block1; + + var begin = block0.GetIterator(); + var end0 = begin; + var end1 = begin; + + end0.Skip(byteRange.Length); + end1.Skip(byteRange.Length * 2); + + // Act + var as0 = begin.GetArraySegment(end0); + var as1 = begin.GetArraySegment(end1); + + // Assert + Assert.Equal(as0.Count, byteRange.Length); + Assert.Equal(as1.Count, byteRange.Length * 2); + + for (var i = 1; i < byteRange.Length; i++) + { + var asb0 = as0.Array[i + as0.Offset]; + var asb1 = as1.Array[i + as1.Offset]; + var b = byteRange[i]; + + Assert.Equal(asb0, b); + Assert.Equal(asb1, b); + } + + for (var i = 1 + byteRange.Length; i < byteRange.Length * 2; i++) + { + var asb1 = as1.Array[i + as1.Offset]; + var b = byteRange[i - byteRange.Length]; + + Assert.Equal(asb1, b); + } + + } + finally + { + if (block0 != null) _pool.Return(block0); + if (block1 != null) _pool.Return(block1); + } + } + [Theory] [InlineData("a", "a", 1)] [InlineData("ab", "a...", 1)] From ba0b7cc55322759094a3170bcb2ee47f35d6fe4e Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 13:55:11 +0000 Subject: [PATCH 18/20] Add 100% coverage Take tests --- .../MemoryPoolIteratorTests.cs | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 07a8bfa666..1416f652df 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -1053,6 +1053,129 @@ namespace Microsoft.AspNetCore.Server.KestrelTests } } + [Fact] + public void TestTake() + { + MemoryPoolBlock block0 = null; + MemoryPoolBlock block1 = null; + MemoryPoolBlock block2 = null; + MemoryPoolBlock emptyBlock0 = null; + MemoryPoolBlock emptyBlock1 = null; + + var byteRange = Enumerable.Range(1, 127).Select(x => (byte)x).ToArray(); + try + { + // Arrange + block0 = _pool.Lease(); + block1 = _pool.Lease(); + block2 = _pool.Lease(); + emptyBlock0 = _pool.Lease(); + emptyBlock1 = _pool.Lease(); + + block0.GetIterator().CopyFrom(byteRange); + block1.GetIterator().CopyFrom(byteRange); + block2.GetIterator().CopyFrom(byteRange); + + var begin = block0.GetIterator(); + + // Single block + for (var i = 0; i < byteRange.Length; i++) + { + var t = begin.Take(); + var b = byteRange[i]; + + Assert.Equal(t, b); + } + + Assert.Equal(begin.Take(), -1); + + // Dual block + block0.Next = block1; + begin = block0.GetIterator(); + + for (var block = 0; block < 2; block++) + { + for (var i = 0; i < byteRange.Length; i++) + { + var t = begin.Take(); + var b = byteRange[i]; + + Assert.Equal(t, b); + } + } + + Assert.Equal(begin.Take(), -1); + + // Multi block + block1.Next = emptyBlock0; + emptyBlock0.Next = emptyBlock1; + emptyBlock1.Next = block2; + begin = block0.GetIterator(); + + for (var block = 0; block < 3; block++) + { + for (var i = 0; i < byteRange.Length; i++) + { + var t = begin.Take(); + var b = byteRange[i]; + + Assert.Equal(t, b); + } + } + + Assert.Equal(begin.Take(), -1); + } + finally + { + if (block0 != null) _pool.Return(block0); + if (block1 != null) _pool.Return(block1); + if (block2 != null) _pool.Return(block2); + if (emptyBlock0 != null) _pool.Return(emptyBlock0); + if (emptyBlock1 != null) _pool.Return(emptyBlock1); + } + } + + [Fact] + public void TestTakeEmptyBlocks() + { + MemoryPoolBlock emptyBlock0 = null; + MemoryPoolBlock emptyBlock1 = null; + MemoryPoolBlock emptyBlock2 = null; + try + { + // Arrange + emptyBlock0 = _pool.Lease(); + emptyBlock1 = _pool.Lease(); + emptyBlock2 = _pool.Lease(); + + var beginEmpty = emptyBlock0.GetIterator(); + + // Assert + + // No blocks + Assert.Equal(default(MemoryPoolIterator).Take(), -1); + + // Single empty block + Assert.Equal(beginEmpty.Take(), -1); + + // Dual empty block + emptyBlock0.Next = emptyBlock1; + beginEmpty = emptyBlock0.GetIterator(); + Assert.Equal(beginEmpty.Take(), -1); + + // Multi empty block + emptyBlock1.Next = emptyBlock2; + beginEmpty = emptyBlock0.GetIterator(); + Assert.Equal(beginEmpty.Take(), -1); + } + finally + { + if (emptyBlock0 != null) _pool.Return(emptyBlock0); + if (emptyBlock1 != null) _pool.Return(emptyBlock1); + if (emptyBlock2 != null) _pool.Return(emptyBlock2); + } + } + [Theory] [InlineData("a", "a", 1)] [InlineData("ab", "a...", 1)] From 9ec4d88fbe7da35305d4609de66ed17d3f618e4c Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 19 Nov 2016 14:15:37 +0000 Subject: [PATCH 19/20] Improve Skip coverage --- .../MemoryPoolIteratorTests.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 1416f652df..0cb46de20d 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -518,20 +518,25 @@ namespace Microsoft.AspNetCore.Server.KestrelTests public void SkipThrowsWhenSkippingMoreBytesThanAvailableInMultipleBlocks() { // Arrange - var block = _pool.Lease(); - block.End += 3; + var firstBlock = _pool.Lease(); + firstBlock.End += 3; - var nextBlock = _pool.Lease(); - nextBlock.End += 2; - block.Next = nextBlock; + var middleBlock = _pool.Lease(); + middleBlock.End += 1; + firstBlock.Next = middleBlock; - var scan = block.GetIterator(); + var finalBlock = _pool.Lease(); + finalBlock.End += 2; + middleBlock.Next = finalBlock; + + var scan = firstBlock.GetIterator(); // Act/Assert Assert.ThrowsAny(() => scan.Skip(8)); - _pool.Return(block); - _pool.Return(nextBlock); + _pool.Return(firstBlock); + _pool.Return(middleBlock); + _pool.Return(finalBlock); } [Theory] From 2eba4017c1aa2f969e5a081f928c9c6a9df95327 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 21 Nov 2016 22:18:17 +0000 Subject: [PATCH 20/20] MemoryPoolIterator feedback --- global.json | 2 +- .../Internal/Http/Frame.cs | 2 +- .../Infrastructure/MemoryPoolIterator.cs | 33 ++++++++++++++----- .../RequestParsing.cs | 1 - .../MemoryPoolIteratorTests.cs | 6 ++-- .../MessageBodyTests.cs | 2 +- test/shared/MockConnection.cs | 2 +- test/shared/SocketInputExtensions.cs | 2 +- 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/global.json b/global.json index d9b4ed63ae..983ba0401e 100644 --- a/global.json +++ b/global.json @@ -1,3 +1,3 @@ { - "projects": [ "src" ] + "projects": ["src"] } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs index 59891a7355..2b7f56e882 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs @@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { public abstract partial class Frame : IFrameControl { - // byte consts don't have a data type annotation so we pre-cast them + // byte types don't have a data type annotation so we pre-cast them; to avoid in-place casts private const byte ByteCR = (byte)'\r'; private const byte ByteLF = (byte)'\n'; private const byte ByteColon = (byte)':'; diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 4cceb50bf7..b1b45eb6e8 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -774,10 +774,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int LocateFirstFoundByte(Vector byteEquals) { - var vector64 = Vector.AsVectorInt64(byteEquals); - long longValue = 0; + var vector64 = Vector.AsVectorUInt64(byteEquals); + ulong longValue = 0; var i = 0; - for (; i < Vector.Count; i++) + // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001 + for (; i < Vector.Count; i++) { longValue = vector64[i]; if (longValue == 0) continue; @@ -785,7 +786,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } // Flag least significant power of two bit - var powerOfTwoFlag = (ulong)(longValue ^ (longValue - 1)); + var powerOfTwoFlag = (longValue ^ (longValue - 1)); // Shift all powers of two into the high byte and extract var foundByteIndex = (int)((powerOfTwoFlag * _xorPowerOfTwoToHighByte) >> 57); // Single LEA instruction with jitted const (using function result) @@ -803,7 +804,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure var block = _block; if (block == null) { - return false; + ThrowInvalidOperationException_PutPassedEndOfBlock(); } var index = _index; @@ -817,7 +818,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return true; } - return wasLastBlock ? false : PutMultiBlock(data); + if (wasLastBlock) + { + ThrowInvalidOperationException_PutPassedEndOfBlock(); + } + + return PutMultiBlock(data); } [MethodImpl(MethodImplOptions.NoInlining)] @@ -841,6 +847,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } if (wasLastBlock) { + ThrowInvalidOperationException_PutPassedEndOfBlock(); return false; } } while (true); @@ -848,13 +855,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return true; } + private static void ThrowInvalidOperationException_PutPassedEndOfBlock() + { + throw new InvalidOperationException("Attempted to put passed end of block."); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetLength(MemoryPoolIterator end) { var block = _block; if (block == null || end.IsDefault) { - return -1; + ThrowInvalidOperationException_GetLengthNullBlock(); } if (block == end._block) @@ -865,6 +877,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure return GetLengthMultiBlock(ref end); } + private static void ThrowInvalidOperationException_GetLengthNullBlock() + { + throw new InvalidOperationException("Attempted GetLength of non existent block."); + } + [MethodImpl(MethodImplOptions.NoInlining)] public int GetLengthMultiBlock(ref MemoryPoolIterator end) { @@ -1234,7 +1251,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure // Vector .ctor doesn't become an intrinsic due to detection issue // However this does cause it to become an intrinsic (with additional multiply and reg->reg copy) // https://github.com/dotnet/coreclr/issues/7459#issuecomment-253965670 - return Vector.AsVectorByte(new Vector(vectorByte * 0x0101010101010101ul)); + return Vector.AsVectorByte(new Vector(vectorByte * 0x01010101u)); } } diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs index 7388c547a2..5ae06776f5 100644 --- a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/RequestParsing.cs @@ -8,7 +8,6 @@ using BenchmarkDotNet.Attributes; using Microsoft.AspNetCore.Server.Kestrel.Internal; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure; -using Microsoft.AspNetCore.Server.KestrelTests.TestHelpers; using Microsoft.AspNetCore.Testing; using RequestLineStatus = Microsoft.AspNetCore.Server.Kestrel.Internal.Http.Frame.RequestLineStatus; diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 0cb46de20d..1db21257a3 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -154,7 +154,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests } // Can't put anything by the end - Assert.False(head.Put(0xFF)); + Assert.ThrowsAny(() => head.Put(0xFF)); for (var i = 0; i < 4; ++i) { @@ -979,7 +979,6 @@ namespace Microsoft.AspNetCore.Server.KestrelTests var end = default(MemoryPoolIterator); Assert.False(default(MemoryPoolIterator).TryPeekLong(out longValue)); - Assert.False(default(MemoryPoolIterator).Put(byteCr)); Assert.Null(default(MemoryPoolIterator).GetAsciiString(ref end)); Assert.Null(default(MemoryPoolIterator).GetUtf8String(ref end)); // Assert.Equal doesn't work for default(ArraySegments) @@ -995,7 +994,8 @@ namespace Microsoft.AspNetCore.Server.KestrelTests default(MemoryPoolIterator).CopyFrom(default(ArraySegment)); default(MemoryPoolIterator).CopyFromAscii(""); - Assert.Equal(default(MemoryPoolIterator).GetLength(end), -1); + Assert.ThrowsAny(() => default(MemoryPoolIterator).Put(byteCr)); + Assert.ThrowsAny(() => default(MemoryPoolIterator).GetLength(end)); Assert.ThrowsAny(() => default(MemoryPoolIterator).Skip(1)); } diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs index a5ed8d7c9f..414bb208e3 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MessageBodyTests.cs @@ -10,11 +10,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Server.Kestrel; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; -using Microsoft.AspNetCore.Server.KestrelTests.TestHelpers; using Microsoft.Extensions.Internal; using Moq; using Xunit; using Xunit.Sdk; +using Microsoft.AspNetCore.Testing; namespace Microsoft.AspNetCore.Server.KestrelTests { diff --git a/test/shared/MockConnection.cs b/test/shared/MockConnection.cs index a782f51ffd..e1025c624c 100644 --- a/test/shared/MockConnection.cs +++ b/test/shared/MockConnection.cs @@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel; using Microsoft.AspNetCore.Server.Kestrel.Internal; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; -namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers +namespace Microsoft.AspNetCore.Testing { public class MockConnection : Connection, IDisposable { diff --git a/test/shared/SocketInputExtensions.cs b/test/shared/SocketInputExtensions.cs index 998e0552c5..d6dbbb7e88 100644 --- a/test/shared/SocketInputExtensions.cs +++ b/test/shared/SocketInputExtensions.cs @@ -4,7 +4,7 @@ using System; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; -namespace Microsoft.AspNetCore.Server.KestrelTests.TestHelpers +namespace Microsoft.AspNetCore.Testing { public static class SocketInputExtensions {