diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs index 36c93192b0..8b6dd828a3 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs @@ -40,11 +40,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Http private static readonly byte[] _bytesDate = Encoding.ASCII.GetBytes("Date: "); private static readonly byte[] _bytesEndHeaders = Encoding.ASCII.GetBytes("\r\n\r\n"); - private static readonly Vector _vectorCRs = new Vector((byte)'\r'); - private static readonly Vector _vectorColons = new Vector((byte)':'); - private static readonly Vector _vectorSpaces = new Vector((byte)' '); - private static readonly Vector _vectorQuestionMarks = new Vector((byte)'?'); - private static readonly Vector _vectorPercentages = new Vector((byte)'%'); + private static Vector _vectorCRs = new Vector((byte)'\r'); + private static Vector _vectorColons = new Vector((byte)':'); + private static Vector _vectorSpaces = new Vector((byte)' '); + private static Vector _vectorQuestionMarks = new Vector((byte)'?'); + private static Vector _vectorPercentages = new Vector((byte)'%'); private readonly object _onStartingSync = new Object(); private readonly object _onCompletedSync = new Object(); @@ -714,7 +714,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http var begin = scan; if (!begin.GetKnownMethod(ref scan,out method)) { - if (scan.Seek(_vectorSpaces) == -1) + if (scan.Seek(ref _vectorSpaces) == -1) { return false; } @@ -725,11 +725,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Http begin = scan; var needDecode = false; - var chFound = scan.Seek(_vectorSpaces, _vectorQuestionMarks, _vectorPercentages); + var chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref _vectorPercentages); if (chFound == '%') { needDecode = true; - chFound = scan.Seek(_vectorSpaces, _vectorQuestionMarks); + chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks); } var pathBegin = begin; @@ -739,7 +739,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http if (chFound == '?') { begin = scan; - if (scan.Seek(_vectorSpaces) != ' ') + if (scan.Seek(ref _vectorSpaces) != ' ') { return false; } @@ -753,7 +753,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http if (!begin.GetKnownVersion(ref scan, out httpVersion)) { scan = begin; - if (scan.Seek(_vectorCRs) == -1) + if (scan.Seek(ref _vectorCRs) == -1) { return false; } @@ -843,7 +843,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http while (!scan.IsEnd) { var beginName = scan; - scan.Seek(_vectorColons, _vectorCRs); + scan.Seek(ref _vectorColons, ref _vectorCRs); var endName = scan; chFirst = scan.Take(); @@ -894,7 +894,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http var wrapping = false; while (!scan.IsEnd) { - if (scan.Seek(_vectorCRs) == -1) + if (scan.Seek(ref _vectorCRs) == -1) { // no "\r" in sight, burn used bytes and go back to await more data return false; diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs index 645e0279d7..00490fe6ad 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs @@ -9,6 +9,8 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure { public struct MemoryPoolIterator2 { + private readonly static int _vectorSpan = Vector.Count; + private MemoryPoolBlock2 _block; private int _index; @@ -208,7 +210,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure } } - public int Seek(Vector byte0Vector) + public unsafe int Seek(ref Vector byte0Vector) { if (IsDefault) { @@ -217,58 +219,77 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure var block = _block; var index = _index; - var array = block.Array; + var following = block.End - index; + byte[] array; + var byte0 = byte0Vector[0]; + while (true) { - while (block.End == index) + while (following == 0) { - if (block.Next == null) + var newBlock = block.Next; + if (newBlock == null) { _block = block; _index = index; return -1; } - block = block.Next; - index = block.Start; - array = block.Array; + index = newBlock.Start; + following = newBlock.End - index; + block = newBlock; } - while (block.End != index) + array = block.Array; + while (following > 0) { - var following = block.End - index; - if (following >= Vector.Count) +// Need unit tests to test Vector path +#if !DEBUG + // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 + if (Vector.IsHardwareAccelerated) { - var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - - if (byte0Equals.Equals(Vector.Zero)) +#endif + if (following >= _vectorSpan) { - index += Vector.Count; - continue; - } + var byte0Equals = Vector.Equals(new Vector(array, index), byte0Vector); - _block = block; - _index = index + FindFirstEqualByte(byte0Equals); - return byte0Vector[0]; - } + if (byte0Equals.Equals(Vector.Zero)) + { + following -= _vectorSpan; + index += _vectorSpan; + continue; + } - var byte0 = byte0Vector[0]; - - while (following > 0) - { - if (block.Array[index] == byte0) - { _block = block; - _index = index; + _index = index + FindFirstEqualByte(ref byte0Equals); return byte0; } - following--; - index++; +// Need unit tests to test Vector path +#if !DEBUG } +#endif + fixed (byte* ptr = &block.Array[index]) + { + var pCurrent = ptr; + var pEnd = pCurrent + following; + do + { + if (*pCurrent == byte0) + { + _block = block; + _index = index; + return byte0; + } + pCurrent++; + index++; + } while (pCurrent < pEnd); + } + + following = 0; + break; } } } - public int Seek(Vector byte0Vector, Vector byte1Vector) + public unsafe int Seek(ref Vector byte0Vector, ref Vector byte1Vector) { if (IsDefault) { @@ -277,85 +298,105 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure var block = _block; var index = _index; - var array = block.Array; + var following = block.End - index; + byte[] array; + int byte0Index = int.MaxValue; + int byte1Index = int.MaxValue; + var byte0 = byte0Vector[0]; + var byte1 = byte1Vector[0]; + while (true) { - while (block.End == index) + while (following == 0) { - if (block.Next == null) + var newBlock = block.Next; + if (newBlock == null) { _block = block; _index = index; return -1; } - block = block.Next; - index = block.Start; - array = block.Array; + index = newBlock.Start; + following = newBlock.End - index; + block = newBlock; } - while (block.End != index) + array = block.Array; + while (following > 0) { - var following = block.End - index; - if (following >= Vector.Count) + +// Need unit tests to test Vector path +#if !DEBUG + // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 + if (Vector.IsHardwareAccelerated) { - var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); - int byte0Index = int.MaxValue; - int byte1Index = int.MaxValue; - - if (!byte0Equals.Equals(Vector.Zero)) +#endif + if (following >= _vectorSpan) { - byte0Index = FindFirstEqualByte(byte0Equals); - } - if (!byte1Equals.Equals(Vector.Zero)) - { - byte1Index = FindFirstEqualByte(byte1Equals); - } + var data = new Vector(array, index); + var byte0Equals = Vector.Equals(data, byte0Vector); + var byte1Equals = Vector.Equals(data, byte1Vector); - if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) - { - index += Vector.Count; - continue; - } + if (!byte0Equals.Equals(Vector.Zero)) + { + byte0Index = FindFirstEqualByte(ref byte0Equals); + } + if (!byte1Equals.Equals(Vector.Zero)) + { + byte1Index = FindFirstEqualByte(ref byte1Equals); + } - _block = block; + if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) + { + following -= _vectorSpan; + index += _vectorSpan; + continue; + } - if (byte0Index < byte1Index) - { - _index = index + byte0Index; - return byte0Vector[0]; - } - - _index = index + byte1Index; - return byte1Vector[0]; - } - - byte byte0 = byte0Vector[0]; - byte byte1 = byte1Vector[0]; - - while (following > 0) - { - var byteIndex = block.Array[index]; - if (byteIndex == byte0) - { _block = block; - _index = index; - return byte0; - } - else if (byteIndex == byte1) - { - _block = block; - _index = index; + + if (byte0Index < byte1Index) + { + _index = index + byte0Index; + return byte0; + } + + _index = index + byte1Index; return byte1; } - following--; - index++; +// Need unit tests to test Vector path +#if !DEBUG } +#endif + fixed (byte* ptr = &block.Array[index]) + { + var pCurrent = ptr; + var pEnd = pCurrent + following; + do + { + if (*pCurrent == byte0) + { + _block = block; + _index = index; + return byte0; + } + if (*pCurrent == byte1) + { + _block = block; + _index = index; + return byte1; + } + pCurrent++; + index++; + } while (pCurrent != pEnd); + } + + following = 0; + break; } } } - public int Seek(Vector byte0Vector, Vector byte1Vector, Vector byte2Vector) + public unsafe int Seek(ref Vector byte0Vector, ref Vector byte1Vector, ref Vector byte2Vector) { if (IsDefault) { @@ -364,119 +405,139 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure var block = _block; var index = _index; - var array = block.Array; + var following = block.End - index; + byte[] array; + int byte0Index = int.MaxValue; + int byte1Index = int.MaxValue; + int byte2Index = int.MaxValue; + var byte0 = byte0Vector[0]; + var byte1 = byte1Vector[0]; + var byte2 = byte2Vector[0]; + while (true) { - while (block.End == index) + while (following == 0) { - if (block.Next == null) + var newBlock = block.Next; + if (newBlock == null) { _block = block; _index = index; return -1; } - block = block.Next; - index = block.Start; - array = block.Array; + index = newBlock.Start; + following = newBlock.End - index; + block = newBlock; } - while (block.End != index) + array = block.Array; + while (following > 0) { - var following = block.End - index; - if (following >= Vector.Count) +// Need unit tests to test Vector path +#if !DEBUG + // Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079 + if (Vector.IsHardwareAccelerated) { - var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); - var byte2Equals = Vector.Equals(data, byte2Vector); - int byte0Index = int.MaxValue; - int byte1Index = int.MaxValue; - int byte2Index = int.MaxValue; +#endif + if (following >= _vectorSpan) + { + var data = new Vector(array, index); + var byte0Equals = Vector.Equals(data, byte0Vector); + var byte1Equals = Vector.Equals(data, byte1Vector); + var byte2Equals = Vector.Equals(data, byte2Vector); - if (!byte0Equals.Equals(Vector.Zero)) - { - byte0Index = FindFirstEqualByte(byte0Equals); - } - if (!byte1Equals.Equals(Vector.Zero)) - { - byte1Index = FindFirstEqualByte(byte1Equals); - } - if (!byte2Equals.Equals(Vector.Zero)) - { - byte2Index = FindFirstEqualByte(byte2Equals); - } - - if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) - { - index += Vector.Count; - continue; - } - - int toReturn, toMove; - if (byte0Index < byte1Index) - { - if (byte0Index < byte2Index) + if (!byte0Equals.Equals(Vector.Zero)) { - toReturn = byte0Vector[0]; - toMove = byte0Index; + byte0Index = FindFirstEqualByte(ref byte0Equals); + } + if (!byte1Equals.Equals(Vector.Zero)) + { + byte1Index = FindFirstEqualByte(ref byte1Equals); + } + if (!byte2Equals.Equals(Vector.Zero)) + { + byte2Index = FindFirstEqualByte(ref byte2Equals); + } + + if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) + { + following -= _vectorSpan; + index += _vectorSpan; + continue; + } + + _block = block; + + int toReturn, toMove; + if (byte0Index < byte1Index) + { + if (byte0Index < byte2Index) + { + toReturn = byte0; + toMove = byte0Index; + } + else + { + toReturn = byte2; + toMove = byte2Index; + } } else { - toReturn = byte2Vector[0]; - toMove = byte2Index; + if (byte1Index < byte2Index) + { + toReturn = byte1; + toMove = byte1Index; + } + else + { + toReturn = byte2; + toMove = byte2Index; + } } - } - else - { - if (byte1Index < byte2Index) - { - toReturn = byte1Vector[0]; - toMove = byte1Index; - } - else - { - toReturn = byte2Vector[0]; - toMove = byte2Index; - } - } - _block = block; - _index = index + toMove; - return toReturn; + _index = index + toMove; + return toReturn; + } +// Need unit tests to test Vector path +#if !DEBUG } - - var byte0 = byte0Vector[0]; - var byte1 = byte1Vector[0]; - var byte2 = byte2Vector[0]; - - while (following > 0) +#endif + fixed (byte* ptr = &block.Array[index]) { - var byteIndex = block.Array[index]; - if (byteIndex == byte0) + var pCurrent = ptr; + var pEnd = pCurrent + following; + do { - _block = block; - _index = index; - return byte0; - } - else if (byteIndex == byte1) - { - _block = block; - _index = index; - return byte1; - } - else if (byteIndex == byte2) - { - _block = block; - _index = index; - return byte2; - } - following--; - index++; + if (*pCurrent == byte0) + { + _block = block; + _index = index; + return byte0; + } + if (*pCurrent == byte1) + { + _block = block; + _index = index; + return byte1; + } + if (*pCurrent == byte2) + { + _block = block; + _index = index; + return byte2; + } + pCurrent++; + index++; + } while (pCurrent != pEnd); } + + following = 0; + break; } } } - private static int FindFirstEqualByte(Vector byteEquals) + private static int FindFirstEqualByte(ref Vector byteEquals) { // Quasi-tree search var vector64 = Vector.AsVectorInt64(byteEquals); @@ -491,15 +552,14 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure if (vector32[shift] != 0) { if (byteEquals[offset] != 0) return offset; - if (byteEquals[++offset] != 0) return offset; - if (byteEquals[++offset] != 0) return offset; - return ++offset; + if (byteEquals[offset + 1] != 0) return offset + 1; + if (byteEquals[offset + 2] != 0) return offset + 2; + return offset + 3; } - offset += 4; - if (byteEquals[offset] != 0) return offset; - if (byteEquals[++offset] != 0) return offset; - if (byteEquals[++offset] != 0) return offset; - return ++offset; + if (byteEquals[offset + 4] != 0) return offset + 4; + if (byteEquals[offset + 5] != 0) return offset + 5; + if (byteEquals[offset + 6] != 0) return offset + 6; + return offset + 7; } throw new InvalidOperationException(); } diff --git a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs index c56f5311d5..2cbd077180 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs @@ -27,27 +27,27 @@ namespace Microsoft.AspNet.Server.KestrelTests var vectorCh = new Vector(ch); var hit = iterator; - hit.Seek(vectorCh); + hit.Seek(ref vectorCh); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorCh, vectorMaxValues); + hit.Seek(ref vectorCh, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorMaxValues, vectorCh); + hit.Seek(ref vectorMaxValues, ref vectorCh); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorCh, vectorMaxValues, vectorMaxValues); + hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorMaxValues, vectorCh, vectorMaxValues); + hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorCh, vectorMaxValues, vectorMaxValues); + hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); } } @@ -56,6 +56,9 @@ namespace Microsoft.AspNet.Server.KestrelTests [Fact] public void SeekWorksAcrossBlocks() { + Console.WriteLine($"Vector.IsHardwareAccelerated == {Vector.IsHardwareAccelerated}"); + Console.WriteLine($"Vector.Count == {Vector.Count}"); + using (var pool = new MemoryPool2()) { var block1 = pool.Lease(256); @@ -83,27 +86,27 @@ namespace Microsoft.AspNet.Server.KestrelTests var vectorCh = new Vector(ch); var hit = iterator; - hit.Seek(vectorCh); + hit.Seek(ref vectorCh); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorCh, vectorMaxValues); + hit.Seek(ref vectorCh, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorMaxValues, vectorCh); + hit.Seek(ref vectorMaxValues, ref vectorCh); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorCh, vectorMaxValues, vectorMaxValues); + hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorMaxValues, vectorCh, vectorMaxValues); + hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues); Assert.Equal(ch, iterator.GetLength(hit)); hit = iterator; - hit.Seek(vectorMaxValues, vectorMaxValues, vectorCh); + hit.Seek(ref vectorMaxValues, ref vectorMaxValues, ref vectorCh); Assert.Equal(ch, iterator.GetLength(hit)); } } diff --git a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolIterator2Tests.cs b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolIterator2Tests.cs index d7b9f494eb..2faee6f55b 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolIterator2Tests.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolIterator2Tests.cs @@ -53,15 +53,21 @@ namespace Microsoft.AspNet.Server.KestrelTests int found = -1; if (searchFor.Length == 1) { - found = begin.Seek(new Vector((byte)searchFor[0])); + var search0 = new Vector((byte)searchFor[0]); + found = begin.Seek(ref search0); } else if (searchFor.Length == 2) { - found = begin.Seek(new Vector((byte)searchFor[0]), new Vector((byte)searchFor[1])); + var search0 = new Vector((byte)searchFor[0]); + var search1 = new Vector((byte)searchFor[1]); + found = begin.Seek(ref search0, ref search1); } else if (searchFor.Length == 3) { - found = begin.Seek(new Vector((byte)searchFor[0]), new Vector((byte)searchFor[1]), new Vector((byte)searchFor[2])); + var search0 = new Vector((byte)searchFor[0]); + var search1 = new Vector((byte)searchFor[1]); + var search2 = new Vector((byte)searchFor[2]); + found = begin.Seek(ref search0, ref search1, ref search2); } else {