From 4460cd34700ac755500cf41cc51ed5f1bf6ae3a1 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 16 Dec 2015 12:57:05 +0000 Subject: [PATCH] Don't use Vector.Dot --- .../Infrastructure/MemoryPoolIterator2.cs | 192 +++++++++--------- .../MemoryPoolBlock2Tests.cs | 64 ++++++ 2 files changed, 164 insertions(+), 92 deletions(-) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs index 41551d056c..90b58948fe 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs @@ -3,25 +3,12 @@ using System; using System.Diagnostics; -using System.Linq; using System.Numerics; namespace Microsoft.AspNet.Server.Kestrel.Infrastructure { public struct MemoryPoolIterator2 { - /// - /// Array of "minus one" bytes of the length of SIMD operations on the current hardware. Used as an argument in the - /// vector dot product that counts matching character occurrence. - /// - private static Vector _dotCount = new Vector(Byte.MaxValue); - - /// - /// Array of negative numbers starting at 0 and continuing for the length of SIMD operations on the current hardware. - /// Used as an argument in the vector dot product that determines matching character index. - /// - private static Vector _dotIndex = new Vector(Enumerable.Range(0, Vector.Count).Select(x => (byte)-x).ToArray()); - private MemoryPoolBlock2 _block; private int _index; @@ -146,7 +133,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure } var byte0 = (byte)char0; - var vectorStride = Vector.Count; var ch0Vector = new Vector(byte0); var block = _block; @@ -169,27 +155,20 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure while (block.End != index) { var following = block.End - index; - if (following >= vectorStride) + if (following >= Vector.Count) { var data = new Vector(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); - var ch0Count = Vector.Dot(ch0Equals, _dotCount); - if (ch0Count == 0) + if (ch0Equals.Equals(Vector.Zero)) { - index += vectorStride; + index += Vector.Count; continue; } - else if (ch0Count == 1) - { - _block = block; - _index = index + Vector.Dot(ch0Equals, _dotIndex); - return char0; - } - else - { - following = vectorStride; - } + + _block = block; + _index = index + FindFirstEqualByte(ch0Equals); + return char0; } while (following > 0) { @@ -215,7 +194,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure var byte0 = (byte)char0; var byte1 = (byte)char1; - var vectorStride = Vector.Count; var ch0Vector = new Vector(byte0); var ch1Vector = new Vector(byte1); @@ -239,40 +217,39 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure while (block.End != index) { var following = block.End - index; - if (following >= vectorStride) + if (following >= Vector.Count) { var data = new Vector(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); - var ch0Count = Vector.Dot(ch0Equals, _dotCount); var ch1Equals = Vector.Equals(data, ch1Vector); - var ch1Count = Vector.Dot(ch1Equals, _dotCount); + int ch0Index = int.MaxValue; + int ch1Index = int.MaxValue; - if (ch0Count == 0 && ch1Count == 0) + if (!ch0Equals.Equals(Vector.Zero)) { - index += vectorStride; + ch0Index = FindFirstEqualByte(ch0Equals); + } + if (!ch1Equals.Equals(Vector.Zero)) + { + ch1Index = FindFirstEqualByte(ch1Equals); + } + + if (ch0Index == int.MaxValue && ch1Index == int.MaxValue) + { + index += Vector.Count; continue; } - else if (ch0Count < 2 && ch1Count < 2) + + _block = block; + + if (ch0Index < ch1Index) { - var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue; - var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue; - if (ch0Index < ch1Index) - { - _block = block; - _index = index + ch0Index; - return char0; - } - else - { - _block = block; - _index = index + ch1Index; - return char1; - } - } - else - { - following = vectorStride; + _index = index + ch0Index; + return char0; } + + _index = index + ch1Index; + return char1; } while (following > 0) { @@ -306,7 +283,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure var byte0 = (byte)char0; var byte1 = (byte)char1; var byte2 = (byte)char2; - var vectorStride = Vector.Count; var ch0Vector = new Vector(byte0); var ch1Vector = new Vector(byte1); var ch2Vector = new Vector(byte2); @@ -331,64 +307,68 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure while (block.End != index) { var following = block.End - index; - if (following >= vectorStride) + if (following >= Vector.Count) { var data = new Vector(array, index); var ch0Equals = Vector.Equals(data, ch0Vector); - var ch0Count = Vector.Dot(ch0Equals, _dotCount); var ch1Equals = Vector.Equals(data, ch1Vector); - var ch1Count = Vector.Dot(ch1Equals, _dotCount); var ch2Equals = Vector.Equals(data, ch2Vector); - var ch2Count = Vector.Dot(ch2Equals, _dotCount); + int ch0Index = int.MaxValue; + int ch1Index = int.MaxValue; + int ch2Index = int.MaxValue; - if (ch0Count == 0 && ch1Count == 0 && ch2Count == 0) + if (!ch0Equals.Equals(Vector.Zero)) { - index += vectorStride; + ch0Index = FindFirstEqualByte(ch0Equals); + } + if (!ch1Equals.Equals(Vector.Zero)) + { + ch1Index = FindFirstEqualByte(ch1Equals); + } + if (!ch2Equals.Equals(Vector.Zero)) + { + ch2Index = FindFirstEqualByte(ch2Equals); + } + + if (ch0Index == int.MaxValue && ch1Index == int.MaxValue && ch2Index == int.MaxValue) + { + index += Vector.Count; continue; } - else if (ch0Count < 2 && ch1Count < 2 && ch2Count < 2) - { - var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue; - var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue; - var ch2Index = ch2Count == 1 ? Vector.Dot(ch2Equals, _dotIndex) : byte.MaxValue; - int toReturn, toMove; - if (ch0Index < ch1Index) + int toReturn, toMove; + if (ch0Index < ch1Index) + { + if (ch0Index < ch2Index) { - if (ch0Index < ch2Index) - { - toReturn = char0; - toMove = ch0Index; - } - else - { - toReturn = char2; - toMove = ch2Index; - } + toReturn = char0; + toMove = ch0Index; } else { - if (ch1Index < ch2Index) - { - toReturn = char1; - toMove = ch1Index; - } - else - { - toReturn = char2; - toMove = ch2Index; - } + toReturn = char2; + toMove = ch2Index; } - - _block = block; - _index = index + toMove; - return toReturn; } else { - following = vectorStride; + if (ch1Index < ch2Index) + { + toReturn = char1; + toMove = ch1Index; + } + else + { + toReturn = char2; + toMove = ch2Index; + } } + + _block = block; + _index = index + toMove; + return toReturn; } + while (following > 0) { var byteIndex = block.Array[index]; @@ -417,6 +397,34 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure } } + private static int FindFirstEqualByte(Vector chEquals) + { + // Quasi-tree search + var vector64 = Vector.AsVectorInt64(chEquals); + for (var i = 0; i < Vector.Count; i++) + { + var longValue = vector64[i]; + if (longValue == 0) continue; + + var shift = i << 1; + var offset = shift << 2; + var vector32 = Vector.AsVectorInt32(chEquals); + if (vector32[shift] != 0) + { + if (chEquals[offset] != 0) return offset; + if (chEquals[++offset] != 0) return offset; + if (chEquals[++offset] != 0) return offset; + return ++offset; + } + offset += 4; + if (chEquals[offset] != 0) return offset; + if (chEquals[++offset] != 0) return offset; + if (chEquals[++offset] != 0) return offset; + return ++offset; + } + throw new InvalidOperationException(); + } + /// /// Save the data at the current location then move to the next available space. /// diff --git a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs index 239175e198..8618564004 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs @@ -31,6 +31,70 @@ namespace Microsoft.AspNet.Server.KestrelTests hit = iterator; hit.Seek(byte.MaxValue, ch); Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(ch, byte.MaxValue, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(byte.MaxValue, ch, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(ch, byte.MaxValue, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); + } + } + } + + [Fact] + public void SeekWorksAcrossBlocks() + { + using (var pool = new MemoryPool2()) + { + var block1 = pool.Lease(256); + var block2 = block1.Next = pool.Lease(256); + var block3 = block2.Next = pool.Lease(256); + + foreach (var ch in Enumerable.Range(0, 34).Select(x => (byte)x)) + { + block1.Array[block1.End++] = ch; + } + foreach (var ch in Enumerable.Range(34, 25).Select(x => (byte)x)) + { + block2.Array[block2.End++] = ch; + } + foreach (var ch in Enumerable.Range(59, 197).Select(x => (byte)x)) + { + block3.Array[block3.End++] = ch; + } + + var iterator = block1.GetIterator(); + foreach (var ch in Enumerable.Range(0, 256).Select(x => (char)x)) + { + var hit = iterator; + hit.Seek(ch); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(ch, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(byte.MaxValue, ch); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(ch, byte.MaxValue, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(byte.MaxValue, ch, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); + + hit = iterator; + hit.Seek(ch, byte.MaxValue, byte.MaxValue); + Assert.Equal(ch, iterator.GetLength(hit)); } } }