diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs
index 524c7a0278..cab6fbe9ef 100644
--- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs
+++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs
@@ -3,25 +3,12 @@
using System;
using System.Diagnostics;
-using System.Linq;
using System.Numerics;
namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
{
public struct MemoryPoolIterator2
{
- ///
- /// Array of "minus one" bytes of the length of SIMD operations on the current hardware. Used as an argument in the
- /// vector dot product that counts matching character occurrence.
- ///
- private static Vector _dotCount = new Vector(Byte.MaxValue);
-
- ///
- /// Array of negative numbers starting at 0 and continuing for the length of SIMD operations on the current hardware.
- /// Used as an argument in the vector dot product that determines matching character index.
- ///
- private static Vector _dotIndex = new Vector(Enumerable.Range(0, Vector.Count).Select(x => (byte)-x).ToArray());
-
private MemoryPoolBlock2 _block;
private int _index;
@@ -189,7 +176,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
}
var byte0 = (byte)char0;
- var vectorStride = Vector.Count;
var ch0Vector = new Vector(byte0);
var block = _block;
@@ -212,27 +198,20 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
while (block.End != index)
{
var following = block.End - index;
- if (following >= vectorStride)
+ if (following >= Vector.Count)
{
var data = new Vector(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
- var ch0Count = Vector.Dot(ch0Equals, _dotCount);
- if (ch0Count == 0)
+ if (ch0Equals.Equals(Vector.Zero))
{
- index += vectorStride;
+ index += Vector.Count;
continue;
}
- else if (ch0Count == 1)
- {
- _block = block;
- _index = index + Vector.Dot(ch0Equals, _dotIndex);
- return char0;
- }
- else
- {
- following = vectorStride;
- }
+
+ _block = block;
+ _index = index + FindFirstEqualByte(ch0Equals);
+ return char0;
}
while (following > 0)
{
@@ -258,7 +237,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
var byte0 = (byte)char0;
var byte1 = (byte)char1;
- var vectorStride = Vector.Count;
var ch0Vector = new Vector(byte0);
var ch1Vector = new Vector(byte1);
@@ -282,40 +260,39 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
while (block.End != index)
{
var following = block.End - index;
- if (following >= vectorStride)
+ if (following >= Vector.Count)
{
var data = new Vector(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
- var ch0Count = Vector.Dot(ch0Equals, _dotCount);
var ch1Equals = Vector.Equals(data, ch1Vector);
- var ch1Count = Vector.Dot(ch1Equals, _dotCount);
+ int ch0Index = int.MaxValue;
+ int ch1Index = int.MaxValue;
- if (ch0Count == 0 && ch1Count == 0)
+ if (!ch0Equals.Equals(Vector.Zero))
{
- index += vectorStride;
+ ch0Index = FindFirstEqualByte(ch0Equals);
+ }
+ if (!ch1Equals.Equals(Vector.Zero))
+ {
+ ch1Index = FindFirstEqualByte(ch1Equals);
+ }
+
+ if (ch0Index == int.MaxValue && ch1Index == int.MaxValue)
+ {
+ index += Vector.Count;
continue;
}
- else if (ch0Count < 2 && ch1Count < 2)
+
+ _block = block;
+
+ if (ch0Index < ch1Index)
{
- var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
- var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
- if (ch0Index < ch1Index)
- {
- _block = block;
- _index = index + ch0Index;
- return char0;
- }
- else
- {
- _block = block;
- _index = index + ch1Index;
- return char1;
- }
- }
- else
- {
- following = vectorStride;
+ _index = index + ch0Index;
+ return char0;
}
+
+ _index = index + ch1Index;
+ return char1;
}
while (following > 0)
{
@@ -349,7 +326,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
var byte0 = (byte)char0;
var byte1 = (byte)char1;
var byte2 = (byte)char2;
- var vectorStride = Vector.Count;
var ch0Vector = new Vector(byte0);
var ch1Vector = new Vector(byte1);
var ch2Vector = new Vector(byte2);
@@ -374,64 +350,68 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
while (block.End != index)
{
var following = block.End - index;
- if (following >= vectorStride)
+ if (following >= Vector.Count)
{
var data = new Vector(array, index);
var ch0Equals = Vector.Equals(data, ch0Vector);
- var ch0Count = Vector.Dot(ch0Equals, _dotCount);
var ch1Equals = Vector.Equals(data, ch1Vector);
- var ch1Count = Vector.Dot(ch1Equals, _dotCount);
var ch2Equals = Vector.Equals(data, ch2Vector);
- var ch2Count = Vector.Dot(ch2Equals, _dotCount);
+ int ch0Index = int.MaxValue;
+ int ch1Index = int.MaxValue;
+ int ch2Index = int.MaxValue;
- if (ch0Count == 0 && ch1Count == 0 && ch2Count == 0)
+ if (!ch0Equals.Equals(Vector.Zero))
{
- index += vectorStride;
+ ch0Index = FindFirstEqualByte(ch0Equals);
+ }
+ if (!ch1Equals.Equals(Vector.Zero))
+ {
+ ch1Index = FindFirstEqualByte(ch1Equals);
+ }
+ if (!ch2Equals.Equals(Vector.Zero))
+ {
+ ch2Index = FindFirstEqualByte(ch2Equals);
+ }
+
+ if (ch0Index == int.MaxValue && ch1Index == int.MaxValue && ch2Index == int.MaxValue)
+ {
+ index += Vector.Count;
continue;
}
- else if (ch0Count < 2 && ch1Count < 2 && ch2Count < 2)
- {
- var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
- var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
- var ch2Index = ch2Count == 1 ? Vector.Dot(ch2Equals, _dotIndex) : byte.MaxValue;
- int toReturn, toMove;
- if (ch0Index < ch1Index)
+ int toReturn, toMove;
+ if (ch0Index < ch1Index)
+ {
+ if (ch0Index < ch2Index)
{
- if (ch0Index < ch2Index)
- {
- toReturn = char0;
- toMove = ch0Index;
- }
- else
- {
- toReturn = char2;
- toMove = ch2Index;
- }
+ toReturn = char0;
+ toMove = ch0Index;
}
else
{
- if (ch1Index < ch2Index)
- {
- toReturn = char1;
- toMove = ch1Index;
- }
- else
- {
- toReturn = char2;
- toMove = ch2Index;
- }
+ toReturn = char2;
+ toMove = ch2Index;
}
-
- _block = block;
- _index = index + toMove;
- return toReturn;
}
else
{
- following = vectorStride;
+ if (ch1Index < ch2Index)
+ {
+ toReturn = char1;
+ toMove = ch1Index;
+ }
+ else
+ {
+ toReturn = char2;
+ toMove = ch2Index;
+ }
}
+
+ _block = block;
+ _index = index + toMove;
+ return toReturn;
}
+
while (following > 0)
{
var byteIndex = block.Array[index];
@@ -460,6 +440,34 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
}
}
+ private static int FindFirstEqualByte(Vector chEquals)
+ {
+ // Quasi-tree search
+ var vector64 = Vector.AsVectorInt64(chEquals);
+ for (var i = 0; i < Vector.Count; i++)
+ {
+ var longValue = vector64[i];
+ if (longValue == 0) continue;
+
+ var shift = i << 1;
+ var offset = shift << 2;
+ var vector32 = Vector.AsVectorInt32(chEquals);
+ if (vector32[shift] != 0)
+ {
+ if (chEquals[offset] != 0) return offset;
+ if (chEquals[++offset] != 0) return offset;
+ if (chEquals[++offset] != 0) return offset;
+ return ++offset;
+ }
+ offset += 4;
+ if (chEquals[offset] != 0) return offset;
+ if (chEquals[++offset] != 0) return offset;
+ if (chEquals[++offset] != 0) return offset;
+ return ++offset;
+ }
+ throw new InvalidOperationException();
+ }
+
///
/// Save the data at the current location then move to the next available space.
///
diff --git a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs
index 239175e198..8618564004 100644
--- a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs
+++ b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs
@@ -31,6 +31,70 @@ namespace Microsoft.AspNet.Server.KestrelTests
hit = iterator;
hit.Seek(byte.MaxValue, ch);
Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(ch, byte.MaxValue, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(byte.MaxValue, ch, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(ch, byte.MaxValue, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
+ }
+ }
+ }
+
+ [Fact]
+ public void SeekWorksAcrossBlocks()
+ {
+ using (var pool = new MemoryPool2())
+ {
+ var block1 = pool.Lease(256);
+ var block2 = block1.Next = pool.Lease(256);
+ var block3 = block2.Next = pool.Lease(256);
+
+ foreach (var ch in Enumerable.Range(0, 34).Select(x => (byte)x))
+ {
+ block1.Array[block1.End++] = ch;
+ }
+ foreach (var ch in Enumerable.Range(34, 25).Select(x => (byte)x))
+ {
+ block2.Array[block2.End++] = ch;
+ }
+ foreach (var ch in Enumerable.Range(59, 197).Select(x => (byte)x))
+ {
+ block3.Array[block3.End++] = ch;
+ }
+
+ var iterator = block1.GetIterator();
+ foreach (var ch in Enumerable.Range(0, 256).Select(x => (char)x))
+ {
+ var hit = iterator;
+ hit.Seek(ch);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(ch, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(byte.MaxValue, ch);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(ch, byte.MaxValue, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(byte.MaxValue, ch, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
+
+ hit = iterator;
+ hit.Seek(ch, byte.MaxValue, byte.MaxValue);
+ Assert.Equal(ch, iterator.GetLength(hit));
}
}
}