huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'benaadams/memorypool-seek' into dev

This commit is contained in:
Stephen Halter 2016-01-05 12:09:04 -08:00
parent 406d785b5e bc19a07291
commit 48233cd081
4 changed files with 278 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs
View File

@ -40,11 +40,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
private static readonly byte[] _bytesDate = Encoding.ASCII.GetBytes("Date: "); private static readonly byte[] _bytesDate = Encoding.ASCII.GetBytes("Date: ");
private static readonly byte[] _bytesEndHeaders = Encoding.ASCII.GetBytes("\r\n\r\n"); private static readonly byte[] _bytesEndHeaders = Encoding.ASCII.GetBytes("\r\n\r\n");
private static readonly Vector<byte> _vectorCRs = new Vector<byte>((byte)'\r'); private static Vector<byte> _vectorCRs = new Vector<byte>((byte)'\r');
private static readonly Vector<byte> _vectorColons = new Vector<byte>((byte)':'); private static Vector<byte> _vectorColons = new Vector<byte>((byte)':');
private static readonly Vector<byte> _vectorSpaces = new Vector<byte>((byte)' '); private static Vector<byte> _vectorSpaces = new Vector<byte>((byte)' ');
private static readonly Vector<byte> _vectorQuestionMarks = new Vector<byte>((byte)'?'); private static Vector<byte> _vectorQuestionMarks = new Vector<byte>((byte)'?');
private static readonly Vector<byte> _vectorPercentages = new Vector<byte>((byte)'%'); private static Vector<byte> _vectorPercentages = new Vector<byte>((byte)'%');
private readonly object _onStartingSync = new Object(); private readonly object _onStartingSync = new Object();
private readonly object _onCompletedSync = new Object(); private readonly object _onCompletedSync = new Object();
@ -714,7 +714,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
var begin = scan; var begin = scan;
if (!begin.GetKnownMethod(ref scan,out method)) if (!begin.GetKnownMethod(ref scan,out method))
{ {
if (scan.Seek(_vectorSpaces) == -1) if (scan.Seek(ref _vectorSpaces) == -1)
{ {
return false; return false;
} }
@ -725,11 +725,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
begin = scan; begin = scan;
var needDecode = false; var needDecode = false;
var chFound = scan.Seek(_vectorSpaces, _vectorQuestionMarks, _vectorPercentages); var chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref _vectorPercentages);
if (chFound == '%') if (chFound == '%')
{ {
needDecode = true; needDecode = true;
chFound = scan.Seek(_vectorSpaces, _vectorQuestionMarks); chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks);
} }
var pathBegin = begin; var pathBegin = begin;
@ -739,7 +739,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
if (chFound == '?') if (chFound == '?')
{ {
begin = scan; begin = scan;
if (scan.Seek(_vectorSpaces) != ' ') if (scan.Seek(ref _vectorSpaces) != ' ')
{ {
return false; return false;
} }
@ -753,7 +753,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
if (!begin.GetKnownVersion(ref scan, out httpVersion)) if (!begin.GetKnownVersion(ref scan, out httpVersion))
{ {
scan = begin; scan = begin;
if (scan.Seek(_vectorCRs) == -1) if (scan.Seek(ref _vectorCRs) == -1)
{ {
return false; return false;
} }
@ -843,7 +843,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
while (!scan.IsEnd) while (!scan.IsEnd)
{ {
var beginName = scan; var beginName = scan;
scan.Seek(_vectorColons, _vectorCRs); scan.Seek(ref _vectorColons, ref _vectorCRs);
var endName = scan; var endName = scan;
chFirst = scan.Take(); chFirst = scan.Take();
@ -894,7 +894,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
var wrapping = false; var wrapping = false;
while (!scan.IsEnd) while (!scan.IsEnd)
{ {
if (scan.Seek(_vectorCRs) == -1) if (scan.Seek(ref _vectorCRs) == -1)
{ {
// no "\r" in sight, burn used bytes and go back to await more data // no "\r" in sight, burn used bytes and go back to await more data
return false; return false;

424
src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs
View File

@ -9,6 +9,8 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
{ {
public struct MemoryPoolIterator2 public struct MemoryPoolIterator2
{ {
private readonly static int _vectorSpan = Vector<byte>.Count;
private MemoryPoolBlock2 _block; private MemoryPoolBlock2 _block;
private int _index; private int _index;
@ -208,7 +210,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
} }
} }
public int Seek(Vector<byte> byte0Vector) public unsafe int Seek(ref Vector<byte> byte0Vector)
{ {
if (IsDefault) if (IsDefault)
{ {
@ -217,58 +219,77 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
var block = _block; var block = _block;
var index = _index; var index = _index;
var array = block.Array; var following = block.End - index;
byte[] array;
var byte0 = byte0Vector[0];
while (true) while (true)
{ {
while (block.End == index) while (following == 0)
{ {
if (block.Next == null) var newBlock = block.Next;
if (newBlock == null)
{ {
_block = block; _block = block;
_index = index; _index = index;
return -1; return -1;
} }
block = block.Next; index = newBlock.Start;
index = block.Start; following = newBlock.End - index;
array = block.Array; block = newBlock;
} }
while (block.End != index) array = block.Array;
while (following > 0)
{ {
var following = block.End - index; // Need unit tests to test Vector path
if (following >= Vector<byte>.Count) #if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{ {
var data = new Vector<byte>(array, index); #endif
var byte0Equals = Vector.Equals(data, byte0Vector); if (following >= _vectorSpan)
if (byte0Equals.Equals(Vector<byte>.Zero))
{ {
index += Vector<byte>.Count; var byte0Equals = Vector.Equals(new Vector<byte>(array, index), byte0Vector);
continue;
}
_block = block; if (byte0Equals.Equals(Vector<byte>.Zero))
_index = index + FindFirstEqualByte(byte0Equals); {
return byte0Vector[0]; following -= _vectorSpan;
} index += _vectorSpan;
continue;
}
var byte0 = byte0Vector[0];
while (following > 0)
{
if (block.Array[index] == byte0)
{
_block = block; _block = block;
_index = index; _index = index + FindFirstEqualByte(ref byte0Equals);
return byte0; return byte0;
} }
following--; // Need unit tests to test Vector path
index++; #if !DEBUG
} }
#endif
fixed (byte* ptr = &block.Array[index])
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
pCurrent++;
index++;
} while (pCurrent < pEnd);
}
following = 0;
break;
} }
} }
} }
public int Seek(Vector<byte> byte0Vector, Vector<byte> byte1Vector) public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vector)
{ {
if (IsDefault) if (IsDefault)
{ {
@ -277,85 +298,105 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
var block = _block; var block = _block;
var index = _index; var index = _index;
var array = block.Array; var following = block.End - index;
byte[] array;
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
var byte0 = byte0Vector[0];
var byte1 = byte1Vector[0];
while (true) while (true)
{ {
while (block.End == index) while (following == 0)
{ {
if (block.Next == null) var newBlock = block.Next;
if (newBlock == null)
{ {
_block = block; _block = block;
_index = index; _index = index;
return -1; return -1;
} }
block = block.Next; index = newBlock.Start;
index = block.Start; following = newBlock.End - index;
array = block.Array; block = newBlock;
} }
while (block.End != index) array = block.Array;
while (following > 0)
{ {
var following = block.End - index;
if (following >= Vector<byte>.Count) // Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{ {
var data = new Vector<byte>(array, index); #endif
var byte0Equals = Vector.Equals(data, byte0Vector); if (following >= _vectorSpan)
var byte1Equals = Vector.Equals(data, byte1Vector);
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
if (!byte0Equals.Equals(Vector<byte>.Zero))
{ {
byte0Index = FindFirstEqualByte(byte0Equals); var data = new Vector<byte>(array, index);
} var byte0Equals = Vector.Equals(data, byte0Vector);
if (!byte1Equals.Equals(Vector<byte>.Zero)) var byte1Equals = Vector.Equals(data, byte1Vector);
{
byte1Index = FindFirstEqualByte(byte1Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) if (!byte0Equals.Equals(Vector<byte>.Zero))
{ {
index += Vector<byte>.Count; byte0Index = FindFirstEqualByte(ref byte0Equals);
continue; }
} if (!byte1Equals.Equals(Vector<byte>.Zero))
{
byte1Index = FindFirstEqualByte(ref byte1Equals);
}
_block = block; if (byte0Index == int.MaxValue && byte1Index == int.MaxValue)
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
if (byte0Index < byte1Index)
{
_index = index + byte0Index;
return byte0Vector[0];
}
_index = index + byte1Index;
return byte1Vector[0];
}
byte byte0 = byte0Vector[0];
byte byte1 = byte1Vector[0];
while (following > 0)
{
var byteIndex = block.Array[index];
if (byteIndex == byte0)
{
_block = block; _block = block;
_index = index;
return byte0; if (byte0Index < byte1Index)
} {
else if (byteIndex == byte1) _index = index + byte0Index;
{ return byte0;
_block = block; }
_index = index;
_index = index + byte1Index;
return byte1; return byte1;
} }
following--; // Need unit tests to test Vector path
index++; #if !DEBUG
} }
#endif
fixed (byte* ptr = &block.Array[index])
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return byte1;
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
}
following = 0;
break;
} }
} }
} }
public int Seek(Vector<byte> byte0Vector, Vector<byte> byte1Vector, Vector<byte> byte2Vector) public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vector, ref Vector<byte> byte2Vector)
{ {
if (IsDefault) if (IsDefault)
{ {
@ -364,119 +405,139 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
var block = _block; var block = _block;
var index = _index; var index = _index;
var array = block.Array; var following = block.End - index;
byte[] array;
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
int byte2Index = int.MaxValue;
var byte0 = byte0Vector[0];
var byte1 = byte1Vector[0];
var byte2 = byte2Vector[0];
while (true) while (true)
{ {
while (block.End == index) while (following == 0)
{ {
if (block.Next == null) var newBlock = block.Next;
if (newBlock == null)
{ {
_block = block; _block = block;
_index = index; _index = index;
return -1; return -1;
} }
block = block.Next; index = newBlock.Start;
index = block.Start; following = newBlock.End - index;
array = block.Array; block = newBlock;
} }
while (block.End != index) array = block.Array;
while (following > 0)
{ {
var following = block.End - index; // Need unit tests to test Vector path
if (following >= Vector<byte>.Count) #if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{ {
var data = new Vector<byte>(array, index); #endif
var byte0Equals = Vector.Equals(data, byte0Vector); if (following >= _vectorSpan)
var byte1Equals = Vector.Equals(data, byte1Vector); {
var byte2Equals = Vector.Equals(data, byte2Vector); var data = new Vector<byte>(array, index);
int byte0Index = int.MaxValue; var byte0Equals = Vector.Equals(data, byte0Vector);
int byte1Index = int.MaxValue; var byte1Equals = Vector.Equals(data, byte1Vector);
int byte2Index = int.MaxValue; var byte2Equals = Vector.Equals(data, byte2Vector);
if (!byte0Equals.Equals(Vector<byte>.Zero)) if (!byte0Equals.Equals(Vector<byte>.Zero))
{
byte0Index = FindFirstEqualByte(byte0Equals);
}
if (!byte1Equals.Equals(Vector<byte>.Zero))
{
byte1Index = FindFirstEqualByte(byte1Equals);
}
if (!byte2Equals.Equals(Vector<byte>.Zero))
{
byte2Index = FindFirstEqualByte(byte2Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
{
index += Vector<byte>.Count;
continue;
}
int toReturn, toMove;
if (byte0Index < byte1Index)
{
if (byte0Index < byte2Index)
{ {
toReturn = byte0Vector[0]; byte0Index = FindFirstEqualByte(ref byte0Equals);
toMove = byte0Index; }
if (!byte1Equals.Equals(Vector<byte>.Zero))
{
byte1Index = FindFirstEqualByte(ref byte1Equals);
}
if (!byte2Equals.Equals(Vector<byte>.Zero))
{
byte2Index = FindFirstEqualByte(ref byte2Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_block = block;
int toReturn, toMove;
if (byte0Index < byte1Index)
{
if (byte0Index < byte2Index)
{
toReturn = byte0;
toMove = byte0Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
} }
else else
{ {
toReturn = byte2Vector[0]; if (byte1Index < byte2Index)
toMove = byte2Index; {
toReturn = byte1;
toMove = byte1Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
} }
}
else
{
if (byte1Index < byte2Index)
{
toReturn = byte1Vector[0];
toMove = byte1Index;
}
else
{
toReturn = byte2Vector[0];
toMove = byte2Index;
}
}
_block = block; _index = index + toMove;
_index = index + toMove; return toReturn;
return toReturn; }
// Need unit tests to test Vector path
#if !DEBUG
} }
#endif
var byte0 = byte0Vector[0]; fixed (byte* ptr = &block.Array[index])
var byte1 = byte1Vector[0];
var byte2 = byte2Vector[0];
while (following > 0)
{ {
var byteIndex = block.Array[index]; var pCurrent = ptr;
if (byteIndex == byte0) var pEnd = pCurrent + following;
do
{ {
_block = block; if (*pCurrent == byte0)
_index = index; {
return byte0; _block = block;
} _index = index;
else if (byteIndex == byte1) return byte0;
{ }
_block = block; if (*pCurrent == byte1)
_index = index; {
return byte1; _block = block;
} _index = index;
else if (byteIndex == byte2) return byte1;
{ }
_block = block; if (*pCurrent == byte2)
_index = index; {
return byte2; _block = block;
} _index = index;
following--; return byte2;
index++; }
pCurrent++;
index++;
} while (pCurrent != pEnd);
} }
following = 0;
break;
} }
} }
} }
private static int FindFirstEqualByte(Vector<byte> byteEquals) private static int FindFirstEqualByte(ref Vector<byte> byteEquals)
{ {
// Quasi-tree search // Quasi-tree search
var vector64 = Vector.AsVectorInt64(byteEquals); var vector64 = Vector.AsVectorInt64(byteEquals);
@ -491,15 +552,14 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
if (vector32[shift] != 0) if (vector32[shift] != 0)
{ {
if (byteEquals[offset] != 0) return offset; if (byteEquals[offset] != 0) return offset;
if (byteEquals[++offset] != 0) return offset; if (byteEquals[offset + 1] != 0) return offset + 1;
if (byteEquals[++offset] != 0) return offset; if (byteEquals[offset + 2] != 0) return offset + 2;
return ++offset; return offset + 3;
} }
offset += 4; if (byteEquals[offset + 4] != 0) return offset + 4;
if (byteEquals[offset] != 0) return offset; if (byteEquals[offset + 5] != 0) return offset + 5;
if (byteEquals[++offset] != 0) return offset; if (byteEquals[offset + 6] != 0) return offset + 6;
if (byteEquals[++offset] != 0) return offset; return offset + 7;
return ++offset;
} }
throw new InvalidOperationException(); throw new InvalidOperationException();
} }

27
test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs
View File

@ -27,27 +27,27 @@ namespace Microsoft.AspNet.Server.KestrelTests
var vectorCh = new Vector<byte>(ch); var vectorCh = new Vector<byte>(ch);
var hit = iterator; var hit = iterator;
hit.Seek(vectorCh); hit.Seek(ref vectorCh);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorCh, vectorMaxValues); hit.Seek(ref vectorCh, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorMaxValues, vectorCh); hit.Seek(ref vectorMaxValues, ref vectorCh);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorCh, vectorMaxValues, vectorMaxValues); hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorMaxValues, vectorCh, vectorMaxValues); hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorCh, vectorMaxValues, vectorMaxValues); hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
} }
} }
@ -56,6 +56,9 @@ namespace Microsoft.AspNet.Server.KestrelTests
[Fact] [Fact]
public void SeekWorksAcrossBlocks() public void SeekWorksAcrossBlocks()
{ {
Console.WriteLine($"Vector.IsHardwareAccelerated == {Vector.IsHardwareAccelerated}");
Console.WriteLine($"Vector<byte>.Count == {Vector<byte>.Count}");
using (var pool = new MemoryPool2()) using (var pool = new MemoryPool2())
{ {
var block1 = pool.Lease(256); var block1 = pool.Lease(256);
@ -83,27 +86,27 @@ namespace Microsoft.AspNet.Server.KestrelTests
var vectorCh = new Vector<byte>(ch); var vectorCh = new Vector<byte>(ch);
var hit = iterator; var hit = iterator;
hit.Seek(vectorCh); hit.Seek(ref vectorCh);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorCh, vectorMaxValues); hit.Seek(ref vectorCh, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorMaxValues, vectorCh); hit.Seek(ref vectorMaxValues, ref vectorCh);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorCh, vectorMaxValues, vectorMaxValues); hit.Seek(ref vectorCh, ref vectorMaxValues, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorMaxValues, vectorCh, vectorMaxValues); hit.Seek(ref vectorMaxValues, ref vectorCh, ref vectorMaxValues);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
hit = iterator; hit = iterator;
hit.Seek(vectorMaxValues, vectorMaxValues, vectorCh); hit.Seek(ref vectorMaxValues, ref vectorMaxValues, ref vectorCh);
Assert.Equal(ch, iterator.GetLength(hit)); Assert.Equal(ch, iterator.GetLength(hit));
} }
} }

12
test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolIterator2Tests.cs
View File

@ -53,15 +53,21 @@ namespace Microsoft.AspNet.Server.KestrelTests
int found = -1; int found = -1;
if (searchFor.Length == 1) if (searchFor.Length == 1)
{ {
found = begin.Seek(new Vector<byte>((byte)searchFor[0])); var search0 = new Vector<byte>((byte)searchFor[0]);
found = begin.Seek(ref search0);
} }
else if (searchFor.Length == 2) else if (searchFor.Length == 2)
{ {
found = begin.Seek(new Vector<byte>((byte)searchFor[0]), new Vector<byte>((byte)searchFor[1])); var search0 = new Vector<byte>((byte)searchFor[0]);
var search1 = new Vector<byte>((byte)searchFor[1]);
found = begin.Seek(ref search0, ref search1);
} }
else if (searchFor.Length == 3) else if (searchFor.Length == 3)
{ {
found = begin.Seek(new Vector<byte>((byte)searchFor[0]), new Vector<byte>((byte)searchFor[1]), new Vector<byte>((byte)searchFor[2])); var search0 = new Vector<byte>((byte)searchFor[0]);
var search1 = new Vector<byte>((byte)searchFor[1]);
var search2 = new Vector<byte>((byte)searchFor[2]);
found = begin.Seek(ref search0, ref search1, ref search2);
} }
else else
{ {