From caaf9d473b6ad48ace098e37dacea99540aecdc6 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 23 Dec 2015 08:38:48 +0000 Subject: [PATCH] Faster CopyFrom --- .../Http/Connection.cs | 2 +- .../Http/SocketInput.cs | 2 +- .../Http/SocketOutput.cs | 2 +- .../Infrastructure/MemoryPoolBlock2.cs | 6 + .../Infrastructure/MemoryPoolIterator2.cs | 195 ++++++++++++------ .../Networking/UvWriteReq.cs | 2 +- .../MemoryPoolBlock2Tests.cs | 2 +- .../SocketOutputTests.cs | 4 +- 8 files changed, 143 insertions(+), 72 deletions(-) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs index 6f2ea40f27..a93b0794ee 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs @@ -144,7 +144,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http return handle.Libuv.buf_init( result.Pin() + result.End, - result.Data.Offset + result.Data.Count - result.End); + result.BlockEndOffset - result.End); } private static void ReadCallback(UvStreamHandle handle, int status, object state) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs index 944457af60..37641d7df4 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs @@ -42,7 +42,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http { const int minimumSize = 2048; - if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End) + if (_tail != null && minimumSize <= _tail.BlockEndOffset - _tail.End) { _pinned = _tail; } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs index cf3f8dc528..c1b5bd0e63 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs @@ -400,7 +400,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http return; } - bytes = start.Block.Data.Offset + start.Block.Data.Count - start.Index; + bytes = start.Block.BlockEndOffset - start.Index; buffers = 1; for (var block = start.Block.Next; block != end.Block; block = block.Next) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs index 26156c6cc0..920b4e9e49 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs @@ -52,6 +52,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure /// public byte[] Array => Data.Array; + /// + /// Fixed end offset of the block + /// + public int BlockEndOffset { get; private set; } + /// /// The Start represents the offset into Array where the range of "active" bytes begins. At the point when the block is leased /// the Start is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and @@ -144,6 +149,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure Slab = slab, Start = data.Offset, End = data.Offset, + BlockEndOffset = data.Offset + data.Count }; } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs index c6326465f9..ec1d23d88a 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs @@ -692,46 +692,52 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure CopyFrom(buffer.Array, buffer.Offset, buffer.Count); } - public void CopyFrom(byte[] data, int offset, int count) + public unsafe void CopyFrom(byte[] data, int offset, int count) { Debug.Assert(_block != null); Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); - - var pool = _block.Pool; + var block = _block; var blockIndex = _index; + var bytesLeftInBlock = block.BlockEndOffset - blockIndex; - var bufferIndex = offset; - var remaining = count; - var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; + if (bytesLeftInBlock >= count) + { + _index = blockIndex + count; + Buffer.BlockCopy(data, offset, block.Array, blockIndex, count); + block.End = _index; + return; + } - while (remaining > 0) + do { if (bytesLeftInBlock == 0) { - var nextBlock = pool.Lease(); - block.End = blockIndex; + var nextBlock = block.Pool.Lease(); + blockIndex = nextBlock.Data.Offset; + bytesLeftInBlock = nextBlock.Data.Count; block.Next = nextBlock; block = nextBlock; - - blockIndex = block.Data.Offset; - bytesLeftInBlock = block.Data.Count; } - var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock; - - Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy); - - blockIndex += bytesToCopy; - bufferIndex += bytesToCopy; - remaining -= bytesToCopy; - bytesLeftInBlock -= bytesToCopy; - } - - block.End = blockIndex; - _block = block; - _index = blockIndex; + if (count > bytesLeftInBlock) + { + count -= bytesLeftInBlock; + Buffer.BlockCopy(data, offset, block.Array, blockIndex, bytesLeftInBlock); + offset += bytesLeftInBlock; + block.End = blockIndex + bytesLeftInBlock; + bytesLeftInBlock = 0; + } + else + { + _index = blockIndex + count; + Buffer.BlockCopy(data, offset, block.Array, blockIndex, count); + block.End = _index; + _block = block; + return; + } + } while (true); } public unsafe void CopyFromAscii(string data) @@ -740,64 +746,123 @@ namespace Microsoft.AspNet.Server.Kestrel.Infrastructure Debug.Assert(_block.Next == null); Debug.Assert(_block.End == _index); - var pool = _block.Pool; var block = _block; var blockIndex = _index; - var length = data.Length; + var count = data.Length; - var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; - var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; + var blockRemaining = block.BlockEndOffset - blockIndex; - fixed (char* pData = data) + fixed (char* pInput = data) { - var input = pData; - var inputEnd = pData + length; - var inputEndMinusSpan = inputEnd - 3; - - while (input < inputEnd) + if (blockRemaining >= count) { - if (bytesLeftInBlock == 0) + _index = blockIndex + count; + + fixed (byte* pOutput = &block.Data.Array[blockIndex]) { - var nextBlock = pool.Lease(); - block.End = blockIndex; + CopyFromAscii(pInput, pOutput, count); + } + + block.End = _index; + return; + } + + var input = pInput; + do + { + if (blockRemaining == 0) + { + var nextBlock = block.Pool.Lease(); + blockIndex = nextBlock.Data.Offset; + blockRemaining = nextBlock.Data.Count; block.Next = nextBlock; block = nextBlock; - - blockIndex = block.Data.Offset; - bytesLeftInBlock = block.Data.Count; - bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; } - fixed (byte* pOutput = &block.Data.Array[block.End]) + if (count > blockRemaining) { - //this line is needed to allow output be an register var - var output = pOutput; + count -= blockRemaining; - var copied = 0; - for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4) + fixed (byte* pOutput = &block.Data.Array[blockIndex]) { - *(output) = (byte)*(input); - *(output + 1) = (byte)*(input + 1); - *(output + 2) = (byte)*(input + 2); - *(output + 3) = (byte)*(input + 3); - output += 4; - input += 4; - } - for (; input < inputEnd && copied < bytesLeftInBlock; copied++) - { - *(output++) = (byte)*(input++); + CopyFromAscii(input, pOutput, blockRemaining); } - blockIndex += copied; - bytesLeftInBlockMinusSpan -= copied; - bytesLeftInBlock -= copied; + block.End = blockIndex + blockRemaining; + input += blockRemaining; + blockRemaining = 0; } - } - } + else + { + _index = blockIndex + count; - block.End = blockIndex; - _block = block; - _index = blockIndex; + fixed (byte* pOutput = &block.Data.Array[blockIndex]) + { + CopyFromAscii(input, pOutput, count); + } + + block.End = _index; + _block = block; + return; + } + } while (true); + } + } + + private unsafe static void CopyFromAscii(char* pInput, byte* pOutput, int count) + { + var i = 0; + //these line is needed to allow input/output be an register var + var input = pInput; + var output = pOutput; + + while (i < count - 11) + { + i += 12; + *(output) = (byte)*(input); + *(output + 1) = (byte)*(input + 1); + *(output + 2) = (byte)*(input + 2); + *(output + 3) = (byte)*(input + 3); + *(output + 4) = (byte)*(input + 4); + *(output + 5) = (byte)*(input + 5); + *(output + 6) = (byte)*(input + 6); + *(output + 7) = (byte)*(input + 7); + *(output + 8) = (byte)*(input + 8); + *(output + 9) = (byte)*(input + 9); + *(output + 10) = (byte)*(input + 10); + *(output + 11) = (byte)*(input + 11); + output += 12; + input += 12; + } + if (i < count - 5) + { + i += 6; + *(output) = (byte)*(input); + *(output + 1) = (byte)*(input + 1); + *(output + 2) = (byte)*(input + 2); + *(output + 3) = (byte)*(input + 3); + *(output + 4) = (byte)*(input + 4); + *(output + 5) = (byte)*(input + 5); + output += 6; + input += 6; + } + if (i < count - 3) + { + i += 4; + *(output) = (byte)*(input); + *(output + 1) = (byte)*(input + 1); + *(output + 2) = (byte)*(input + 2); + *(output + 3) = (byte)*(input + 3); + output += 4; + input += 4; + } + while (i < count) + { + i++; + *output = (byte)*input; + output++; + input++; + } } } } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs index 21a1aaf778..af8d334d27 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs @@ -66,7 +66,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking for (var index = 0; index < nBuffers; index++) { var blockStart = block == start.Block ? start.Index : block.Data.Offset; - var blockEnd = block == end.Block ? end.Index : block.Data.Offset + block.Data.Count; + var blockEnd = block == end.Block ? end.Index : block.BlockEndOffset; // create and pin each segment being written pBuffers[index] = Libuv.buf_init( diff --git a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs index 2cbd077180..6edb835841 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs @@ -248,7 +248,7 @@ namespace Microsoft.AspNet.Server.KestrelTests Assert.Null(block1.Next); - end.CopyFrom(new ArraySegment(buffer)); + end.CopyFrom(buffer, 0, buffer.Length); Assert.NotNull(block1.Next); diff --git a/test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs b/test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs index bda6bfb651..927194ac5c 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs @@ -308,11 +308,11 @@ namespace Microsoft.AspNet.Server.KestrelTests // block 1 var start = socketOutput.ProducingStart(); - start.Block.End = start.Block.Data.Offset + start.Block.Data.Count; + start.Block.End = start.Block.BlockEndOffset; // block 2 var block2 = memory.Lease(); - block2.End = block2.Data.Offset + block2.Data.Count; + block2.End = block2.BlockEndOffset; start.Block.Next = block2; var end = new MemoryPoolIterator2(block2, block2.End);