Optimize calls into StreamSocketOuput to WriteFast further (#1538)

* Optimize calls into StreamSocketOuput to WriteFast further
- Added overloads to array, offset, length to avoid implicit conversions to
ReadOnlySpan.
- Use similar optimizations for multi buffer writes for strings and ints
- Use ref locals in multi write instead of pointers and pinning
This commit is contained in:
David Fowler 2017-03-22 02:16:28 -07:00 committed by GitHub
parent 751a0e2e7e
commit c65734667a
4 changed files with 109 additions and 90 deletions

View File

@ -118,8 +118,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Adapter.Internal
{
await _outputStream.FlushAsync();
}
if (buffer.IsSingleSpan)
else if (buffer.IsSingleSpan)
{
var array = buffer.First.GetArray();
await _outputStream.WriteAsync(array.Array, array.Offset, array.Count);
@ -135,7 +134,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Adapter.Internal
}
finally
{
_pipe.Reader.Advance(readResult.Buffer.End);
_pipe.Reader.Advance(buffer.End);
}
}
}

View File

@ -7771,7 +7771,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Connection[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 17, 14));
output.WriteFast(_headerBytes, 17, 14);
output.WriteAscii(value);
}
}
@ -7797,7 +7797,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Date[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 31, 8));
output.WriteFast(_headerBytes, 31, 8);
output.WriteAscii(value);
}
}
@ -7818,7 +7818,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ContentType[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 133, 16));
output.WriteFast(_headerBytes, 133, 16);
output.WriteAscii(value);
}
}
@ -7844,7 +7844,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Server[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 350, 10));
output.WriteFast(_headerBytes, 350, 10);
output.WriteAscii(value);
}
}
@ -7858,7 +7858,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
}
if ((tempBits & -9223372036854775808L) != 0)
{
output.WriteFast(new Span<byte>(_headerBytes, 592, 18));
output.WriteFast(_headerBytes, 592, 18);
output.WriteNumeric((ulong)ContentLength.Value);
if((tempBits & ~-9223372036854775808L) == 0)
@ -7876,7 +7876,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._CacheControl[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 0, 17));
output.WriteFast(_headerBytes, 0, 17);
output.WriteAscii(value);
}
}
@ -7897,7 +7897,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._KeepAlive[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 39, 14));
output.WriteFast(_headerBytes, 39, 14);
output.WriteAscii(value);
}
}
@ -7918,7 +7918,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Pragma[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 53, 10));
output.WriteFast(_headerBytes, 53, 10);
output.WriteAscii(value);
}
}
@ -7939,7 +7939,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Trailer[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 63, 11));
output.WriteFast(_headerBytes, 63, 11);
output.WriteAscii(value);
}
}
@ -7965,7 +7965,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._TransferEncoding[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 74, 21));
output.WriteFast(_headerBytes, 74, 21);
output.WriteAscii(value);
}
}
@ -7986,7 +7986,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Upgrade[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 95, 11));
output.WriteFast(_headerBytes, 95, 11);
output.WriteAscii(value);
}
}
@ -8007,7 +8007,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Via[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 106, 7));
output.WriteFast(_headerBytes, 106, 7);
output.WriteAscii(value);
}
}
@ -8028,7 +8028,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Warning[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 113, 11));
output.WriteFast(_headerBytes, 113, 11);
output.WriteAscii(value);
}
}
@ -8049,7 +8049,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Allow[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 124, 9));
output.WriteFast(_headerBytes, 124, 9);
output.WriteAscii(value);
}
}
@ -8070,7 +8070,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ContentEncoding[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 149, 20));
output.WriteFast(_headerBytes, 149, 20);
output.WriteAscii(value);
}
}
@ -8091,7 +8091,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ContentLanguage[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 169, 20));
output.WriteFast(_headerBytes, 169, 20);
output.WriteAscii(value);
}
}
@ -8112,7 +8112,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ContentLocation[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 189, 20));
output.WriteFast(_headerBytes, 189, 20);
output.WriteAscii(value);
}
}
@ -8133,7 +8133,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ContentMD5[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 209, 15));
output.WriteFast(_headerBytes, 209, 15);
output.WriteAscii(value);
}
}
@ -8154,7 +8154,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ContentRange[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 224, 17));
output.WriteFast(_headerBytes, 224, 17);
output.WriteAscii(value);
}
}
@ -8175,7 +8175,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Expires[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 241, 11));
output.WriteFast(_headerBytes, 241, 11);
output.WriteAscii(value);
}
}
@ -8196,7 +8196,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._LastModified[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 252, 17));
output.WriteFast(_headerBytes, 252, 17);
output.WriteAscii(value);
}
}
@ -8217,7 +8217,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AcceptRanges[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 269, 17));
output.WriteFast(_headerBytes, 269, 17);
output.WriteAscii(value);
}
}
@ -8238,7 +8238,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Age[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 286, 7));
output.WriteFast(_headerBytes, 286, 7);
output.WriteAscii(value);
}
}
@ -8259,7 +8259,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ETag[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 293, 8));
output.WriteFast(_headerBytes, 293, 8);
output.WriteAscii(value);
}
}
@ -8280,7 +8280,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Location[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 301, 12));
output.WriteFast(_headerBytes, 301, 12);
output.WriteAscii(value);
}
}
@ -8301,7 +8301,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._ProxyAuthenticate[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 313, 22));
output.WriteFast(_headerBytes, 313, 22);
output.WriteAscii(value);
}
}
@ -8322,7 +8322,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._RetryAfter[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 335, 15));
output.WriteFast(_headerBytes, 335, 15);
output.WriteAscii(value);
}
}
@ -8343,7 +8343,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._SetCookie[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 360, 14));
output.WriteFast(_headerBytes, 360, 14);
output.WriteAscii(value);
}
}
@ -8364,7 +8364,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._Vary[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 374, 8));
output.WriteFast(_headerBytes, 374, 8);
output.WriteAscii(value);
}
}
@ -8385,7 +8385,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._WWWAuthenticate[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 382, 20));
output.WriteFast(_headerBytes, 382, 20);
output.WriteAscii(value);
}
}
@ -8406,7 +8406,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AccessControlAllowCredentials[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 402, 36));
output.WriteFast(_headerBytes, 402, 36);
output.WriteAscii(value);
}
}
@ -8427,7 +8427,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AccessControlAllowHeaders[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 438, 32));
output.WriteFast(_headerBytes, 438, 32);
output.WriteAscii(value);
}
}
@ -8448,7 +8448,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AccessControlAllowMethods[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 470, 32));
output.WriteFast(_headerBytes, 470, 32);
output.WriteAscii(value);
}
}
@ -8469,7 +8469,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AccessControlAllowOrigin[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 502, 31));
output.WriteFast(_headerBytes, 502, 31);
output.WriteAscii(value);
}
}
@ -8490,7 +8490,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AccessControlExposeHeaders[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 533, 33));
output.WriteFast(_headerBytes, 533, 33);
output.WriteAscii(value);
}
}
@ -8511,7 +8511,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._AccessControlMaxAge[i];
if (value != null)
{
output.WriteFast(new Span<byte>(_headerBytes, 566, 26));
output.WriteFast(_headerBytes, 566, 26);
output.WriteAscii(value);
}
}

View File

@ -94,7 +94,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
}
// Temporary until the fast write implementation propagates from corefx
public unsafe static void WriteFast(this WritableBuffer buffer, ReadOnlySpan<byte> source)
public unsafe static void WriteFast(this WritableBuffer buffer, byte[] source)
{
buffer.WriteFast(source, 0, source.Length);
}
public unsafe static void WriteFast(this WritableBuffer buffer, ArraySegment<byte> source)
{
buffer.WriteFast(source.Array, source.Offset, source.Count);
}
public unsafe static void WriteFast(this WritableBuffer buffer, byte[] source, int offset, int length)
{
var dest = buffer.Buffer.Span;
var destLength = dest.Length;
@ -108,74 +118,79 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
destLength = dest.Length;
}
var sourceLength = source.Length;
var sourceLength = length;
if (sourceLength <= destLength)
{
ref byte pSource = ref source.DangerousGetPinnableReference();
ref byte pSource = ref source[offset];
ref byte pDest = ref dest.DangerousGetPinnableReference();
Unsafe.CopyBlockUnaligned(ref pDest, ref pSource, (uint)sourceLength);
buffer.Advance(sourceLength);
return;
}
buffer.WriteMultiBuffer(source);
buffer.WriteMultiBuffer(source, offset, length);
}
private static unsafe void WriteMultiBuffer(this WritableBuffer buffer, ReadOnlySpan<byte> source)
private static unsafe void WriteMultiBuffer(this WritableBuffer buffer, byte[] source, int offset, int length)
{
var remaining = source.Length;
var offset = 0;
var remaining = length;
fixed (byte* pSource = &source.DangerousGetPinnableReference())
while (remaining > 0)
{
while (remaining > 0)
var writable = Math.Min(remaining, buffer.Buffer.Length);
buffer.Ensure(writable);
if (writable == 0)
{
var writable = Math.Min(remaining, buffer.Buffer.Length);
buffer.Ensure(writable);
if (writable == 0)
{
continue;
}
fixed (byte* pDest = &buffer.Buffer.Span.DangerousGetPinnableReference())
{
Unsafe.CopyBlockUnaligned(pDest, pSource + offset, (uint)writable);
}
remaining -= writable;
offset += writable;
buffer.Advance(writable);
continue;
}
ref byte pSource = ref source[offset];
ref byte pDest = ref buffer.Buffer.Span.DangerousGetPinnableReference();
Unsafe.CopyBlockUnaligned(ref pDest, ref pSource, (uint)writable);
remaining -= writable;
offset += writable;
buffer.Advance(writable);
}
}
public unsafe static void WriteAscii(this WritableBuffer buffer, string data)
{
if (!string.IsNullOrEmpty(data))
if (string.IsNullOrEmpty(data))
{
if (buffer.Buffer.IsEmpty)
return;
}
var dest = buffer.Buffer.Span;
var destLength = dest.Length;
var sourceLength = data.Length;
if (destLength == 0)
{
buffer.Ensure();
dest = buffer.Buffer.Span;
destLength = dest.Length;
}
// Fast path, try copying to the available memory directly
if (sourceLength <= destLength)
{
fixed (char* input = data)
fixed (byte* output = &dest.DangerousGetPinnableReference())
{
buffer.Ensure();
EncodeAsciiCharsToBytes(input, output, sourceLength);
}
// Fast path, try copying to the available memory directly
if (data.Length <= buffer.Buffer.Length)
{
fixed (char* input = data)
fixed (byte* output = &buffer.Buffer.Span.DangerousGetPinnableReference())
{
EncodeAsciiCharsToBytes(input, output, data.Length);
}
buffer.Advance(data.Length);
}
else
{
buffer.WriteAsciiMultiWrite(data);
}
buffer.Advance(sourceLength);
}
else
{
buffer.WriteAsciiMultiWrite(data);
}
}
@ -184,15 +199,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
{
const byte AsciiDigitStart = (byte)'0';
if (buffer.Buffer.IsEmpty)
var span = buffer.Buffer.Span;
var bytesLeftInBlock = span.Length;
if (bytesLeftInBlock == 0)
{
buffer.Ensure();
span = buffer.Buffer.Span;
bytesLeftInBlock = span.Length;
}
// Fast path, try copying to the available memory directly
var bytesLeftInBlock = buffer.Buffer.Length;
var simpleWrite = true;
fixed (byte* output = &buffer.Buffer.Span.DangerousGetPinnableReference())
fixed (byte* output = &span.DangerousGetPinnableReference())
{
var start = output;
if (number < 10 && bytesLeftInBlock >= 1)
@ -250,7 +270,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
while (value != 0);
var length = _maxULongByteLength - position;
buffer.WriteFast(new ReadOnlySpan<byte>(byteBuffer, position, length));
buffer.WriteFast(new ArraySegment<byte>(byteBuffer, position, length));
}
[MethodImpl(MethodImplOptions.NoInlining)]

View File

@ -540,7 +540,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
var value = _headers._{header.Identifier}[i];
if (value != null)
{{
output.WriteFast(new Span<byte>(_headerBytes, {header.BytesOffset}, {header.BytesCount}));
output.WriteFast(_headerBytes, {header.BytesOffset}, {header.BytesCount});
output.WriteAscii(value);
}}
}}
@ -554,7 +554,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
}}{(header.Identifier == "Server" ? $@"
if ((tempBits & {1L << 63}L) != 0)
{{
output.WriteFast(new Span<byte>(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount}));
output.WriteFast(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount});
output.WriteNumeric((ulong)ContentLength.Value);
if((tempBits & ~{1L << 63}L) == 0)