From c65734667a3f33977af0134e9710a079abf40550 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 22 Mar 2017 02:16:28 -0700 Subject: [PATCH] Optimize calls into StreamSocketOuput to WriteFast further (#1538) * Optimize calls into StreamSocketOuput to WriteFast further - Added overloads to array, offset, length to avoid implicit conversions to ReadOnlySpan. - Use similar optimizations for multi buffer writes for strings and ints - Use ref locals in multi write instead of pointers and pinning --- .../Adapter/Internal/StreamSocketOutput.cs | 5 +- .../Internal/Http/FrameHeaders.Generated.cs | 72 +++++------ .../Internal/Http/PipelineExtensions.cs | 118 ++++++++++-------- tools/CodeGenerator/KnownHeaders.cs | 4 +- 4 files changed, 109 insertions(+), 90 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs index 92a32c2a28..34ef6ae317 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs @@ -118,8 +118,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Adapter.Internal { await _outputStream.FlushAsync(); } - - if (buffer.IsSingleSpan) + else if (buffer.IsSingleSpan) { var array = buffer.First.GetArray(); await _outputStream.WriteAsync(array.Array, array.Offset, array.Count); @@ -135,7 +134,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Adapter.Internal } finally { - _pipe.Reader.Advance(readResult.Buffer.End); + _pipe.Reader.Advance(buffer.End); } } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs index ff13b9383c..c4c43f0eb2 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs @@ -7771,7 +7771,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Connection[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 17, 14)); + output.WriteFast(_headerBytes, 17, 14); output.WriteAscii(value); } } @@ -7797,7 +7797,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Date[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 31, 8)); + output.WriteFast(_headerBytes, 31, 8); output.WriteAscii(value); } } @@ -7818,7 +7818,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentType[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 133, 16)); + output.WriteFast(_headerBytes, 133, 16); output.WriteAscii(value); } } @@ -7844,7 +7844,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Server[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 350, 10)); + output.WriteFast(_headerBytes, 350, 10); output.WriteAscii(value); } } @@ -7858,7 +7858,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } if ((tempBits & -9223372036854775808L) != 0) { - output.WriteFast(new Span(_headerBytes, 592, 18)); + output.WriteFast(_headerBytes, 592, 18); output.WriteNumeric((ulong)ContentLength.Value); if((tempBits & ~-9223372036854775808L) == 0) @@ -7876,7 +7876,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._CacheControl[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 0, 17)); + output.WriteFast(_headerBytes, 0, 17); output.WriteAscii(value); } } @@ -7897,7 +7897,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._KeepAlive[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 39, 14)); + output.WriteFast(_headerBytes, 39, 14); output.WriteAscii(value); } } @@ -7918,7 +7918,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Pragma[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 53, 10)); + output.WriteFast(_headerBytes, 53, 10); output.WriteAscii(value); } } @@ -7939,7 +7939,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Trailer[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 63, 11)); + output.WriteFast(_headerBytes, 63, 11); output.WriteAscii(value); } } @@ -7965,7 +7965,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._TransferEncoding[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 74, 21)); + output.WriteFast(_headerBytes, 74, 21); output.WriteAscii(value); } } @@ -7986,7 +7986,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Upgrade[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 95, 11)); + output.WriteFast(_headerBytes, 95, 11); output.WriteAscii(value); } } @@ -8007,7 +8007,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Via[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 106, 7)); + output.WriteFast(_headerBytes, 106, 7); output.WriteAscii(value); } } @@ -8028,7 +8028,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Warning[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 113, 11)); + output.WriteFast(_headerBytes, 113, 11); output.WriteAscii(value); } } @@ -8049,7 +8049,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Allow[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 124, 9)); + output.WriteFast(_headerBytes, 124, 9); output.WriteAscii(value); } } @@ -8070,7 +8070,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentEncoding[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 149, 20)); + output.WriteFast(_headerBytes, 149, 20); output.WriteAscii(value); } } @@ -8091,7 +8091,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentLanguage[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 169, 20)); + output.WriteFast(_headerBytes, 169, 20); output.WriteAscii(value); } } @@ -8112,7 +8112,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentLocation[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 189, 20)); + output.WriteFast(_headerBytes, 189, 20); output.WriteAscii(value); } } @@ -8133,7 +8133,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentMD5[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 209, 15)); + output.WriteFast(_headerBytes, 209, 15); output.WriteAscii(value); } } @@ -8154,7 +8154,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentRange[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 224, 17)); + output.WriteFast(_headerBytes, 224, 17); output.WriteAscii(value); } } @@ -8175,7 +8175,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Expires[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 241, 11)); + output.WriteFast(_headerBytes, 241, 11); output.WriteAscii(value); } } @@ -8196,7 +8196,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._LastModified[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 252, 17)); + output.WriteFast(_headerBytes, 252, 17); output.WriteAscii(value); } } @@ -8217,7 +8217,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AcceptRanges[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 269, 17)); + output.WriteFast(_headerBytes, 269, 17); output.WriteAscii(value); } } @@ -8238,7 +8238,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Age[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 286, 7)); + output.WriteFast(_headerBytes, 286, 7); output.WriteAscii(value); } } @@ -8259,7 +8259,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ETag[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 293, 8)); + output.WriteFast(_headerBytes, 293, 8); output.WriteAscii(value); } } @@ -8280,7 +8280,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Location[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 301, 12)); + output.WriteFast(_headerBytes, 301, 12); output.WriteAscii(value); } } @@ -8301,7 +8301,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ProxyAuthenticate[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 313, 22)); + output.WriteFast(_headerBytes, 313, 22); output.WriteAscii(value); } } @@ -8322,7 +8322,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._RetryAfter[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 335, 15)); + output.WriteFast(_headerBytes, 335, 15); output.WriteAscii(value); } } @@ -8343,7 +8343,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._SetCookie[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 360, 14)); + output.WriteFast(_headerBytes, 360, 14); output.WriteAscii(value); } } @@ -8364,7 +8364,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Vary[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 374, 8)); + output.WriteFast(_headerBytes, 374, 8); output.WriteAscii(value); } } @@ -8385,7 +8385,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._WWWAuthenticate[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 382, 20)); + output.WriteFast(_headerBytes, 382, 20); output.WriteAscii(value); } } @@ -8406,7 +8406,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowCredentials[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 402, 36)); + output.WriteFast(_headerBytes, 402, 36); output.WriteAscii(value); } } @@ -8427,7 +8427,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowHeaders[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 438, 32)); + output.WriteFast(_headerBytes, 438, 32); output.WriteAscii(value); } } @@ -8448,7 +8448,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowMethods[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 470, 32)); + output.WriteFast(_headerBytes, 470, 32); output.WriteAscii(value); } } @@ -8469,7 +8469,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowOrigin[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 502, 31)); + output.WriteFast(_headerBytes, 502, 31); output.WriteAscii(value); } } @@ -8490,7 +8490,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlExposeHeaders[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 533, 33)); + output.WriteFast(_headerBytes, 533, 33); output.WriteAscii(value); } } @@ -8511,7 +8511,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlMaxAge[i]; if (value != null) { - output.WriteFast(new Span(_headerBytes, 566, 26)); + output.WriteFast(_headerBytes, 566, 26); output.WriteAscii(value); } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs index d3bbe8a46c..6396b56781 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs @@ -94,7 +94,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } // Temporary until the fast write implementation propagates from corefx - public unsafe static void WriteFast(this WritableBuffer buffer, ReadOnlySpan source) + public unsafe static void WriteFast(this WritableBuffer buffer, byte[] source) + { + buffer.WriteFast(source, 0, source.Length); + } + + public unsafe static void WriteFast(this WritableBuffer buffer, ArraySegment source) + { + buffer.WriteFast(source.Array, source.Offset, source.Count); + } + + public unsafe static void WriteFast(this WritableBuffer buffer, byte[] source, int offset, int length) { var dest = buffer.Buffer.Span; var destLength = dest.Length; @@ -108,74 +118,79 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http destLength = dest.Length; } - var sourceLength = source.Length; + var sourceLength = length; if (sourceLength <= destLength) { - ref byte pSource = ref source.DangerousGetPinnableReference(); + ref byte pSource = ref source[offset]; ref byte pDest = ref dest.DangerousGetPinnableReference(); Unsafe.CopyBlockUnaligned(ref pDest, ref pSource, (uint)sourceLength); buffer.Advance(sourceLength); return; } - buffer.WriteMultiBuffer(source); + buffer.WriteMultiBuffer(source, offset, length); } - private static unsafe void WriteMultiBuffer(this WritableBuffer buffer, ReadOnlySpan source) + private static unsafe void WriteMultiBuffer(this WritableBuffer buffer, byte[] source, int offset, int length) { - var remaining = source.Length; - var offset = 0; + var remaining = length; - fixed (byte* pSource = &source.DangerousGetPinnableReference()) + while (remaining > 0) { - while (remaining > 0) + var writable = Math.Min(remaining, buffer.Buffer.Length); + + buffer.Ensure(writable); + + if (writable == 0) { - var writable = Math.Min(remaining, buffer.Buffer.Length); - - buffer.Ensure(writable); - - if (writable == 0) - { - continue; - } - - fixed (byte* pDest = &buffer.Buffer.Span.DangerousGetPinnableReference()) - { - Unsafe.CopyBlockUnaligned(pDest, pSource + offset, (uint)writable); - } - - remaining -= writable; - offset += writable; - - buffer.Advance(writable); + continue; } + + ref byte pSource = ref source[offset]; + ref byte pDest = ref buffer.Buffer.Span.DangerousGetPinnableReference(); + + Unsafe.CopyBlockUnaligned(ref pDest, ref pSource, (uint)writable); + + remaining -= writable; + offset += writable; + + buffer.Advance(writable); } } public unsafe static void WriteAscii(this WritableBuffer buffer, string data) { - if (!string.IsNullOrEmpty(data)) + if (string.IsNullOrEmpty(data)) { - if (buffer.Buffer.IsEmpty) + return; + } + + var dest = buffer.Buffer.Span; + var destLength = dest.Length; + var sourceLength = data.Length; + + if (destLength == 0) + { + buffer.Ensure(); + + dest = buffer.Buffer.Span; + destLength = dest.Length; + } + + // Fast path, try copying to the available memory directly + if (sourceLength <= destLength) + { + fixed (char* input = data) + fixed (byte* output = &dest.DangerousGetPinnableReference()) { - buffer.Ensure(); + EncodeAsciiCharsToBytes(input, output, sourceLength); } - // Fast path, try copying to the available memory directly - if (data.Length <= buffer.Buffer.Length) - { - fixed (char* input = data) - fixed (byte* output = &buffer.Buffer.Span.DangerousGetPinnableReference()) - { - EncodeAsciiCharsToBytes(input, output, data.Length); - } - - buffer.Advance(data.Length); - } - else - { - buffer.WriteAsciiMultiWrite(data); - } + buffer.Advance(sourceLength); + } + else + { + buffer.WriteAsciiMultiWrite(data); } } @@ -184,15 +199,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { const byte AsciiDigitStart = (byte)'0'; - if (buffer.Buffer.IsEmpty) + var span = buffer.Buffer.Span; + var bytesLeftInBlock = span.Length; + + if (bytesLeftInBlock == 0) { buffer.Ensure(); + + span = buffer.Buffer.Span; + bytesLeftInBlock = span.Length; } // Fast path, try copying to the available memory directly - var bytesLeftInBlock = buffer.Buffer.Length; var simpleWrite = true; - fixed (byte* output = &buffer.Buffer.Span.DangerousGetPinnableReference()) + fixed (byte* output = &span.DangerousGetPinnableReference()) { var start = output; if (number < 10 && bytesLeftInBlock >= 1) @@ -250,7 +270,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http while (value != 0); var length = _maxULongByteLength - position; - buffer.WriteFast(new ReadOnlySpan(byteBuffer, position, length)); + buffer.WriteFast(new ArraySegment(byteBuffer, position, length)); } [MethodImpl(MethodImplOptions.NoInlining)] diff --git a/tools/CodeGenerator/KnownHeaders.cs b/tools/CodeGenerator/KnownHeaders.cs index f12aea58d6..5d76a53305 100644 --- a/tools/CodeGenerator/KnownHeaders.cs +++ b/tools/CodeGenerator/KnownHeaders.cs @@ -540,7 +540,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._{header.Identifier}[i]; if (value != null) {{ - output.WriteFast(new Span(_headerBytes, {header.BytesOffset}, {header.BytesCount})); + output.WriteFast(_headerBytes, {header.BytesOffset}, {header.BytesCount}); output.WriteAscii(value); }} }} @@ -554,7 +554,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http }}{(header.Identifier == "Server" ? $@" if ((tempBits & {1L << 63}L) != 0) {{ - output.WriteFast(new Span(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount})); + output.WriteFast(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount}); output.WriteNumeric((ulong)ContentLength.Value); if((tempBits & ~{1L << 63}L) == 0)