diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs index 0e17b99ae1..ed1d069f59 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Adapter/Internal/StreamSocketOutput.cs @@ -48,12 +48,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Adapter.Internal if (chunk) { ChunkWriter.WriteBeginChunkBytes(ref writableBuffer, buffer.Count); - writableBuffer.Write(buffer); + writableBuffer.WriteFast(buffer); ChunkWriter.WriteEndChunkBytes(ref writableBuffer); } else { - writableBuffer.Write(buffer); + writableBuffer.WriteFast(buffer); } } @@ -119,8 +119,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Adapter.Internal { _pipe.Reader.Advance(readResult.Buffer.End); } - - // REVIEW: Should we flush here? } } finally diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/ChunkWriter.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/ChunkWriter.cs index 9846d42a04..8b69ba4603 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/ChunkWriter.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/ChunkWriter.cs @@ -51,13 +51,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http public static int WriteBeginChunkBytes(ref WritableBuffer start, int dataCount) { var chunkSegment = BeginChunkBytes(dataCount); - start.Write(chunkSegment); + start.WriteFast(chunkSegment); return chunkSegment.Count; } public static void WriteEndChunkBytes(ref WritableBuffer start) { - start.Write(_endChunkBytes); + start.WriteFast(_endChunkBytes); } } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs index f5d1b49b84..8c5f032a1e 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs @@ -891,8 +891,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var hasTransferEncoding = responseHeaders.HasTransferEncoding; var transferCoding = FrameHeaders.GetFinalTransferCoding(responseHeaders.HeaderTransferEncoding); - var end = Output.Alloc(); - if (_keepAlive && hasConnection) { _keepAlive = (connectionOptions & ConnectionOptions.KeepAlive) == ConnectionOptions.KeepAlive; @@ -974,12 +972,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http responseHeaders.SetRawDate(dateHeaderValues.String, dateHeaderValues.Bytes); } - end.Write(_bytesHttpVersion11); - end.Write(statusBytes); - responseHeaders.CopyTo(ref end); - end.Write(_bytesEndHeaders); - - end.Commit(); + var writableBuffer = Output.Alloc(); + writableBuffer.WriteFast(_bytesHttpVersion11); + writableBuffer.WriteFast(statusBytes); + responseHeaders.CopyTo(ref writableBuffer); + writableBuffer.WriteFast(_bytesEndHeaders); + writableBuffer.Commit(); } public void ParseRequest(ReadableBuffer buffer, out ReadCursor consumed, out ReadCursor examined) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs index da9147f69f..ff13b9383c 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameHeaders.Generated.cs @@ -7761,7 +7761,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { if (_headers._rawConnection != null) { - output.Write(_headers._rawConnection); + output.WriteFast(_headers._rawConnection); } else { @@ -7771,7 +7771,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Connection[i]; if (value != null) { - output.Write(new Span(_headerBytes, 17, 14)); + output.WriteFast(new Span(_headerBytes, 17, 14)); output.WriteAscii(value); } } @@ -7787,7 +7787,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { if (_headers._rawDate != null) { - output.Write(_headers._rawDate); + output.WriteFast(_headers._rawDate); } else { @@ -7797,7 +7797,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Date[i]; if (value != null) { - output.Write(new Span(_headerBytes, 31, 8)); + output.WriteFast(new Span(_headerBytes, 31, 8)); output.WriteAscii(value); } } @@ -7818,7 +7818,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentType[i]; if (value != null) { - output.Write(new Span(_headerBytes, 133, 16)); + output.WriteFast(new Span(_headerBytes, 133, 16)); output.WriteAscii(value); } } @@ -7834,7 +7834,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { if (_headers._rawServer != null) { - output.Write(_headers._rawServer); + output.WriteFast(_headers._rawServer); } else { @@ -7844,7 +7844,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Server[i]; if (value != null) { - output.Write(new Span(_headerBytes, 350, 10)); + output.WriteFast(new Span(_headerBytes, 350, 10)); output.WriteAscii(value); } } @@ -7858,7 +7858,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http } if ((tempBits & -9223372036854775808L) != 0) { - output.Write(new Span(_headerBytes, 592, 18)); + output.WriteFast(new Span(_headerBytes, 592, 18)); output.WriteNumeric((ulong)ContentLength.Value); if((tempBits & ~-9223372036854775808L) == 0) @@ -7876,7 +7876,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._CacheControl[i]; if (value != null) { - output.Write(new Span(_headerBytes, 0, 17)); + output.WriteFast(new Span(_headerBytes, 0, 17)); output.WriteAscii(value); } } @@ -7897,7 +7897,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._KeepAlive[i]; if (value != null) { - output.Write(new Span(_headerBytes, 39, 14)); + output.WriteFast(new Span(_headerBytes, 39, 14)); output.WriteAscii(value); } } @@ -7918,7 +7918,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Pragma[i]; if (value != null) { - output.Write(new Span(_headerBytes, 53, 10)); + output.WriteFast(new Span(_headerBytes, 53, 10)); output.WriteAscii(value); } } @@ -7939,7 +7939,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Trailer[i]; if (value != null) { - output.Write(new Span(_headerBytes, 63, 11)); + output.WriteFast(new Span(_headerBytes, 63, 11)); output.WriteAscii(value); } } @@ -7955,7 +7955,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { if (_headers._rawTransferEncoding != null) { - output.Write(_headers._rawTransferEncoding); + output.WriteFast(_headers._rawTransferEncoding); } else { @@ -7965,7 +7965,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._TransferEncoding[i]; if (value != null) { - output.Write(new Span(_headerBytes, 74, 21)); + output.WriteFast(new Span(_headerBytes, 74, 21)); output.WriteAscii(value); } } @@ -7986,7 +7986,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Upgrade[i]; if (value != null) { - output.Write(new Span(_headerBytes, 95, 11)); + output.WriteFast(new Span(_headerBytes, 95, 11)); output.WriteAscii(value); } } @@ -8007,7 +8007,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Via[i]; if (value != null) { - output.Write(new Span(_headerBytes, 106, 7)); + output.WriteFast(new Span(_headerBytes, 106, 7)); output.WriteAscii(value); } } @@ -8028,7 +8028,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Warning[i]; if (value != null) { - output.Write(new Span(_headerBytes, 113, 11)); + output.WriteFast(new Span(_headerBytes, 113, 11)); output.WriteAscii(value); } } @@ -8049,7 +8049,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Allow[i]; if (value != null) { - output.Write(new Span(_headerBytes, 124, 9)); + output.WriteFast(new Span(_headerBytes, 124, 9)); output.WriteAscii(value); } } @@ -8070,7 +8070,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentEncoding[i]; if (value != null) { - output.Write(new Span(_headerBytes, 149, 20)); + output.WriteFast(new Span(_headerBytes, 149, 20)); output.WriteAscii(value); } } @@ -8091,7 +8091,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentLanguage[i]; if (value != null) { - output.Write(new Span(_headerBytes, 169, 20)); + output.WriteFast(new Span(_headerBytes, 169, 20)); output.WriteAscii(value); } } @@ -8112,7 +8112,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentLocation[i]; if (value != null) { - output.Write(new Span(_headerBytes, 189, 20)); + output.WriteFast(new Span(_headerBytes, 189, 20)); output.WriteAscii(value); } } @@ -8133,7 +8133,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentMD5[i]; if (value != null) { - output.Write(new Span(_headerBytes, 209, 15)); + output.WriteFast(new Span(_headerBytes, 209, 15)); output.WriteAscii(value); } } @@ -8154,7 +8154,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ContentRange[i]; if (value != null) { - output.Write(new Span(_headerBytes, 224, 17)); + output.WriteFast(new Span(_headerBytes, 224, 17)); output.WriteAscii(value); } } @@ -8175,7 +8175,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Expires[i]; if (value != null) { - output.Write(new Span(_headerBytes, 241, 11)); + output.WriteFast(new Span(_headerBytes, 241, 11)); output.WriteAscii(value); } } @@ -8196,7 +8196,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._LastModified[i]; if (value != null) { - output.Write(new Span(_headerBytes, 252, 17)); + output.WriteFast(new Span(_headerBytes, 252, 17)); output.WriteAscii(value); } } @@ -8217,7 +8217,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AcceptRanges[i]; if (value != null) { - output.Write(new Span(_headerBytes, 269, 17)); + output.WriteFast(new Span(_headerBytes, 269, 17)); output.WriteAscii(value); } } @@ -8238,7 +8238,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Age[i]; if (value != null) { - output.Write(new Span(_headerBytes, 286, 7)); + output.WriteFast(new Span(_headerBytes, 286, 7)); output.WriteAscii(value); } } @@ -8259,7 +8259,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ETag[i]; if (value != null) { - output.Write(new Span(_headerBytes, 293, 8)); + output.WriteFast(new Span(_headerBytes, 293, 8)); output.WriteAscii(value); } } @@ -8280,7 +8280,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Location[i]; if (value != null) { - output.Write(new Span(_headerBytes, 301, 12)); + output.WriteFast(new Span(_headerBytes, 301, 12)); output.WriteAscii(value); } } @@ -8301,7 +8301,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._ProxyAuthenticate[i]; if (value != null) { - output.Write(new Span(_headerBytes, 313, 22)); + output.WriteFast(new Span(_headerBytes, 313, 22)); output.WriteAscii(value); } } @@ -8322,7 +8322,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._RetryAfter[i]; if (value != null) { - output.Write(new Span(_headerBytes, 335, 15)); + output.WriteFast(new Span(_headerBytes, 335, 15)); output.WriteAscii(value); } } @@ -8343,7 +8343,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._SetCookie[i]; if (value != null) { - output.Write(new Span(_headerBytes, 360, 14)); + output.WriteFast(new Span(_headerBytes, 360, 14)); output.WriteAscii(value); } } @@ -8364,7 +8364,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._Vary[i]; if (value != null) { - output.Write(new Span(_headerBytes, 374, 8)); + output.WriteFast(new Span(_headerBytes, 374, 8)); output.WriteAscii(value); } } @@ -8385,7 +8385,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._WWWAuthenticate[i]; if (value != null) { - output.Write(new Span(_headerBytes, 382, 20)); + output.WriteFast(new Span(_headerBytes, 382, 20)); output.WriteAscii(value); } } @@ -8406,7 +8406,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowCredentials[i]; if (value != null) { - output.Write(new Span(_headerBytes, 402, 36)); + output.WriteFast(new Span(_headerBytes, 402, 36)); output.WriteAscii(value); } } @@ -8427,7 +8427,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowHeaders[i]; if (value != null) { - output.Write(new Span(_headerBytes, 438, 32)); + output.WriteFast(new Span(_headerBytes, 438, 32)); output.WriteAscii(value); } } @@ -8448,7 +8448,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowMethods[i]; if (value != null) { - output.Write(new Span(_headerBytes, 470, 32)); + output.WriteFast(new Span(_headerBytes, 470, 32)); output.WriteAscii(value); } } @@ -8469,7 +8469,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlAllowOrigin[i]; if (value != null) { - output.Write(new Span(_headerBytes, 502, 31)); + output.WriteFast(new Span(_headerBytes, 502, 31)); output.WriteAscii(value); } } @@ -8490,7 +8490,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlExposeHeaders[i]; if (value != null) { - output.Write(new Span(_headerBytes, 533, 33)); + output.WriteFast(new Span(_headerBytes, 533, 33)); output.WriteAscii(value); } } @@ -8511,7 +8511,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._AccessControlMaxAge[i]; if (value != null) { - output.Write(new Span(_headerBytes, 566, 26)); + output.WriteFast(new Span(_headerBytes, 566, 26)); output.WriteAscii(value); } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs index 439a76bd6a..ea51f47d10 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs @@ -46,9 +46,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { if (value != null) { - output.Write(_CrLf); + output.WriteFast(_CrLf); output.WriteAscii(kv.Key); - output.Write(_colonSpace); + output.WriteFast(_colonSpace); output.WriteAscii(value); } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs index 0ab6f20a68..d986106d03 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs @@ -95,6 +95,65 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http return result; } + // Temporary until the fast write implementation propagates from corefx + public unsafe static void WriteFast(this WritableBuffer buffer, ReadOnlySpan source) + { + var dest = buffer.Memory.Span; + var destLength = dest.Length; + + if (destLength == 0) + { + buffer.Ensure(); + + // Get the new span and length + dest = buffer.Memory.Span; + destLength = dest.Length; + } + + var sourceLength = source.Length; + if (sourceLength <= destLength) + { + ref byte pSource = ref source.DangerousGetPinnableReference(); + ref byte pDest = ref dest.DangerousGetPinnableReference(); + Unsafe.CopyBlockUnaligned(ref pDest, ref pSource, (uint)sourceLength); + buffer.Advance(sourceLength); + return; + } + + buffer.WriteMultiBuffer(source); + } + + private static unsafe void WriteMultiBuffer(this WritableBuffer buffer, ReadOnlySpan source) + { + var remaining = source.Length; + var offset = 0; + + fixed (byte* pSource = &source.DangerousGetPinnableReference()) + { + while (remaining > 0) + { + var writable = Math.Min(remaining, buffer.Memory.Length); + + buffer.Ensure(writable); + + if (writable == 0) + { + continue; + } + + fixed (byte* pDest = &buffer.Memory.Span.DangerousGetPinnableReference()) + { + Unsafe.CopyBlockUnaligned(pDest, pSource + offset, (uint)writable); + } + + remaining -= writable; + offset += writable; + + buffer.Advance(writable); + } + } + } + public unsafe static void WriteAscii(this WritableBuffer buffer, string data) { if (!string.IsNullOrEmpty(data)) @@ -193,7 +252,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http while (value != 0); var length = _maxULongByteLength - position; - buffer.Write(new ReadOnlySpan(byteBuffer, position, length)); + buffer.WriteFast(new ReadOnlySpan(byteBuffer, position, length)); } [MethodImpl(MethodImplOptions.NoInlining)] @@ -274,7 +333,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http i += 4; } - trailing: + trailing: for (; i < length; i++) { char ch = *(input + i); diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/WritableBufferExtensions.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/WritableBufferExtensions.cs deleted file mode 100644 index 04a538e4be..0000000000 --- a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/WritableBufferExtensions.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO.Pipelines; -using System.Text; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - internal static class WritableBufferExtensions - { - public static void WriteFast(this WritableBuffer buffer, ReadOnlySpan source) - { - if (buffer.Memory.IsEmpty) - { - buffer.Ensure(); - } - - // Fast path, try copying to the available memory directly - if (source.Length <= buffer.Memory.Length) - { - source.CopyToFast(buffer.Memory.Span); - buffer.Advance(source.Length); - return; - } - - var remaining = source.Length; - var offset = 0; - - while (remaining > 0) - { - var writable = Math.Min(remaining, buffer.Memory.Length); - - buffer.Ensure(writable); - - if (writable == 0) - { - continue; - } - - source.Slice(offset, writable).CopyToFast(buffer.Memory.Span); - - remaining -= writable; - offset += writable; - - buffer.Advance(writable); - } - } - - private unsafe static void CopyToFast(this ReadOnlySpan source, Span destination) - { - if (destination.Length < source.Length) - { - throw new InvalidOperationException(); - } - - // Assume it fits - fixed (byte* pSource = &source.DangerousGetPinnableReference()) - fixed (byte* pDest = &destination.DangerousGetPinnableReference()) - { - Buffer.MemoryCopy(pSource, pDest, destination.Length, source.Length); - } - } - } -} diff --git a/tools/CodeGenerator/KnownHeaders.cs b/tools/CodeGenerator/KnownHeaders.cs index dee9665889..f12aea58d6 100644 --- a/tools/CodeGenerator/KnownHeaders.cs +++ b/tools/CodeGenerator/KnownHeaders.cs @@ -530,7 +530,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http {{ {(header.EnhancedSetter == false ? "" : $@" if (_headers._raw{header.Identifier} != null) {{ - output.Write(_headers._raw{header.Identifier}); + output.WriteFast(_headers._raw{header.Identifier}); }} else ")} {{ @@ -540,7 +540,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http var value = _headers._{header.Identifier}[i]; if (value != null) {{ - output.Write(new Span(_headerBytes, {header.BytesOffset}, {header.BytesCount})); + output.WriteFast(new Span(_headerBytes, {header.BytesOffset}, {header.BytesCount})); output.WriteAscii(value); }} }} @@ -554,7 +554,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http }}{(header.Identifier == "Server" ? $@" if ((tempBits & {1L << 63}L) != 0) {{ - output.Write(new Span(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount})); + output.WriteFast(new Span(_headerBytes, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesOffset}, {loop.Headers.First(x => x.Identifier == "ContentLength").BytesCount})); output.WriteNumeric((ulong)ContentLength.Value); if((tempBits & ~{1L << 63}L) == 0)