Work around potential race in PipeWriter Redux (#11065)

- Make sure we always await the last flush task before calling FlushAsync
  again instead of preemptively calling FlushAsync and checking to see
  if the ValueTask is incomplete before bothering to acquire the _flushLock
- This now acquires the _flushLock fore every call to Response.Body.Write
  whereas this only happened for truly async writes before.
- I don't think this is a big concern since this should normally be uncontested,
  and DefaultPipeWriter.FlushAsync/GetResult already acquire a lock.
This commit is contained in:
Stephen Halter 2019-06-13 20:03:29 -07:00 committed by GitHub
parent 902369610a
commit e727101957
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 270 additions and 60 deletions

View File

@ -22,7 +22,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
private readonly IKestrelTrace _log;
private readonly object _flushLock = new object();
private Task<FlushResult> _lastFlushTask = null;
// This field should only be get or set under the _flushLock. This is a ValueTask that was either:
// 1. The default value where "IsCompleted" is true
// 2. Created by an async method
// 3. Constructed explicitely from a completed result
// This means it should be safe to await a single _lastFlushTask instance multiple times.
private ValueTask<FlushResult> _lastFlushTask;
public TimingPipeFlusher(
PipeWriter writer,
@ -51,35 +57,41 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
public ValueTask<FlushResult> FlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
{
var flushValueTask = _writer.FlushAsync(cancellationToken);
// https://github.com/dotnet/corefxlab/issues/1334
// Pipelines don't support multiple awaiters on flush.
lock (_flushLock)
{
if (_lastFlushTask.IsCompleted)
{
_lastFlushTask = TimeFlushAsync(minRate, count, outputAborter, cancellationToken);
}
else
{
_lastFlushTask = AwaitLastFlushAndTimeFlushAsync(_lastFlushTask, minRate, count, outputAborter, cancellationToken);
}
return _lastFlushTask;
}
}
private ValueTask<FlushResult> TimeFlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
{
var pipeFlushTask = _writer.FlushAsync(cancellationToken);
if (minRate != null)
{
_timeoutControl.BytesWrittenToBuffer(minRate, count);
}
if (flushValueTask.IsCompletedSuccessfully)
if (pipeFlushTask.IsCompletedSuccessfully)
{
return new ValueTask<FlushResult>(flushValueTask.Result);
return new ValueTask<FlushResult>(pipeFlushTask.Result);
}
// https://github.com/dotnet/corefxlab/issues/1334
// Pipelines don't support multiple awaiters on flush.
// While it's acceptable to call PipeWriter.FlushAsync again before the last FlushAsync completes,
// it is not acceptable to attach a new continuation (via await, AsTask(), etc..). In this case,
// we find previous flush Task which still accounts for any newly committed bytes and await that.
lock (_flushLock)
{
if (_lastFlushTask == null || _lastFlushTask.IsCompleted)
{
_lastFlushTask = flushValueTask.AsTask();
}
return TimeFlushAsync(minRate, count, outputAborter, cancellationToken);
}
return TimeFlushAsyncAwaited(pipeFlushTask, minRate, count, outputAborter, cancellationToken);
}
private async ValueTask<FlushResult> TimeFlushAsync(MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
private async ValueTask<FlushResult> TimeFlushAsyncAwaited(ValueTask<FlushResult> pipeFlushTask, MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
{
if (minRate != null)
{
@ -88,7 +100,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
try
{
return await _lastFlushTask;
return await pipeFlushTask;
}
catch (OperationCanceledException ex) when (outputAborter != null)
{
@ -111,5 +123,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
return default;
}
private async ValueTask<FlushResult> AwaitLastFlushAndTimeFlushAsync(ValueTask<FlushResult> lastFlushTask, MinDataRate minRate, long count, IHttpOutputAborter outputAborter, CancellationToken cancellationToken)
{
await lastFlushTask;
return await TimeFlushAsync(minRate, count, outputAborter, cancellationToken);
}
}
}

View File

@ -42,7 +42,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
public Http1ConnectionTests()
{
_pipelineFactory = MemoryPoolFactory.Create();
_pipelineFactory = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(_pipelineFactory, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -21,7 +21,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
[Fact]
public void InitialDictionaryIsEmpty()
{
using (var memoryPool = MemoryPoolFactory.Create())
using (var memoryPool = SlabMemoryPoolFactory.Create())
{
var options = new PipeOptions(memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -21,7 +21,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
public OutputProducerTests()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
}
public void Dispose()

View File

@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
private const int _ulongMaxValueLength = 20;
private readonly Pipe _pipe;
private readonly MemoryPool<byte> _memoryPool = MemoryPoolFactory.Create();
private readonly MemoryPool<byte> _memoryPool = SlabMemoryPoolFactory.Create();
public PipelineExtensionTests()
{

View File

@ -499,7 +499,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
public StartLineTests()
{
MemoryPool = MemoryPoolFactory.Create();
MemoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(MemoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);
Transport = pair.Transport;

View File

@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
public TestInput()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(pool: _memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);
Transport = pair.Transport;

View File

@ -0,0 +1,67 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.IO.Pipelines;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
using Moq;
using Xunit;
namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
{
public class TimingPipeFlusherTests
{
[Fact]
public async Task IfFlushIsCalledAgainBeforeTheLastFlushCompletedItWaitsForTheLastCall()
{
var mockPipeWriter = new Mock<PipeWriter>();
var pipeWriterFlushTcsArray = new[] {
new TaskCompletionSource<FlushResult>(TaskCreationOptions.RunContinuationsAsynchronously),
new TaskCompletionSource<FlushResult>(TaskCreationOptions.RunContinuationsAsynchronously),
new TaskCompletionSource<FlushResult>(TaskCreationOptions.RunContinuationsAsynchronously),
};
var pipeWriterFlushCallCount = 0;
mockPipeWriter.Setup(p => p.FlushAsync(CancellationToken.None)).Returns(() =>
{
return new ValueTask<FlushResult>(pipeWriterFlushTcsArray[pipeWriterFlushCallCount++].Task);
});
var timingPipeFlusher = new TimingPipeFlusher(mockPipeWriter.Object, null, null);
var flushTask0 = timingPipeFlusher.FlushAsync();
var flushTask1 = timingPipeFlusher.FlushAsync();
var flushTask2 = timingPipeFlusher.FlushAsync();
Assert.False(flushTask0.IsCompleted);
Assert.False(flushTask1.IsCompleted);
Assert.False(flushTask2.IsCompleted);
Assert.Equal(1, pipeWriterFlushCallCount);
pipeWriterFlushTcsArray[0].SetResult(default);
await flushTask0.AsTask().DefaultTimeout();
Assert.True(flushTask0.IsCompleted);
Assert.False(flushTask1.IsCompleted);
Assert.False(flushTask2.IsCompleted);
Assert.True(pipeWriterFlushCallCount <= 2);
pipeWriterFlushTcsArray[1].SetResult(default);
await flushTask1.AsTask().DefaultTimeout();
Assert.True(flushTask0.IsCompleted);
Assert.True(flushTask1.IsCompleted);
Assert.False(flushTask2.IsCompleted);
Assert.True(pipeWriterFlushCallCount <= 3);
pipeWriterFlushTcsArray[2].SetResult(default);
await flushTask2.AsTask().DefaultTimeout();
Assert.True(flushTask0.IsCompleted);
Assert.True(flushTask1.IsCompleted);
Assert.True(flushTask2.IsCompleted);
Assert.Equal(3, pipeWriterFlushCallCount);
}
}
}

View File

@ -31,7 +31,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv
public long? MaxWriteBufferSize { get; set; } = 64 * 1024;
internal Func<MemoryPool<byte>> MemoryPoolFactory { get; set; } = System.Buffers.MemoryPoolFactory.Create;
internal Func<MemoryPool<byte>> MemoryPoolFactory { get; set; } = System.Buffers.SlabMemoryPoolFactory.Create;
private static int ProcessorThreadCount
{

View File

@ -41,7 +41,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.Tests
public LibuvOutputConsumerTests()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
_mockLibuv = new MockLibuv();
var context = new TestLibuvTransportContext();
@ -560,15 +560,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.Tests
Assert.False(task1Waits.IsFaulted);
// following tasks should wait.
var task3Canceled = outputProducer.WriteDataAsync(fullBuffer, cancellationToken: abortedSource.Token);
var task2Canceled = outputProducer.WriteDataAsync(fullBuffer, cancellationToken: abortedSource.Token);
// Give time for tasks to percolate
await _mockLibuv.OnPostTask;
// Third task is not completed
Assert.False(task3Canceled.IsCompleted);
Assert.False(task3Canceled.IsCanceled);
Assert.False(task3Canceled.IsFaulted);
// Second task is not completed
Assert.False(task2Canceled.IsCompleted);
Assert.False(task2Canceled.IsCanceled);
Assert.False(task2Canceled.IsFaulted);
abortedSource.Cancel();
@ -578,29 +578,29 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.Tests
await _libuvThread.PostAsync(cb => cb(0), triggerNextCompleted);
}
// First task is completed
// First task is completed
Assert.True(task1Waits.IsCompleted);
Assert.False(task1Waits.IsCanceled);
Assert.False(task1Waits.IsFaulted);
// A final write guarantees that the error is observed by OutputProducer,
// but doesn't return a canceled/faulted task.
var task4Success = outputProducer.WriteDataAsync(fullBuffer);
Assert.True(task4Success.IsCompleted);
Assert.False(task4Success.IsCanceled);
Assert.False(task4Success.IsFaulted);
// Second task is now canceled
await Assert.ThrowsAsync<OperationCanceledException>(() => task2Canceled);
Assert.True(task2Canceled.IsCanceled);
// Third task is now canceled
await Assert.ThrowsAsync<OperationCanceledException>(() => task3Canceled);
Assert.True(task3Canceled.IsCanceled);
// A final write can still succeed.
var task3Success = outputProducer.WriteDataAsync(fullBuffer);
await _mockLibuv.OnPostTask;
// Complete the 4th write
// Complete the 3rd write
while (completeQueue.TryDequeue(out var triggerNextCompleted))
{
await _libuvThread.PostAsync(cb => cb(0), triggerNextCompleted);
}
Assert.True(task3Success.IsCompleted);
Assert.False(task3Success.IsCanceled);
Assert.False(task3Success.IsFaulted);;
}
});
}

View File

@ -28,6 +28,6 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets
public long? MaxWriteBufferSize { get; set; } = 64 * 1024;
internal Func<MemoryPool<byte>> MemoryPoolFactory { get; set; } = System.Buffers.MemoryPoolFactory.Create;
internal Func<MemoryPool<byte>> MemoryPoolFactory { get; set; } = System.Buffers.SlabMemoryPoolFactory.Create;
}
}

View File

@ -20,7 +20,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[GlobalSetup]
public void Setup()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
var pipe = new Pipe(new PipeOptions(_memoryPool));
_reader = pipe.Reader;
_writer = pipe.Writer;

View File

@ -26,7 +26,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[GlobalSetup]
public void Setup()
{
var memoryPool = MemoryPoolFactory.Create();
var memoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[IterationSetup]
public void Setup()
{
var memoryPool = MemoryPoolFactory.Create();
var memoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -0,0 +1,117 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.IO.Pipelines;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
using Microsoft.AspNetCore.Testing;
namespace Microsoft.AspNetCore.Server.Kestrel.Performance
{
public class Http1LargeWritingBenchmark
{
private TestHttp1Connection _http1Connection;
private DuplexPipe.DuplexPipePair _pair;
private MemoryPool<byte> _memoryPool;
private Task _consumeResponseBodyTask;
// Keep this divisable by 10 so it can be evenly segmented.
private readonly byte[] _writeData = new byte[10 * 1024 * 1024];
[GlobalSetup]
public void GlobalSetup()
{
_memoryPool = SlabMemoryPoolFactory.Create();
_http1Connection = MakeHttp1Connection();
_consumeResponseBodyTask = ConsumeResponseBody();
}
[IterationSetup]
public void Setup()
{
_http1Connection.Reset();
_http1Connection.RequestHeaders.ContentLength = _writeData.Length;
_http1Connection.FlushAsync().GetAwaiter().GetResult();
}
[Benchmark]
public Task WriteAsync()
{
return _http1Connection.ResponseBody.WriteAsync(_writeData, 0, _writeData.Length, default);
}
[Benchmark]
public Task WriteSegmentsUnawaitedAsync()
{
// Write a 10th the of the data at a time
var segmentSize = _writeData.Length / 10;
for (int i = 0; i < 9; i++)
{
// Ignore the first nine tasks.
_ = _http1Connection.ResponseBody.WriteAsync(_writeData, i * segmentSize, segmentSize, default);
}
return _http1Connection.ResponseBody.WriteAsync(_writeData, 9 * segmentSize, segmentSize, default);
}
private TestHttp1Connection MakeHttp1Connection()
{
var options = new PipeOptions(_memoryPool, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);
_pair = pair;
var serviceContext = new ServiceContext
{
DateHeaderValueManager = new DateHeaderValueManager(),
ServerOptions = new KestrelServerOptions(),
Log = new MockTrace(),
HttpParser = new HttpParser<Http1ParsingHandler>()
};
var http1Connection = new TestHttp1Connection(new HttpConnectionContext
{
ServiceContext = serviceContext,
ConnectionFeatures = new FeatureCollection(),
MemoryPool = _memoryPool,
TimeoutControl = new TimeoutControl(timeoutHandler: null),
Transport = pair.Transport
});
http1Connection.Reset();
http1Connection.InitializeBodyControl(MessageBody.ZeroContentLengthKeepAlive);
serviceContext.DateHeaderValueManager.OnHeartbeat(DateTimeOffset.UtcNow);
return http1Connection;
}
private async Task ConsumeResponseBody()
{
var reader = _pair.Application.Input;
var readResult = await reader.ReadAsync();
while (!readResult.IsCompleted)
{
reader.AdvanceTo(readResult.Buffer.End);
readResult = await reader.ReadAsync();
}
reader.Complete();
}
[GlobalCleanup]
public void Dispose()
{
_pair.Transport.Output.Complete();
_consumeResponseBodyTask.GetAwaiter().GetResult();
_memoryPool?.Dispose();
}
}
}

View File

@ -34,7 +34,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[GlobalSetup]
public void GlobalSetup()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
_http1Connection = MakeHttp1Connection();
}

View File

@ -28,14 +28,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
private TestHttp1Connection _http1Connection;
private DuplexPipe.DuplexPipePair _pair;
private MemoryPool<byte> _memoryPool;
private Task _consumeResponseBodyTask;
private readonly byte[] _writeData = Encoding.ASCII.GetBytes("Hello, World!");
[GlobalSetup]
public void GlobalSetup()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
_http1Connection = MakeHttp1Connection();
_consumeResponseBodyTask = ConsumeResponseBody();
}
[Params(true, false)]
@ -124,14 +126,18 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
return http1Connection;
}
[IterationCleanup]
public void Cleanup()
private async Task ConsumeResponseBody()
{
var reader = _pair.Application.Input;
if (reader.TryRead(out var readResult))
var readResult = await reader.ReadAsync();
while (!readResult.IsCompleted)
{
reader.AdvanceTo(readResult.Buffer.End);
readResult = await reader.ReadAsync();
}
reader.Complete();
}
public enum Startup
@ -144,6 +150,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[GlobalCleanup]
public void Dispose()
{
_pair.Transport.Output.Complete();
_consumeResponseBodyTask.GetAwaiter().GetResult();
_memoryPool?.Dispose();
}
}

View File

@ -78,7 +78,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
public HttpProtocolFeatureCollection()
{
var memoryPool = MemoryPoolFactory.Create();
var memoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[IterationSetup]
public void Setup()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
_pipe = new Pipe(new PipeOptions(_memoryPool));
}

View File

@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[IterationSetup]
public void Setup()
{
_memoryPool = MemoryPoolFactory.Create();
_memoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(_memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -172,7 +172,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
[IterationSetup]
public void Setup()
{
var memoryPool = MemoryPoolFactory.Create();
var memoryPool = SlabMemoryPoolFactory.Create();
var options = new PipeOptions(memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(options, options);

View File

@ -74,7 +74,7 @@ namespace Microsoft.AspNetCore.Testing
public MockSystemClock MockSystemClock { get; set; }
public Func<MemoryPool<byte>> MemoryPoolFactory { get; set; } = System.Buffers.MemoryPoolFactory.Create;
public Func<MemoryPool<byte>> MemoryPoolFactory { get; set; } = System.Buffers.SlabMemoryPoolFactory.Create;
public int ExpectedConnectionMiddlewareCount { get; set; }

View File

@ -114,7 +114,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
protected static readonly byte[] _noData = new byte[0];
protected static readonly byte[] _maxData = Encoding.ASCII.GetBytes(new string('a', Http2PeerSettings.MinAllowedMaxFrameSize));
private readonly MemoryPool<byte> _memoryPool = MemoryPoolFactory.Create();
private readonly MemoryPool<byte> _memoryPool = SlabMemoryPoolFactory.Create();
internal readonly Http2PeerSettings _clientSettings = new Http2PeerSettings();
internal readonly HPackEncoder _hpackEncoder = new HPackEncoder();

View File

@ -3,7 +3,7 @@
namespace System.Buffers
{
internal static class MemoryPoolFactory
internal static class SlabMemoryPoolFactory
{
public static MemoryPool<byte> Create()
{