Fix flaky heartbeat test (#1794)
This commit is contained in:
parent
749e282102
commit
7122b6c4aa
|
|
@ -12,29 +12,21 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
|
||||||
public static readonly TimeSpan Interval = TimeSpan.FromSeconds(1);
|
public static readonly TimeSpan Interval = TimeSpan.FromSeconds(1);
|
||||||
|
|
||||||
private readonly IHeartbeatHandler[] _callbacks;
|
private readonly IHeartbeatHandler[] _callbacks;
|
||||||
private readonly TimeSpan _interval;
|
|
||||||
private readonly ISystemClock _systemClock;
|
private readonly ISystemClock _systemClock;
|
||||||
private readonly IKestrelTrace _trace;
|
private readonly IKestrelTrace _trace;
|
||||||
private Timer _timer;
|
private Timer _timer;
|
||||||
private int _executingOnHeartbeat;
|
private int _executingOnHeartbeat;
|
||||||
|
|
||||||
public Heartbeat(IHeartbeatHandler[] callbacks, ISystemClock systemClock, IKestrelTrace trace)
|
public Heartbeat(IHeartbeatHandler[] callbacks, ISystemClock systemClock, IKestrelTrace trace)
|
||||||
: this(callbacks, systemClock, trace, Interval)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
// For testing
|
|
||||||
internal Heartbeat(IHeartbeatHandler[] callbacks, ISystemClock systemClock, IKestrelTrace trace, TimeSpan interval)
|
|
||||||
{
|
{
|
||||||
_callbacks = callbacks;
|
_callbacks = callbacks;
|
||||||
_interval = interval;
|
|
||||||
_systemClock = systemClock;
|
_systemClock = systemClock;
|
||||||
_trace = trace;
|
_trace = trace;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Start()
|
public void Start()
|
||||||
{
|
{
|
||||||
_timer = new Timer(OnHeartbeat, state: this, dueTime: _interval, period: _interval);
|
_timer = new Timer(OnHeartbeat, state: this, dueTime: Interval, period: Interval);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void OnHeartbeat(object state)
|
private static void OnHeartbeat(object state)
|
||||||
|
|
@ -67,7 +59,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_trace.TimerSlow(_interval, now);
|
_trace.HeartbeatSlow(Interval, now);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
|
||||||
|
|
||||||
void NotAllConnectionsAborted();
|
void NotAllConnectionsAborted();
|
||||||
|
|
||||||
void TimerSlow(TimeSpan interval, DateTimeOffset now);
|
void HeartbeatSlow(TimeSpan interval, DateTimeOffset now);
|
||||||
|
|
||||||
void ApplicationNeverCompleted(string connectionId);
|
void ApplicationNeverCompleted(string connectionId);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
|
||||||
private static readonly Action<ILogger, Exception> _notAllConnectionsAborted =
|
private static readonly Action<ILogger, Exception> _notAllConnectionsAborted =
|
||||||
LoggerMessage.Define(LogLevel.Debug, 21, "Some connections failed to abort during server shutdown.");
|
LoggerMessage.Define(LogLevel.Debug, 21, "Some connections failed to abort during server shutdown.");
|
||||||
|
|
||||||
private static readonly Action<ILogger, TimeSpan, DateTimeOffset, Exception> _timerSlow =
|
private static readonly Action<ILogger, TimeSpan, DateTimeOffset, Exception> _heartbeatSlow =
|
||||||
LoggerMessage.Define<TimeSpan, DateTimeOffset>(LogLevel.Warning, 22, @"Heartbeat took longer than ""{interval}"" at ""{now}"".");
|
LoggerMessage.Define<TimeSpan, DateTimeOffset>(LogLevel.Warning, 22, @"Heartbeat took longer than ""{interval}"" at ""{now}"".");
|
||||||
|
|
||||||
private static readonly Action<ILogger, string, Exception> _applicationNeverCompleted =
|
private static readonly Action<ILogger, string, Exception> _applicationNeverCompleted =
|
||||||
|
|
@ -121,9 +121,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
|
||||||
_notAllConnectionsAborted(_logger, null);
|
_notAllConnectionsAborted(_logger, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public virtual void TimerSlow(TimeSpan interval, DateTimeOffset now)
|
public virtual void HeartbeatSlow(TimeSpan interval, DateTimeOffset now)
|
||||||
{
|
{
|
||||||
_timerSlow(_logger, interval, now, null);
|
_heartbeatSlow(_logger, interval, now, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public virtual void ApplicationNeverCompleted(string connectionId)
|
public virtual void ApplicationNeverCompleted(string connectionId)
|
||||||
|
|
|
||||||
|
|
@ -43,11 +43,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
|
||||||
UtcNow = now
|
UtcNow = now
|
||||||
};
|
};
|
||||||
|
|
||||||
var timerInterval = TimeSpan.FromSeconds(10);
|
|
||||||
var dateHeaderValueManager = new DateHeaderValueManager(systemClock);
|
var dateHeaderValueManager = new DateHeaderValueManager(systemClock);
|
||||||
var testKestrelTrace = new TestKestrelTrace();
|
var testKestrelTrace = new TestKestrelTrace();
|
||||||
|
|
||||||
using (var heartbeat = new Heartbeat(new IHeartbeatHandler[] { dateHeaderValueManager }, systemClock, testKestrelTrace, timerInterval))
|
using (var heartbeat = new Heartbeat(new IHeartbeatHandler[] { dateHeaderValueManager }, systemClock, testKestrelTrace))
|
||||||
{
|
{
|
||||||
Assert.Equal(now.ToString(Rfc1123DateFormat), dateHeaderValueManager.GetDateHeaderValues().String);
|
Assert.Equal(now.ToString(Rfc1123DateFormat), dateHeaderValueManager.GetDateHeaderValues().String);
|
||||||
systemClock.UtcNow = future;
|
systemClock.UtcNow = future;
|
||||||
|
|
@ -67,13 +66,12 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
|
||||||
UtcNow = now
|
UtcNow = now
|
||||||
};
|
};
|
||||||
|
|
||||||
var timerInterval = TimeSpan.FromMilliseconds(100);
|
|
||||||
var dateHeaderValueManager = new DateHeaderValueManager(systemClock);
|
var dateHeaderValueManager = new DateHeaderValueManager(systemClock);
|
||||||
var testKestrelTrace = new TestKestrelTrace();
|
var testKestrelTrace = new TestKestrelTrace();
|
||||||
|
|
||||||
var mockHeartbeatHandler = new Mock<IHeartbeatHandler>();
|
var mockHeartbeatHandler = new Mock<IHeartbeatHandler>();
|
||||||
|
|
||||||
using (var heartbeat = new Heartbeat(new[] { dateHeaderValueManager, mockHeartbeatHandler.Object }, systemClock, testKestrelTrace, timerInterval))
|
using (var heartbeat = new Heartbeat(new[] { dateHeaderValueManager, mockHeartbeatHandler.Object }, systemClock, testKestrelTrace))
|
||||||
{
|
{
|
||||||
heartbeat.OnHeartbeat();
|
heartbeat.OnHeartbeat();
|
||||||
|
|
||||||
|
|
@ -99,11 +97,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
|
||||||
UtcNow = now
|
UtcNow = now
|
||||||
};
|
};
|
||||||
|
|
||||||
var timerInterval = TimeSpan.FromSeconds(10);
|
|
||||||
var dateHeaderValueManager = new DateHeaderValueManager(systemClock);
|
var dateHeaderValueManager = new DateHeaderValueManager(systemClock);
|
||||||
var testKestrelTrace = new TestKestrelTrace();
|
var testKestrelTrace = new TestKestrelTrace();
|
||||||
|
|
||||||
using (var heatbeat = new Heartbeat(new IHeartbeatHandler[] { dateHeaderValueManager }, systemClock, testKestrelTrace, timerInterval))
|
using (var heatbeat = new Heartbeat(new IHeartbeatHandler[] { dateHeaderValueManager }, systemClock, testKestrelTrace))
|
||||||
{
|
{
|
||||||
heatbeat.OnHeartbeat();
|
heatbeat.OnHeartbeat();
|
||||||
Assert.Equal(now.ToString(Rfc1123DateFormat), dateHeaderValueManager.GetDateHeaderValues().String);
|
Assert.Equal(now.ToString(Rfc1123DateFormat), dateHeaderValueManager.GetDateHeaderValues().String);
|
||||||
|
|
|
||||||
|
|
@ -19,39 +19,38 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
|
||||||
public void BlockedHeartbeatDoesntCauseOverlapsAndIsLoggedAsError()
|
public void BlockedHeartbeatDoesntCauseOverlapsAndIsLoggedAsError()
|
||||||
{
|
{
|
||||||
var systemClock = new MockSystemClock();
|
var systemClock = new MockSystemClock();
|
||||||
var heartbeatInterval = TimeSpan.FromMilliseconds(10);
|
|
||||||
var heartbeatHandler = new Mock<IHeartbeatHandler>();
|
var heartbeatHandler = new Mock<IHeartbeatHandler>();
|
||||||
var kestrelTrace = new Mock<IKestrelTrace>();
|
var kestrelTrace = new Mock<IKestrelTrace>();
|
||||||
var handlerMre = new ManualResetEventSlim();
|
var handlerMre = new ManualResetEventSlim();
|
||||||
var traceMre = new ManualResetEventSlim();
|
var traceMre = new ManualResetEventSlim();
|
||||||
|
|
||||||
heartbeatHandler.Setup(h => h.OnHeartbeat(systemClock.UtcNow)).Callback(() => handlerMre.Wait());
|
heartbeatHandler.Setup(h => h.OnHeartbeat(systemClock.UtcNow)).Callback(() => handlerMre.Wait());
|
||||||
kestrelTrace.Setup(t => t.TimerSlow(heartbeatInterval, systemClock.UtcNow)).Callback(() => traceMre.Set());
|
kestrelTrace.Setup(t => t.HeartbeatSlow(Heartbeat.Interval, systemClock.UtcNow)).Callback(() => traceMre.Set());
|
||||||
|
|
||||||
using (var heartbeat = new Heartbeat(new[] { heartbeatHandler.Object }, systemClock, kestrelTrace.Object, heartbeatInterval))
|
using (var heartbeat = new Heartbeat(new[] { heartbeatHandler.Object }, systemClock, kestrelTrace.Object))
|
||||||
{
|
{
|
||||||
heartbeat.Start();
|
Task.Run(() => heartbeat.OnHeartbeat());
|
||||||
|
Task.Run(() => heartbeat.OnHeartbeat());
|
||||||
Assert.True(traceMre.Wait(TimeSpan.FromSeconds(10)));
|
Assert.True(traceMre.Wait(TimeSpan.FromSeconds(10)));
|
||||||
}
|
}
|
||||||
|
|
||||||
handlerMre.Set();
|
handlerMre.Set();
|
||||||
|
|
||||||
heartbeatHandler.Verify(h => h.OnHeartbeat(systemClock.UtcNow), Times.Once());
|
heartbeatHandler.Verify(h => h.OnHeartbeat(systemClock.UtcNow), Times.Once());
|
||||||
kestrelTrace.Verify(t => t.TimerSlow(heartbeatInterval, systemClock.UtcNow), Times.AtLeastOnce());
|
kestrelTrace.Verify(t => t.HeartbeatSlow(Heartbeat.Interval, systemClock.UtcNow), Times.Once());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ExceptionFromHeartbeatHandlerIsLoggedAsError()
|
public void ExceptionFromHeartbeatHandlerIsLoggedAsError()
|
||||||
{
|
{
|
||||||
var systemClock = new MockSystemClock();
|
var systemClock = new MockSystemClock();
|
||||||
var heartbeatInterval = TimeSpan.FromMilliseconds(10);
|
|
||||||
var heartbeatHandler = new Mock<IHeartbeatHandler>();
|
var heartbeatHandler = new Mock<IHeartbeatHandler>();
|
||||||
var kestrelTrace = new TestKestrelTrace();
|
var kestrelTrace = new TestKestrelTrace();
|
||||||
var ex = new Exception();
|
var ex = new Exception();
|
||||||
|
|
||||||
heartbeatHandler.Setup(h => h.OnHeartbeat(systemClock.UtcNow)).Throws(ex);
|
heartbeatHandler.Setup(h => h.OnHeartbeat(systemClock.UtcNow)).Throws(ex);
|
||||||
|
|
||||||
using (var heartbeat = new Heartbeat(new[] { heartbeatHandler.Object }, systemClock, kestrelTrace, heartbeatInterval))
|
using (var heartbeat = new Heartbeat(new[] { heartbeatHandler.Object }, systemClock, kestrelTrace))
|
||||||
{
|
{
|
||||||
heartbeat.OnHeartbeat();
|
heartbeat.OnHeartbeat();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -33,7 +33,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
|
||||||
public void NotAllConnectionsAborted() { }
|
public void NotAllConnectionsAborted() { }
|
||||||
public void NotAllConnectionsClosedGracefully() { }
|
public void NotAllConnectionsClosedGracefully() { }
|
||||||
public void RequestProcessingError(string connectionId, Exception ex) { }
|
public void RequestProcessingError(string connectionId, Exception ex) { }
|
||||||
public void TimerSlow(TimeSpan interval, DateTimeOffset now) { }
|
public void HeartbeatSlow(TimeSpan interval, DateTimeOffset now) { }
|
||||||
public void ApplicationNeverCompleted(string connectionId) { }
|
public void ApplicationNeverCompleted(string connectionId) { }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue