Mark tests flaky instead of skipped (#8727)

Mark tests flaky instead of skipped
This commit is contained in:
Ryan Brandenburg 2019-03-26 09:51:37 -07:00 committed by GitHub
parent 341cd25e05
commit 70cd5f5f9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 73 additions and 58 deletions

View File

@ -135,7 +135,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
}
[Fact]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1987", FlakyOn.All)]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1987", FlakyOn.AzP.Windows)]
public void InputEvent_RespondsOnKeystrokes()
{
MountTestComponent<InputEventComponent>();

View File

@ -33,7 +33,8 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests
await ExecuteShutdownTest(nameof(ShutdownTestRun), "Run");
}
[ConditionalFact(Skip = "https://github.com/aspnet/Hosting/issues/1214")]
[ConditionalFact]
[Flaky("https://github.com/aspnet/Hosting/issues/1214", FlakyOn.All)]
[OSSkipCondition(OperatingSystems.Windows)]
[OSSkipCondition(OperatingSystems.MacOSX)]
public async Task ShutdownTestWaitForShutdown()

View File

@ -292,4 +292,4 @@ namespace Microsoft.AspNetCore.Identity.EntityFrameworkCore.Test
=> Task.CompletedTask;
}
}
}

View File

@ -8,6 +8,7 @@ using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Testing;
using Microsoft.AspNetCore.Testing.xunit;
using Microsoft.AspNetCore.WebSockets.Internal;
using Microsoft.Extensions.Logging.Testing;
@ -549,7 +550,8 @@ namespace Microsoft.AspNetCore.WebSockets.Test
}
}
[Fact(Skip = "https://github.com/aspnet/AspNetCore/issues/8187")]
[Fact]
[Flaky("https://github.com/aspnet/AspNetCore/issues/8187", FlakyOn.Helix.All)]
public async Task OriginIsNotValidatedForNonWebSocketRequests()
{
using (var server = KestrelWebSocketHelpers.CreateServer(LoggerFactory, out var port, context =>

View File

@ -1004,7 +1004,7 @@ namespace Microsoft.AspNetCore.Mvc.ApplicationModels
Assert.Empty(action.Attributes);
}
[Theory(Skip = "See aspnet/AspNetCore#4417")]
[Theory(Skip = "https://github.com/aspnet/AspNetCore/issues/4417")]
[InlineData(typeof(SingleRouteAttributeController))]
[InlineData(typeof(MultipleRouteAttributeController))]
public void CreateActionModel_RouteOnController_CreatesOneActionInfoPerRouteTemplateOnAction(Type controller)

View File

@ -37,7 +37,7 @@ namespace Microsoft.AspNetCore.Mvc.Formatters
{
return base.ReadAsync_UsesTryAddModelValidationErrorsToModelState();
}
[Fact(Skip = "https://github.com/dotnet/corefx/issues/36026")]
public override Task ReadAsync_ReadsValidArray_AsCollectionOfT()
{

View File

@ -22,4 +22,4 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
return base.JsonInputFormatter_ReturnsDefaultValue_ForValueTypes(input);
}
}
}
}

View File

@ -8,6 +8,8 @@ using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Testing;
using Microsoft.AspNetCore.Testing.xunit;
using Microsoft.Net.Http.Headers;
using Xunit;
@ -28,7 +30,8 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
private static void ConfigureWebHostBuilder(IWebHostBuilder builder) =>
builder.UseStartup<BasicWebSite.StartupWithCookieTempDataProviderAndCookieConsent>();
[Fact(Skip = "https://github.com/aspnet/AspNetCore-Internal/issues/1803")]
[Fact]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1803", FlakyOn.AzP.Windows)]
public async Task CookieTempDataProviderCookie_SetInResponse_OnGrantingConsent()
{
// Arrange
@ -61,7 +64,8 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
}
[Fact(Skip = "https://github.com/aspnet/AspNetCore-Internal/issues/1803")]
[Fact]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1803", FlakyOn.AzP.Windows)]
public async Task CookieTempDataProviderCookie_NotSetInResponse_OnNoConsent()
{
// Arrange

View File

@ -464,19 +464,6 @@ Partial";
Assert.Equal(expected, responseContent, ignoreLineEndingDifferences: true);
}
[Fact(Skip = "https://github.com/aspnet/Mvc/issues/8754")]
public async Task ViewEngine_ResolvesPathsWithSlashesThatDoNotHaveExtensions()
{
// Arrange
var expected = @"<embdedded-layout>Hello from EmbeddedHome\EmbeddedPartial</embdedded-layout>";
// Act
var responseContent = await Client.GetStringAsync("/EmbeddedViews/RelativeNonPath");
// Assert
Assert.Equal(expected, responseContent.Trim());
}
[Fact]
public async Task ViewEngine_DiscoversViewsFromPagesSharedDirectory()
{

View File

@ -221,7 +221,8 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
{
var deploymentResult = await AssertStarts(hostingModel);
var load = Helpers.StressLoad(deploymentResult.HttpClient, "/HelloWorld", response => {
var load = Helpers.StressLoad(deploymentResult.HttpClient, "/HelloWorld", response =>
{
var statusCode = (int)response.StatusCode;
// Test failure involves the stress load receiving a 400 Bad Request.
// We think it is due to IIS returning the 400 itself, but need to confirm the hypothesis.

View File

@ -34,7 +34,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
await deploymentResult.AssertStarts();
// Just "touching" web.config should be enough
deploymentResult.ModifyWebConfig(element => {});
deploymentResult.ModifyWebConfig(element => { });
await deploymentResult.AssertRecycledAsync();
}
@ -49,7 +49,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
var processBefore = await deploymentResult.HttpClient.GetStringAsync("/ProcessId");
// Just "touching" web.config should be enough
deploymentResult.ModifyWebConfig(element => {});
deploymentResult.ModifyWebConfig(element => { });
// Have to retry here to allow ANCM to receive notification and react to it
// Verify that worker process gets restarted with new process id
@ -88,7 +88,8 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
var deploymentResult = await DeployAsync(Fixture.GetBaseDeploymentParameters(hostingModel));
await deploymentResult.AssertStarts();
var load = Helpers.StressLoad(deploymentResult.HttpClient, "/HelloWorld", response => {
var load = Helpers.StressLoad(deploymentResult.HttpClient, "/HelloWorld", response =>
{
var statusCode = (int)response.StatusCode;
Assert.True(statusCode == 200 || statusCode == 503, "Status code was " + statusCode);
});
@ -97,7 +98,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
{
// ModifyWebConfig might fail if web.config is being read by IIS
RetryHelper.RetryOperation(
() => deploymentResult.ModifyWebConfig(element => {}),
() => deploymentResult.ModifyWebConfig(element => { }),
e => Logger.LogError($"Failed to touch web.config : {e.Message}"),
retryCount: 3,
retryDelayMilliseconds: RetryDelay.Milliseconds);

View File

@ -430,42 +430,48 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
var pathWithSpace = "\u03c0 \u2260 3\u00b714";
dictionary.Add("App in bin subdirectory full path to dll using exec and quotes",
parameters => {
parameters =>
{
MoveApplication(parameters, "bin");
parameters.TransformArguments((arguments, root) => "exec " + Path.Combine(root, "bin", arguments));
return "";
});
dictionary.Add("App in subdirectory with space",
parameters => {
parameters =>
{
MoveApplication(parameters, pathWithSpace);
parameters.TransformArguments((arguments, root) => Path.Combine(pathWithSpace, arguments));
return "";
});
dictionary.Add("App in subdirectory with space and full path to dll",
parameters => {
parameters =>
{
MoveApplication(parameters, pathWithSpace);
parameters.TransformArguments((arguments, root) => Path.Combine(root, pathWithSpace, arguments));
return "";
});
dictionary.Add("App in bin subdirectory with space full path to dll using exec and quotes",
parameters => {
parameters =>
{
MoveApplication(parameters, pathWithSpace);
parameters.TransformArguments((arguments, root) => "exec \"" + Path.Combine(root, pathWithSpace, arguments) + "\" extra arguments");
return "extra|arguments";
});
dictionary.Add("App in bin subdirectory and quoted argument",
parameters => {
parameters =>
{
MoveApplication(parameters, "bin");
parameters.TransformArguments((arguments, root) => Path.Combine("bin", arguments) + " \"extra argument\"");
return "extra argument";
});
dictionary.Add("App in bin subdirectory full path to dll",
parameters => {
parameters =>
{
MoveApplication(parameters, "bin");
parameters.TransformArguments((arguments, root) => Path.Combine(root, "bin", arguments) + " extra arguments");
return "extra|arguments";
@ -495,7 +501,8 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
var pathWithSpace = "\u03c0 \u2260 3\u00b714";
dictionary.Add("App in subdirectory",
parameters => {
parameters =>
{
MoveApplication(parameters, pathWithSpace);
parameters.TransformPath((path, root) => Path.Combine(pathWithSpace, path));
parameters.TransformArguments((arguments, root) => "\"additional argument\"");
@ -503,7 +510,8 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
});
dictionary.Add("App in bin subdirectory full path",
parameters => {
parameters =>
{
MoveApplication(parameters, pathWithSpace);
parameters.TransformPath((path, root) => Path.Combine(root, pathWithSpace, path));
parameters.TransformArguments((arguments, root) => "additional arguments");

View File

@ -32,7 +32,8 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
{
var deploymentParameters = Fixture.GetBaseDeploymentParameters(variant);
deploymentParameters.ServerConfigActionList.Add(
(element, _) => {
(element, _) =>
{
element
.RequiredElement("system.webServer")
.RequiredElement("security")
@ -68,7 +69,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
Assert.Equal(deploymentResult.ContentRoot, await deploymentResult.HttpClient.GetStringAsync("/ContentRootPath"));
Assert.Equal(deploymentResult.ContentRoot + "\\wwwroot", await deploymentResult.HttpClient.GetStringAsync("/WebRootPath"));
var expectedDll = "aspnetcorev2.dll";
Assert.Contains(deploymentResult.HostProcess.Modules.OfType<ProcessModule>(), m=> m.FileName.Contains(expectedDll));
Assert.Contains(deploymentResult.HostProcess.Modules.OfType<ProcessModule>(), m => m.FileName.Contains(expectedDll));
if (DeployerSelector.HasNewHandler && variant.HostingModel == HostingModel.InProcess)
{

View File

@ -221,7 +221,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
}
[ConditionalFact]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1817", FlakyOn.All)]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1817", FlakyOn.AzP.Windows)]
public async Task ReaderThrowsResetExceptionOnInvalidBody()
{
var requestStartedCompletionSource = CreateTaskCompletionSource();
@ -319,7 +319,8 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
var requestAborted = CreateTaskCompletionSource();
using (var testServer = await TestServer.Create(
async ctx => {
async ctx =>
{
ctx.RequestAborted.Register(() => requestAborted.SetResult(true));
requestStarted.SetResult(true);
await requestAborted.Task;

View File

@ -112,7 +112,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
private bool LargeUploadRetryPredicate(Exception e)
=> e is IOException && e.Message.Contains("Unable to read data from the transport connection: The I/O operation has been aborted because of either a thread exit or an application request");
[Theory(Skip = "https://github.com/aspnet/AspNetCore/issues/8054")]
[Theory]
[Flaky("https://github.com/aspnet/AspNetCore/issues/8054", FlakyOn.AzP.Windows)]
[RetryTest(nameof(LargeUploadRetryPredicate),
"Active investigation into potential corefx sockets bug: https://github.com/dotnet/corefx/issues/30691",
OperatingSystems.Windows,

View File

@ -341,7 +341,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
Assert.True(requestAborted.Task.IsCompleted);
}
[Theory(Skip = "https://github.com/aspnet/AspNetCore/issues/7342")]
[Theory]
[Flaky("https://github.com/aspnet/AspNetCore/issues/7342", FlakyOn.All)]
[MemberData(nameof(ConnectionAdapterData))]
public async Task AppCanHandleClientAbortingConnectionMidResponse(ListenOptions listenOptions)
{

View File

@ -128,7 +128,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
public async Task HEADERS_ReceivedWithoutAllCONTINUATIONs_WithinRequestHeadersTimeout_AbortsConnection()
{
var mockSystemClock = _serviceContext.MockSystemClock;
var limits = _serviceContext.ServerOptions.Limits;;
var limits = _serviceContext.ServerOptions.Limits; ;
_timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks);
@ -154,8 +154,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
Http2ErrorCode.INTERNAL_ERROR,
CoreStrings.BadRequest_RequestHeadersTimeout);
_mockConnectionContext.Verify(c =>c.Abort(It.Is<ConnectionAbortedException>(e =>
e.Message == CoreStrings.BadRequest_RequestHeadersTimeout)), Times.Once);
_mockConnectionContext.Verify(c => c.Abort(It.Is<ConnectionAbortedException>(e =>
e.Message == CoreStrings.BadRequest_RequestHeadersTimeout)), Times.Once);
_mockTimeoutHandler.VerifyNoOtherCalls();
_mockConnectionContext.VerifyNoOtherCalls();
@ -175,7 +175,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
await WaitForConnectionStopAsync(expectedLastStreamId: 0, ignoreNonGoAwayFrames: false);
AdvanceClock(TimeSpan.FromSeconds(_bytesReceived / limits.MinResponseDataRate.BytesPerSecond) +
AdvanceClock(TimeSpan.FromSeconds(_bytesReceived / limits.MinResponseDataRate.BytesPerSecond) +
limits.MinResponseDataRate.GracePeriod + Heartbeat.Interval - TimeSpan.FromSeconds(.5));
_mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny<TimeoutReason>()), Times.Never);
@ -185,14 +185,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
_mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.WriteDataRate), Times.Once);
_mockConnectionContext.Verify(c =>c.Abort(It.Is<ConnectionAbortedException>(e =>
e.Message == CoreStrings.ConnectionTimedBecauseResponseMininumDataRateNotSatisfied)), Times.Once);
_mockConnectionContext.Verify(c => c.Abort(It.Is<ConnectionAbortedException>(e =>
e.Message == CoreStrings.ConnectionTimedBecauseResponseMininumDataRateNotSatisfied)), Times.Once);
_mockTimeoutHandler.VerifyNoOtherCalls();
_mockConnectionContext.VerifyNoOtherCalls();
}
[Theory(Skip = "https://github.com/aspnet/AspNetCore-Internal/issues/1879")]
[Theory]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1879", FlakyOn.AzP.Linux)]
[InlineData((int)Http2FrameType.DATA)]
[InlineData((int)Http2FrameType.CONTINUATION)]
public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterCooldownExpires(int intFinalFrameType)
@ -267,7 +268,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7000
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7000
public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnSmallWrite_AbortsConnectionAfterGracePeriod()
{
var mockSystemClock = _serviceContext.MockSystemClock;
@ -282,7 +283,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
_timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks);
await InitializeConnectionAsync(_echoApplication);
await StartStreamAsync(1, _browserRequestHeaders, endStream: false);
await SendDataAsync(1, _helloWorldBytes, endStream: true);
@ -316,8 +317,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
Http2ErrorCode.INTERNAL_ERROR,
null);
_mockConnectionContext.Verify(c =>c.Abort(It.Is<ConnectionAbortedException>(e =>
e.Message == CoreStrings.ConnectionTimedBecauseResponseMininumDataRateNotSatisfied)), Times.Once);
_mockConnectionContext.Verify(c => c.Abort(It.Is<ConnectionAbortedException>(e =>
e.Message == CoreStrings.ConnectionTimedBecauseResponseMininumDataRateNotSatisfied)), Times.Once);
_mockTimeoutHandler.VerifyNoOtherCalls();
_mockConnectionContext.VerifyNoOtherCalls();

View File

@ -459,7 +459,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
}
}
[Theory(Skip = "https://github.com/aspnet/AspNetCore/issues/7265")]
[Theory]
[Flaky("https://github.com/aspnet/AspNetCore/issues/7265", FlakyOn.All)]
[InlineData(ClientCertificateMode.AllowCertificate)]
[InlineData(ClientCertificateMode.RequireCertificate)]
public async Task ClientCertificateValidationGetsCalledWithNotNullParameters(ClientCertificateMode mode)

View File

@ -23,6 +23,7 @@ using Microsoft.AspNetCore.Http.Endpoints;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Http.Internal;
using Microsoft.AspNetCore.SignalR.Tests;
using Microsoft.AspNetCore.Testing;
using Microsoft.AspNetCore.Testing.xunit;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@ -2081,7 +2082,8 @@ namespace Microsoft.AspNetCore.Http.Connections.Tests
}
}
[Fact(Skip = "https://github.com/aspnet/AspNetCore/issues/8138")]
[Fact]
[Flaky("https://github.com/aspnet/AspNetCore/issues/8138", FlakyOn.All)]
public async Task ErrorDuringPollWillCloseConnection()
{
bool ExpectedErrors(WriteContext writeContext)

View File

@ -14,6 +14,7 @@ using Microsoft.AspNetCore.Http.Connections;
using Microsoft.AspNetCore.Http.Connections.Client;
using Microsoft.AspNetCore.Http.Connections.Client.Internal;
using Microsoft.AspNetCore.SignalR.Client;
using Microsoft.AspNetCore.Testing;
using Microsoft.AspNetCore.Testing.xunit;
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Extensions.Logging;
@ -307,7 +308,8 @@ namespace Microsoft.AspNetCore.SignalR.Tests
}
}
[ConditionalTheory(Skip = "https://github.com/aspnet/AspNetCore-Internal/issues/1383")]
[ConditionalTheory]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1383", FlakyOn.All)]
[WebSocketsSupportedCondition]
[InlineData(5 * 4096)]
[InlineData(1000 * 4096 + 32)]
@ -530,7 +532,7 @@ namespace Microsoft.AspNetCore.SignalR.Tests
}
}
// Serves a fake transport that lets us verify fallback behavior
// Serves a fake transport that lets us verify fallback behavior
private class TestTransportFactory : ITransportFactory
{
private ITransport _transport;

View File

@ -2,13 +2,13 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Globalization;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Testing.xunit;
using Xunit;
using Xunit.Abstractions;
using Microsoft.AspNetCore.Testing;
namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
{
@ -36,7 +36,8 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
Assert.Equal("1", envValue);
}
[Fact(Skip = "https://github.com/aspnet/AspNetCore-Internal/issues/1826")]
[Fact]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/1826", FlakyOn.All)]
public async Task RunsWithIterationEnvVariable()
{
await _app.StartWatcherAsync();