diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 82cb6cb5bc..eb38671015 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -499,15 +499,15 @@ stages: - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" displayName: Setup IISExpress test certificates and schema afterBuild: - - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true" - displayName: Run Flaky Tests + - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunQuarantinedTests=true" + displayName: Run Quarantined Tests continueOnError: true - task: PublishTestResults@2 - displayName: Publish Flaky Test Results + displayName: Publish Quarantined Test Results inputs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined' artifacts: - name: Windows_Test_Dumps path: artifacts/dumps/ @@ -574,15 +574,15 @@ stages: displayName: Pack Packages (for Template tests) - bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog" displayName: Pack Templates (for Template tests) - - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true - displayName: Run Flaky Tests + - bash: ./build.sh --no-build --ci --test -p:RunQuarantinedTests=true + displayName: Run Quarantined Tests continueOnError: true - task: PublishTestResults@2 - displayName: Publish Flaky Test Results + displayName: Publish Quarantined Test Results inputs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined' artifacts: - name: MacOS_Test_Logs path: artifacts/log/ @@ -611,15 +611,15 @@ stages: displayName: Pack Packages (for Template tests) - bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog" displayName: Pack Templates (for Template tests) - - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true - displayName: Run Flaky Tests + - bash: ./build.sh --no-build --ci --test -p:RunQuarantinedTests=true + displayName: Run Quarantined Tests continueOnError: true - task: PublishTestResults@2 - displayName: Publish Flaky Test Results + displayName: Publish Quarantined Test Results inputs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined' artifacts: - name: Linux_Test_Logs path: artifacts/log/ diff --git a/Directory.Build.props b/Directory.Build.props index 3ef4008890..a2765ff560 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -35,9 +35,9 @@ true - + - + Microsoft ASP.NET Core diff --git a/eng/FlakyTests.AfterArcade.props b/eng/FlakyTests.AfterArcade.props deleted file mode 100644 index 12f631127e..0000000000 --- a/eng/FlakyTests.AfterArcade.props +++ /dev/null @@ -1,7 +0,0 @@ - - - - $(ArtifactsDir)log\$(Configuration)\Flaky\ - $(ArtifactsDir)TestResults\$(Configuration)\Flaky\ - - diff --git a/eng/FlakyTests.BeforeArcade.props b/eng/FlakyTests.BeforeArcade.props deleted file mode 100644 index 02f9fb6c59..0000000000 --- a/eng/FlakyTests.BeforeArcade.props +++ /dev/null @@ -1,18 +0,0 @@ - - - - <_FlakyRunAdditionalArgs>-trait "Flaky:All=true" - <_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true" - - - - - <_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true" - <_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true" - - - - $(_NonFlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments) - $(_FlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments) - - diff --git a/eng/QuarantinedTests.AfterArcade.props b/eng/QuarantinedTests.AfterArcade.props new file mode 100644 index 0000000000..4e0b5cde30 --- /dev/null +++ b/eng/QuarantinedTests.AfterArcade.props @@ -0,0 +1,7 @@ + + + + $(ArtifactsDir)log\$(Configuration)\Quarantined\ + $(ArtifactsDir)TestResults\$(Configuration)\Quarantined\ + + diff --git a/eng/QuarantinedTests.BeforeArcade.props b/eng/QuarantinedTests.BeforeArcade.props new file mode 100644 index 0000000000..5fa5fdcaf6 --- /dev/null +++ b/eng/QuarantinedTests.BeforeArcade.props @@ -0,0 +1,11 @@ + + + <_QuarantinedTestRunAdditionalArgs>-trait "Quarantined=true" + <_NonQuarantinedTestRunAdditionalArgs>-notrait "Quarantined=true" + + + + $(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) + $(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) + + diff --git a/eng/helix/content/runtests.cmd b/eng/helix/content/runtests.cmd index 73cf658c18..5fe1af7eb6 100644 --- a/eng/helix/content/runtests.cmd +++ b/eng/helix/content/runtests.cmd @@ -53,23 +53,21 @@ if %$quarantined%==True ( set %$quarantined=true ) -set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true" -set QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true" +set NONQUARANTINE_FILTER="Quarantined!=true" +set QUARANTINE_FILTER="Quarantined=true" if %$quarantined%==true ( echo Running quarantined tests. %DOTNET_ROOT%\dotnet vstest %$target% --logger:xunit --TestCaseFilter:%QUARANTINE_FILTER% if errorlevel 1 ( - echo Failure in flaky test 1>&2 + echo Failure in quarantined test 1>&2 REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1 ) ) else ( - REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute - REM only puts the explicit filter traits the user provided in REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md echo Running non-quarantined tests. %DOTNET_ROOT%\dotnet vstest %$target% --logger:xunit --TestCaseFilter:%NONQUARANTINE_FILTER% if errorlevel 1 ( - echo Failure in non-flaky test 1>&2 + echo Failure in non-quarantined test 1>&2 set exit_code=1 REM DO NOT EXIT ) diff --git a/eng/helix/content/runtests.sh b/eng/helix/content/runtests.sh index c31297d587..7d319e0a38 100644 --- a/eng/helix/content/runtests.sh +++ b/eng/helix/content/runtests.sh @@ -115,11 +115,9 @@ fi exit_code=0 -# We need to specify all possible quarantined filters that apply to this environment, because the quarantine attribute -# only puts the explicit filter traits the user provided in the flaky attribute # Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md -NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true" -QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true" +NONQUARANTINE_FILTER="Quarantined!=true" +QUARANTINE_FILTER="Quarantined=true" if [ "$quarantined" == true ]; then echo "Running all tests including quarantined." $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$QUARANTINE_FILTER" diff --git a/src/Components/test/E2ETest/Tests/EventTest.cs b/src/Components/test/E2ETest/Tests/EventTest.cs index 06dbbb969f..d2fd55b195 100644 --- a/src/Components/test/E2ETest/Tests/EventTest.cs +++ b/src/Components/test/E2ETest/Tests/EventTest.cs @@ -167,7 +167,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests } [Fact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1987", FlakyOn.AzP.Windows)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1987")] public void InputEvent_RespondsOnKeystrokes() { Browser.MountTestComponent(); diff --git a/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs b/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs index 96157ed4ea..005895f9c0 100644 --- a/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs +++ b/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs @@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Identity.Test [Theory] [MemberData(nameof(ScriptWithFallbackSrcData))] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2267", FlakyOn.AzP.macOS)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2267")] public async Task IdentityUI_ScriptTags_FallbackSourceContent_Matches_CDNContent(ScriptTag scriptTag) { var wwwrootDir = Path.Combine(GetProjectBasePath(), "wwwroot", scriptTag.Version); diff --git a/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs b/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs index 14a4681180..546f862e21 100644 --- a/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs +++ b/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs @@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests builder.UseStartup(); [Fact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1803", FlakyOn.AzP.Windows)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1803")] public async Task CookieTempDataProviderCookie_SetInResponse_OnGrantingConsent() { // Arrange @@ -64,7 +64,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests } [Fact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1803", FlakyOn.AzP.Windows)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1803")] public async Task CookieTempDataProviderCookie_NotSetInResponse_OnNoConsent() { // Arrange diff --git a/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs b/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs index 2f0308af9b..8e2860cf69 100644 --- a/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs +++ b/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs @@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.Listener public class RequestBodyTests { [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1826", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1826")] public async Task RequestBody_SyncReadDisabledByDefault_WorksWhenEnabled() { string address; @@ -142,7 +142,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.Listener } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2206", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2206")] public async Task RequestBody_ReadAsyncPartialBodyAndExpiredTimeout_Canceled() { StaggardContent content = new StaggardContent(); diff --git a/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs b/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs index 2c45b352e3..1112420ed6 100644 --- a/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs +++ b/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs @@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2135", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2135")] public async Task Caching_JustPublic_NotCached() { var requestCount = 1; @@ -86,7 +86,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2207", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2207")] public async Task Caching_WithoutContentType_Cached_OnWin7AndWin2008R2() { if (Utilities.IsWin8orLater) @@ -237,7 +237,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests [ConditionalTheory] [InlineData("0")] [InlineData("-1")] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2208", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2208")] public async Task Caching_InvalidExpires_NotCached(string expiresValue) { var requestCount = 1; @@ -378,7 +378,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2209", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2209")] public async Task Caching_VariousStatusCodes_Cached() { var requestCount = 1; diff --git a/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs b/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs index bf266025b4..fbea889de4 100644 --- a/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs +++ b/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs @@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2267", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2267")] public async Task Server_ShutdownDuringRequest_Success() { Task responseTask; diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs index 13409c545d..89c44e719f 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs @@ -211,7 +211,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests [ConditionalTheory] [InlineData(HostingModel.InProcess)] [InlineData(HostingModel.OutOfProcess)] - [Flaky("https://github.com/dotnet/aspnetcore/issues/7075", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/7075")] public async Task AppOfflineAddedAndRemovedStress(HostingModel hostingModel) { var deploymentResult = await AssertStarts(hostingModel); diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs index 92f581b156..66ffde7f20 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs @@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests [ConditionalTheory] [InlineData(HostingModel.InProcess)] [InlineData(HostingModel.OutOfProcess)] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1794", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1794")] public async Task ConfigurationTouchedStress(HostingModel hostingModel) { var deploymentResult = await DeployAsync(Fixture.GetBaseDeploymentParameters(hostingModel)); diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs index 68c1762048..7573fc021d 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs @@ -80,7 +80,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess // I think this test is flaky due to freb file not being created quickly enough. // Adding extra logging, marking as flaky, and repeating should help [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2570", FlakyOn.Helix.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2570")] [Repeat(10)] [RequiresIIS(IISCapability.FailedRequestTracingModule)] public async Task CheckFrebDisconnect() diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs index 72fcdeefd5..e896acc67c 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs @@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore/issues/7341", FlakyOn.Helix.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/7341")] public async Task ReadAndWriteSynchronously() { for (int i = 0; i < 100; i++) diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs index 6a15e6f4b3..0a60187315 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs @@ -171,7 +171,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests [ConditionalTheory] [MemberData(nameof(TestVariants))] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2200", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2200")] public async Task CheckUTF8File(TestVariant variant) { var path = "CheckConsoleFunctions"; diff --git a/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs b/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs index 5fe5d75a87..354b8a2229 100644 --- a/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs +++ b/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs @@ -225,7 +225,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests } [ConditionalFact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1817", FlakyOn.AzP.Windows)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1817")] public async Task ReaderThrowsResetExceptionOnInvalidBody() { var requestStartedCompletionSource = CreateTaskCompletionSource(); diff --git a/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs b/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs index de4189a207..787d3fe7c8 100644 --- a/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs +++ b/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs @@ -100,7 +100,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests [ConditionalTheory] [MemberData(nameof(IPEndPointRegistrationDataDynamicPort))] [IPv6SupportedCondition] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2074", FlakyOn.AzP.macOS)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2074")] public async Task RegisterIPEndPoint_DynamicPort_Success(IPEndPoint endPoint, string testUrl) { await RegisterIPEndPoint_Success(endPoint, testUrl); @@ -109,7 +109,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests [ConditionalTheory] [MemberData(nameof(IPEndPointRegistrationDataPort443))] [IPv6SupportedCondition] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")] public async Task RegisterIPEndPoint_Port443_Success(IPEndPoint endpoint, string testUrl) { if (!CanBindToEndpoint(endpoint.Address, 443)) @@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests [ConditionalTheory] [MemberData(nameof(AddressRegistrationDataIPv6Port5000Default))] [IPv6SupportedCondition] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")] public async Task RegisterAddresses_IPv6Port5000Default_Success(string addressInput, string[] testUrls) { if (!CanBindToEndpoint(IPAddress.Loopback, 5000) || !CanBindToEndpoint(IPAddress.IPv6Loopback, 5000)) @@ -145,7 +145,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests [ConditionalTheory] [MemberData(nameof(AddressRegistrationDataIPv6Port80))] [IPv6SupportedCondition] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")] public async Task RegisterAddresses_IPv6Port80_Success(string addressInput, string[] testUrls) { if (!CanBindToEndpoint(IPAddress.Loopback, 80) || !CanBindToEndpoint(IPAddress.IPv6Loopback, 80)) @@ -157,7 +157,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests } [ConditionalTheory] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2179", FlakyOn.Helix.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2179")] [MemberData(nameof(AddressRegistrationDataIPv6ScopeId))] [IPv6SupportedCondition] [IPv6ScopeIdPresentCondition] diff --git a/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs b/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs index b3c17c8ea3..59c3b9ce4c 100644 --- a/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs +++ b/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs @@ -108,7 +108,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests } } [Theory] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2489", FlakyOn.AzP.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2489")] [MemberData(nameof(LargeUploadData))] public async Task LargeUpload(long? maxRequestBufferSize, bool connectionAdapter, bool expectPause) { diff --git a/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs b/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs index cca069aeae..dc247c6543 100644 --- a/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs +++ b/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs @@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests #endif [ConditionalFact] [CollectDump] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task TestUnixDomainSocket() { var path = Path.GetTempFileName(); diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs index 43c92b0121..532629f0dd 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs @@ -142,7 +142,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests } [Fact] - [Flaky("https://github.com/aspnet/KestrelHttpServer/issues/2282", FlakyOn.AzP.macOS)] + [QuarantinedTest("https://github.com/aspnet/KestrelHttpServer/issues/2282")] public async Task ConnectionCountingReturnsToZero() { const int count = 100; diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs index 1d18ac20c0..5ae038fed1 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs @@ -263,7 +263,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests } [Fact] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1323", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1323")] public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnSmallWrite_AbortsConnectionAfterGracePeriod() { var mockSystemClock = _serviceContext.MockSystemClock; @@ -316,7 +316,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnLargeWrite_AbortsConnectionAfterRateTimeout() { var mockSystemClock = _serviceContext.MockSystemClock; diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs index ac206bcaa4..a0445ab86f 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs @@ -264,7 +264,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task MissingAuthority_200Status() { var headers = new[] diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs b/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs index d7b2066a3f..862adb0bb4 100644 --- a/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs +++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs @@ -21,7 +21,7 @@ namespace Interop.FunctionalTests { [ConditionalTheory] [MemberData(nameof(H2SpecTestCases))] - [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2225", FlakyOn.Helix.All)] + [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2225")] public async Task RunIndividualTestCase(H2SpecTestCase testCase) { var hostBuilder = new WebHostBuilder() diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs index 59c212d2ed..a3ac8e7656 100644 --- a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs +++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs @@ -1024,7 +1024,7 @@ namespace Interop.FunctionalTests } [Theory] - [Flaky("https://github.com/dotnet/runtime/issues/860", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/runtime/issues/860")] [MemberData(nameof(SupportedSchemes))] public async Task RequestHeaders_MultipleFrames_Accepted(string scheme) { @@ -1346,7 +1346,7 @@ namespace Interop.FunctionalTests // Settings_MaxFrameSize_Larger_Client - Not configurable [Theory] - [Flaky("https://github.com/dotnet/runtime/issues/860", FlakyOn.All)] + [QuarantinedTest("https://github.com/dotnet/runtime/issues/860")] [MemberData(nameof(SupportedSchemes))] public async Task Settings_MaxHeaderListSize_Server(string scheme) { diff --git a/src/Testing/src/FlakyOn.cs b/src/Testing/src/FlakyOn.cs deleted file mode 100644 index 81d9299043..0000000000 --- a/src/Testing/src/FlakyOn.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Microsoft.AspNetCore.Testing -{ - public static class FlakyOn - { - public const string All = "All"; - - public static class Helix - { - public const string All = QueuePrefix + "All"; - - public const string Fedora28Amd64 = QueuePrefix + HelixQueues.Fedora28Amd64; - public const string Fedora27Amd64 = QueuePrefix + HelixQueues.Fedora27Amd64; - public const string Redhat7Amd64 = QueuePrefix + HelixQueues.Redhat7Amd64; - public const string Debian9Amd64 = QueuePrefix + HelixQueues.Debian9Amd64; - public const string Debian8Amd64 = QueuePrefix + HelixQueues.Debian8Amd64; - public const string Centos7Amd64 = QueuePrefix + HelixQueues.Centos7Amd64; - public const string Ubuntu1604Amd64 = QueuePrefix + HelixQueues.Ubuntu1604Amd64; - public const string Ubuntu1810Amd64 = QueuePrefix + HelixQueues.Ubuntu1810Amd64; - public const string macOS1012Amd64 = QueuePrefix + HelixQueues.macOS1012Amd64; - public const string Windows10Amd64 = QueuePrefix + HelixQueues.Windows10Amd64; - - private const string Prefix = "Helix:"; - private const string QueuePrefix = Prefix + "Queue:"; - } - - public static class AzP - { - public const string All = Prefix + "All"; - public const string Windows = OsPrefix + "Windows_NT"; - public const string macOS = OsPrefix + "Darwin"; - public const string Linux = OsPrefix + "Linux"; - - private const string Prefix = "AzP:"; - private const string OsPrefix = Prefix + "OS:"; - } - } -} diff --git a/src/Testing/src/xunit/FlakyAttribute.cs b/src/Testing/src/xunit/FlakyAttribute.cs deleted file mode 100644 index f77c56dfc0..0000000000 --- a/src/Testing/src/xunit/FlakyAttribute.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using System.Collections.Generic; -using Xunit.Sdk; - -namespace Microsoft.AspNetCore.Testing -{ - /// - /// Marks a test as "Flaky" so that the build will sequester it and ignore failures. - /// - /// - /// - /// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute - /// properties. Once these traits are applied, build scripts can include/exclude tests based on them. - /// - /// - /// All flakiness-related traits start with Flaky: and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix. - /// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector. - /// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only - /// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks - /// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude - /// flaky tests. - /// - /// - /// - /// - /// [Fact] - /// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)] - /// public void FlakyTest() - /// { - /// // Flakiness - /// } - /// - /// - /// - /// The above example generates the following facets: - /// - /// - /// - /// - /// Flaky:Helix:Queue:Fedora.28.Amd64.Open = true - /// - /// - /// Flaky:AzP:OS:Darwin = true - /// - /// - /// - /// - /// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing -notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true" - /// to xunit.console.exe. Similarly, it can run only flaky tests using -trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true" - /// - /// - [TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(FlakyTraitDiscoverer), "Microsoft.AspNetCore.Testing")] - [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] - public sealed class FlakyAttribute : Attribute, ITraitAttribute - { - /// - /// Gets a URL to a GitHub issue tracking this flaky test. - /// - public string GitHubIssueUrl { get; } - - public IReadOnlyList Filters { get; } - - /// - /// Initializes a new instance of the class with the specified and a list of . If no - /// filters are provided, the test is considered flaky in all environments. - /// - /// - /// At least one filter is required. - /// - /// The URL to a GitHub issue tracking this flaky test. - /// The first filter that indicates where the test is flaky. Use a value from . - /// A list of additional filters that define where this test is flaky. Use values in . - public FlakyAttribute(string gitHubIssueUrl, string firstFilter, params string[] additionalFilters) - { - if (string.IsNullOrEmpty(gitHubIssueUrl)) - { - throw new ArgumentNullException(nameof(gitHubIssueUrl)); - } - - if (string.IsNullOrEmpty(firstFilter)) - { - throw new ArgumentNullException(nameof(firstFilter)); - } - - GitHubIssueUrl = gitHubIssueUrl; - var filters = new List(); - filters.Add(firstFilter); - filters.AddRange(additionalFilters); - Filters = filters; - } - } -} diff --git a/src/Testing/src/xunit/FlakyTraitDiscoverer.cs b/src/Testing/src/xunit/FlakyTraitDiscoverer.cs deleted file mode 100644 index 4e6bc27b1b..0000000000 --- a/src/Testing/src/xunit/FlakyTraitDiscoverer.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using Xunit.Abstractions; -using Xunit.Sdk; - -// Do not change this namespace without changing the usage in FlakyAttribute -namespace Microsoft.AspNetCore.Testing -{ - public class FlakyTraitDiscoverer : ITraitDiscoverer - { - public IEnumerable> GetTraits(IAttributeInfo traitAttribute) - { - if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute) - { - return GetTraitsCore(flakyAttribute); - } - else - { - throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection."); - } - } - - private IEnumerable> GetTraitsCore(FlakyAttribute attribute) - { - if (attribute.Filters.Count > 0) - { - foreach (var filter in attribute.Filters) - { - yield return new KeyValuePair($"Flaky:{filter}", "true"); - } - } - else - { - yield return new KeyValuePair($"Flaky:All", "true"); - } - } - } -} diff --git a/src/Testing/src/xunit/QuarantinedTestAttribute.cs b/src/Testing/src/xunit/QuarantinedTestAttribute.cs new file mode 100644 index 0000000000..350dcf1e12 --- /dev/null +++ b/src/Testing/src/xunit/QuarantinedTestAttribute.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Marks a test as "Quarantined" so that the build will sequester it and ignore failures. + /// + /// + /// + /// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute + /// properties. Once these traits are applied, build scripts can include/exclude tests based on them. + /// + /// + /// + /// + /// [Fact] + /// [QuarantinedTest] + /// public void FlakyTest() + /// { + /// // Flakiness + /// } + /// + /// + /// + /// The above example generates the following facet: + /// + /// + /// + /// + /// Quarantined = true + /// + /// + /// + [TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(QuarantinedTestTraitDiscoverer), "Microsoft.AspNetCore.Testing")] + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] + public sealed class QuarantinedTestAttribute : Attribute, ITraitAttribute + { + /// + /// Gets an optional reason for the quarantining, such as a link to a GitHub issue URL with more details as to why the test is quarantined. + /// + public string Reason { get; } + + /// + /// Initializes a new instance of the class with an optional . + /// + /// A reason that this test is quarantined. + public QuarantinedTestAttribute(string reason = null) + { + Reason = reason; + } + } +} diff --git a/src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs b/src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs new file mode 100644 index 0000000000..256e2b6cfc --- /dev/null +++ b/src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Xunit.Abstractions; +using Xunit.Sdk; + +// Do not change this namespace without changing the usage in QuarantinedTestAttribute +namespace Microsoft.AspNetCore.Testing +{ + public class QuarantinedTestTraitDiscoverer : ITraitDiscoverer + { + public IEnumerable> GetTraits(IAttributeInfo traitAttribute) + { + if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is QuarantinedTestAttribute quarantinedTestAttribute) + { + yield return new KeyValuePair("Quarantined", "true"); + } + else + { + throw new InvalidOperationException("The 'QuarantinedTest' attribute is only supported via reflection."); + } + } + } +} diff --git a/src/Testing/test/FlakyAttributeTest.cs b/src/Testing/test/FlakyAttributeTest.cs deleted file mode 100644 index ae06e5cf50..0000000000 --- a/src/Testing/test/FlakyAttributeTest.cs +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using Xunit; - -namespace Microsoft.AspNetCore.Testing.Tests -{ - public class FlakyAttributeTest - { - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.All)] - public void AlwaysFlakyInCI() - { - if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) - { - throw new Exception("Flaky!"); - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.Helix.All)] - public void FlakyInHelixOnly() - { - if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX"))) - { - throw new Exception("Flaky on Helix!"); - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.Helix.macOS1012Amd64, FlakyOn.Helix.Fedora28Amd64)] - public void FlakyInSpecificHelixQueue() - { - // Today we don't run Extensions tests on Helix, but this test should light up when we do. - var queueName = Environment.GetEnvironmentVariable("HELIX"); - if (!string.IsNullOrEmpty(queueName)) - { - var failingQueues = new HashSet(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 }; - if (failingQueues.Contains(queueName)) - { - throw new Exception($"Flaky on Helix Queue '{queueName}' !"); - } - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.AzP.All)] - public void FlakyInAzPOnly() - { - if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) - { - throw new Exception("Flaky on AzP!"); - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.AzP.Windows)] - public void FlakyInAzPWindowsOnly() - { - if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Windows_NT")) - { - throw new Exception("Flaky on AzP Windows!"); - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.AzP.macOS)] - public void FlakyInAzPmacOSOnly() - { - if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Darwin")) - { - throw new Exception("Flaky on AzP macOS!"); - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.AzP.Linux)] - public void FlakyInAzPLinuxOnly() - { - if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Linux")) - { - throw new Exception("Flaky on AzP Linux!"); - } - } - - [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] - [Flaky("http://example.com", FlakyOn.AzP.Linux, FlakyOn.AzP.macOS)] - public void FlakyInAzPNonWindowsOnly() - { - var agentOs = Environment.GetEnvironmentVariable("AGENT_OS"); - if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, "Darwin")) - { - throw new Exception("Flaky on AzP non-Windows!"); - } - } - } -} diff --git a/src/Testing/test/QuarantinedTestAttributeTest.cs b/src/Testing/test/QuarantinedTestAttributeTest.cs new file mode 100644 index 0000000000..5fc6c58041 --- /dev/null +++ b/src/Testing/test/QuarantinedTestAttributeTest.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.AspNetCore.Testing.Tests +{ + public class QuarantinedTestAttributeTest + { + [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")] + [QuarantinedTest] + public void AlwaysFlakyInCI() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) + { + throw new Exception("Flaky!"); + } + } + } +} diff --git a/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs b/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs index 26c9d3c1a1..0336cfdef2 100644 --- a/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs +++ b/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs @@ -17,7 +17,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests public OpenApiAddURLTests(ITestOutputHelper output) : base(output){ } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url_WithContentDisposition() { var project = CreateBasicProject(withOpenApi: false); @@ -50,7 +50,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenAPI_Add_Url_NoContentDisposition() { var project = CreateBasicProject(withOpenApi: false); @@ -84,7 +84,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenAPI_Add_Url_NoExtension_AssumesJson() { var project = CreateBasicProject(withOpenApi: false); @@ -118,7 +118,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url_NoSegment() { var project = CreateBasicProject(withOpenApi: false); @@ -152,7 +152,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url() { var project = CreateBasicProject(withOpenApi: false); @@ -185,7 +185,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url_SameName_UniqueFile() { var project = CreateBasicProject(withOpenApi: false); @@ -246,7 +246,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url_NSwagCSharp() { var project = CreateBasicProject(withOpenApi: false); @@ -279,7 +279,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url_NSwagTypeScript() { var project = CreateBasicProject(withOpenApi: false); @@ -312,7 +312,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_Url_OutputFile() { var project = CreateBasicProject(withOpenApi: false); @@ -345,7 +345,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenApi_Add_URL_FileAlreadyExists_Fail() { var project = CreateBasicProject(withOpenApi: false); @@ -404,7 +404,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public void OpenApi_Add_URL_MultipleTimes_OnlyOneReference() { var project = CreateBasicProject(withOpenApi: false); @@ -431,7 +431,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task OpenAPi_Add_URL_InvalidUrl() { var project = CreateBasicProject(withOpenApi: false); @@ -459,7 +459,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests Assert.False(File.Exists(jsonFile)); } - [Flaky("", FlakyOn.All)] + [QuarantinedTest] [Fact] public void OpenApi_Add_URL_ActualResponse() { diff --git a/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs b/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs index 3ba82db5d5..d299c208ca 100644 --- a/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs +++ b/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs @@ -57,6 +57,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests } [Fact] + [QuarantinedTest] public void AddsEscapedSpecificSecretIdToProject() { const string SecretId = @"&"; diff --git a/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs b/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs index 6d79642a4c..c16a89197f 100644 --- a/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs +++ b/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs @@ -37,7 +37,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests [Theory] [InlineData(null)] [InlineData("")] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public void Error_MissingId(string id) { var project = Path.Combine(_fixture.CreateProject(id), "TestProject.csproj"); @@ -83,7 +83,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests [Theory] [InlineData(true)] [InlineData(false)] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public void SetSecrets(bool fromCurrentDirectory) { var secrets = new KeyValuePair[] @@ -149,19 +149,20 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests } [Fact] + [QuarantinedTest] public void SetSecret_Update_Existing_Secret() { var projectPath = _fixture.GetTempSecretProject(); var secretManager = CreateProgram(); - secretManager.RunInternal("set", "secret1", "value1", "-p", projectPath); + secretManager.RunInternal("set", "secret1", "value1", "-p", projectPath, "--verbose"); Assert.Contains("Successfully saved secret1 = value1 to the secret store.", _console.GetOutput()); - secretManager.RunInternal("set", "secret1", "value2", "-p", projectPath); + secretManager.RunInternal("set", "secret1", "value2", "-p", projectPath, "--verbose"); Assert.Contains("Successfully saved secret1 = value2 to the secret store.", _console.GetOutput()); _console.ClearOutput(); - secretManager.RunInternal("list", "-p", projectPath); + secretManager.RunInternal("list", "-p", projectPath, "--verbose"); Assert.Contains("secret1 = value2", _console.GetOutput()); } @@ -186,7 +187,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public void Remove_Non_Existing_Secret() { var projectPath = _fixture.GetTempSecretProject(); @@ -212,7 +213,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public void List_Flattens_Nested_Objects() { string secretId; @@ -262,15 +263,16 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests } [Fact] + [QuarantinedTest] public void List_Empty_Secrets_File() { var projectPath = _fixture.GetTempSecretProject(); var secretManager = CreateProgram(); - secretManager.RunInternal("list", "-p", projectPath); + secretManager.RunInternal("list", "-p", projectPath, "--verbose"); Assert.Contains(Resources.Error_No_Secrets_Found, _console.GetOutput()); } - [Flaky("", FlakyOn.All)] + [QuarantinedTest] [Theory] [InlineData(true)] [InlineData(false)] diff --git a/src/Tools/dotnet-watch/test/AppWithDepsTests.cs b/src/Tools/dotnet-watch/test/AppWithDepsTests.cs index 1ec8a59284..5954f540b5 100644 --- a/src/Tools/dotnet-watch/test/AppWithDepsTests.cs +++ b/src/Tools/dotnet-watch/test/AppWithDepsTests.cs @@ -20,7 +20,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task ChangeFileInDependency() { await _app.StartWatcherAsync(); diff --git a/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs b/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs index 3ef636b009..d07f4e25a4 100644 --- a/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs +++ b/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs @@ -23,7 +23,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task RunsWithDotnetWatchEnvVariable() { Assert.True(string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_WATCH")), "DOTNET_WATCH cannot be set already when this test is running"); @@ -36,7 +36,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task RunsWithIterationEnvVariable() { await _app.StartWatcherAsync(); diff --git a/src/Tools/dotnet-watch/test/GlobbingAppTests.cs b/src/Tools/dotnet-watch/test/GlobbingAppTests.cs index 25dc5b3616..9383b2d728 100644 --- a/src/Tools/dotnet-watch/test/GlobbingAppTests.cs +++ b/src/Tools/dotnet-watch/test/GlobbingAppTests.cs @@ -24,7 +24,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests [Theory] [InlineData(true)] [InlineData(false)] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task ChangeCompiledFile(bool usePollingWatcher) { _app.UsePollingWatcher = usePollingWatcher; @@ -43,7 +43,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task DeleteCompiledFile() { await _app.StartWatcherAsync(); @@ -60,7 +60,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task DeleteSourceFolder() { await _app.StartWatcherAsync(); @@ -77,7 +77,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task RenameCompiledFile() { await _app.StartWatcherAsync(); @@ -90,7 +90,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task ChangeExcludedFile() { await _app.StartWatcherAsync(); diff --git a/src/Tools/dotnet-watch/test/NoDepsAppTests.cs b/src/Tools/dotnet-watch/test/NoDepsAppTests.cs index 5b1cc36788..b7171d5ad2 100644 --- a/src/Tools/dotnet-watch/test/NoDepsAppTests.cs +++ b/src/Tools/dotnet-watch/test/NoDepsAppTests.cs @@ -24,7 +24,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task RestartProcessOnFileChange() { await _app.StartWatcherAsync(new[] { "--no-exit" }); @@ -43,7 +43,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests } [Fact] - [Flaky("", FlakyOn.All)] + [QuarantinedTest] public async Task RestartProcessThatTerminatesAfterFileChange() { await _app.StartWatcherAsync();