huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rename Flaky attribute (#19469)

This commit is contained in:
Brennan 2020-03-04 08:10:18 -08:00 committed by GitHub
parent cda762685a
commit 56fb4b4d2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 209 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.azure/pipelines/ci.yml
View File

@ -499,15 +499,15 @@ stages:
- powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
displayName: Setup IISExpress test certificates and schema
afterBuild:
- powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true"
displayName: Run Flaky Tests
- powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunQuarantinedTests=true"
displayName: Run Quarantined Tests
continueOnError: true
- task: PublishTestResults@2
displayName: Publish Flaky Test Results
displayName: Publish Quarantined Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
artifacts:
- name: Windows_Test_Dumps
path: artifacts/dumps/
@ -574,15 +574,15 @@ stages:
displayName: Pack Packages (for Template tests)
- bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog"
displayName: Pack Templates (for Template tests)
- bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
displayName: Run Flaky Tests
- bash: ./build.sh --no-build --ci --test -p:RunQuarantinedTests=true
displayName: Run Quarantined Tests
continueOnError: true
- task: PublishTestResults@2
displayName: Publish Flaky Test Results
displayName: Publish Quarantined Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
artifacts:
- name: MacOS_Test_Logs
path: artifacts/log/
@ -611,15 +611,15 @@ stages:
displayName: Pack Packages (for Template tests)
- bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog"
displayName: Pack Templates (for Template tests)
- bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
displayName: Run Flaky Tests
- bash: ./build.sh --no-build --ci --test -p:RunQuarantinedTests=true
displayName: Run Quarantined Tests
continueOnError: true
- task: PublishTestResults@2
displayName: Publish Flaky Test Results
displayName: Publish Quarantined Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
artifacts:
- name: Linux_Test_Logs
path: artifacts/log/

4
Directory.Build.props
View File

@ -35,9 +35,9 @@
<DisableImplicitComponentsAnalyzers>true</DisableImplicitComponentsAnalyzers>
</PropertyGroup>
<Import Project="eng\FlakyTests.BeforeArcade.props" />
<Import Project="eng\QuarantinedTests.BeforeArcade.props" />
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="eng\FlakyTests.AfterArcade.props" />
<Import Project="eng\QuarantinedTests.AfterArcade.props" />
<PropertyGroup>
<Product>Microsoft ASP.NET Core</Product>

7
eng/FlakyTests.AfterArcade.props
View File

@ -1,7 +0,0 @@
<Project>
<!-- Override where xUnit logs and results go if we're doing the flaky run -->
<PropertyGroup Condition="'$(RunFlakyTests)' == 'true'">
<ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Flaky\</ArtifactsLogDir>
<ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Flaky\</ArtifactsTestResultsDir>
</PropertyGroup>
</Project>

18
eng/FlakyTests.BeforeArcade.props
View File

@ -1,18 +0,0 @@
<Project>
<!-- Local Dev Flakiness -->
<PropertyGroup>
<_FlakyRunAdditionalArgs>-trait "Flaky:All=true"</_FlakyRunAdditionalArgs>
<_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true"</_NonFlakyRunAdditionalArgs>
</PropertyGroup>
<!-- Azure Pipelines Flakiness -->
<PropertyGroup Condition="'$(TF_BUILD)' != ''">
<_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
<_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
</PropertyGroup>
<PropertyGroup>
<TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == ''">$(_NonFlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == 'true'">$(_FlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
</PropertyGroup>
</Project>

7
eng/QuarantinedTests.AfterArcade.props Normal file
View File

@ -0,0 +1,7 @@
<Project>
<!-- Override where xUnit logs and results go if we're doing the Quarantined run -->
<PropertyGroup Condition="'$(RunQuarantinedTests)' == 'true'">
<ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Quarantined\</ArtifactsLogDir>
<ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Quarantined\</ArtifactsTestResultsDir>
</PropertyGroup>
</Project>

11
eng/QuarantinedTests.BeforeArcade.props Normal file
View File

@ -0,0 +1,11 @@
<Project>
<PropertyGroup>
<_QuarantinedTestRunAdditionalArgs>-trait "Quarantined=true"</_QuarantinedTestRunAdditionalArgs>
<_NonQuarantinedTestRunAdditionalArgs>-notrait "Quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
</PropertyGroup>
<PropertyGroup>
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == ''">$(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
</PropertyGroup>
</Project>

10
eng/helix/content/runtests.cmd
View File

@ -53,23 +53,21 @@ if %$quarantined%==True (
set %$quarantined=true
)
set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
set QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
set NONQUARANTINE_FILTER="Quarantined!=true"
set QUARANTINE_FILTER="Quarantined=true"
if %$quarantined%==true (
echo Running quarantined tests.
%DOTNET_ROOT%\dotnet vstest %$target% --logger:xunit --TestCaseFilter:%QUARANTINE_FILTER%
if errorlevel 1 (
echo Failure in flaky test 1>&2
echo Failure in quarantined test 1>&2
REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
)
) else (
REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
REM only puts the explicit filter traits the user provided in
REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
echo Running non-quarantined tests.
%DOTNET_ROOT%\dotnet vstest %$target% --logger:xunit --TestCaseFilter:%NONQUARANTINE_FILTER%
if errorlevel 1 (
echo Failure in non-flaky test 1>&2
echo Failure in non-quarantined test 1>&2
set exit_code=1
REM DO NOT EXIT
)

6
eng/helix/content/runtests.sh
View File

@ -115,11 +115,9 @@ fi
exit_code=0
# We need to specify all possible quarantined filters that apply to this environment, because the quarantine attribute
# only puts the explicit filter traits the user provided in the flaky attribute
# Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true"
NONQUARANTINE_FILTER="Quarantined!=true"
QUARANTINE_FILTER="Quarantined=true"
if [ "$quarantined" == true ]; then
echo "Running all tests including quarantined."
$DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$QUARANTINE_FILTER"

2
src/Components/test/E2ETest/Tests/EventTest.cs
View File

@ -167,7 +167,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
}
[Fact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1987", FlakyOn.AzP.Windows)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1987")]
public void InputEvent_RespondsOnKeystrokes()
{
Browser.MountTestComponent<InputEventComponent>();

2
src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs
View File

@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Identity.Test
[Theory]
[MemberData(nameof(ScriptWithFallbackSrcData))]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2267", FlakyOn.AzP.macOS)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2267")]
public async Task IdentityUI_ScriptTags_FallbackSourceContent_Matches_CDNContent(ScriptTag scriptTag)
{
var wwwrootDir = Path.Combine(GetProjectBasePath(), "wwwroot", scriptTag.Version);

4
src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs
View File

@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
builder.UseStartup<BasicWebSite.StartupWithCookieTempDataProviderAndCookieConsent>();
[Fact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1803", FlakyOn.AzP.Windows)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1803")]
public async Task CookieTempDataProviderCookie_SetInResponse_OnGrantingConsent()
{
// Arrange
@ -64,7 +64,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
}
[Fact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1803", FlakyOn.AzP.Windows)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1803")]
public async Task CookieTempDataProviderCookie_NotSetInResponse_OnNoConsent()
{
// Arrange

4
src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs
View File

@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.Listener
public class RequestBodyTests
{
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1826", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1826")]
public async Task RequestBody_SyncReadDisabledByDefault_WorksWhenEnabled()
{
string address;
@ -142,7 +142,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.Listener
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2206", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2206")]
public async Task RequestBody_ReadAsyncPartialBodyAndExpiredTimeout_Canceled()
{
StaggardContent content = new StaggardContent();

8
src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs
View File

@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2135", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2135")]
public async Task Caching_JustPublic_NotCached()
{
var requestCount = 1;
@ -86,7 +86,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2207", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2207")]
public async Task Caching_WithoutContentType_Cached_OnWin7AndWin2008R2()
{
if (Utilities.IsWin8orLater)
@ -237,7 +237,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
[ConditionalTheory]
[InlineData("0")]
[InlineData("-1")]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2208", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2208")]
public async Task Caching_InvalidExpires_NotCached(string expiresValue)
{
var requestCount = 1;
@ -378,7 +378,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2209", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2209")]
public async Task Caching_VariousStatusCodes_Cached()
{
var requestCount = 1;

2
src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs
View File

@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2267", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2267")]
public async Task Server_ShutdownDuringRequest_Success()
{
Task<string> responseTask;

2
src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs
View File

@ -211,7 +211,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[ConditionalTheory]
[InlineData(HostingModel.InProcess)]
[InlineData(HostingModel.OutOfProcess)]
[Flaky("https://github.com/dotnet/aspnetcore/issues/7075", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/7075")]
public async Task AppOfflineAddedAndRemovedStress(HostingModel hostingModel)
{
var deploymentResult = await AssertStarts(hostingModel);

2
src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs
View File

@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[ConditionalTheory]
[InlineData(HostingModel.InProcess)]
[InlineData(HostingModel.OutOfProcess)]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1794", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1794")]
public async Task ConfigurationTouchedStress(HostingModel hostingModel)
{
var deploymentResult = await DeployAsync(Fixture.GetBaseDeploymentParameters(hostingModel));

2
src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs
View File

@ -80,7 +80,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess
// I think this test is flaky due to freb file not being created quickly enough.
// Adding extra logging, marking as flaky, and repeating should help
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2570", FlakyOn.Helix.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2570")]
[Repeat(10)]
[RequiresIIS(IISCapability.FailedRequestTracingModule)]
public async Task CheckFrebDisconnect()

2
src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs
View File

@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore/issues/7341", FlakyOn.Helix.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/7341")]
public async Task ReadAndWriteSynchronously()
{
for (int i = 0; i < 100; i++)

2
src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs
View File

@ -171,7 +171,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(TestVariants))]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2200", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2200")]
public async Task CheckUTF8File(TestVariant variant)
{
var path = "CheckConsoleFunctions";

2
src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs
View File

@ -225,7 +225,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
}
[ConditionalFact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1817", FlakyOn.AzP.Windows)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1817")]
public async Task ReaderThrowsResetExceptionOnInvalidBody()
{
var requestStartedCompletionSource = CreateTaskCompletionSource();

10
src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs
View File

@ -100,7 +100,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(IPEndPointRegistrationDataDynamicPort))]
[IPv6SupportedCondition]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2074", FlakyOn.AzP.macOS)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2074")]
public async Task RegisterIPEndPoint_DynamicPort_Success(IPEndPoint endPoint, string testUrl)
{
await RegisterIPEndPoint_Success(endPoint, testUrl);
@ -109,7 +109,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(IPEndPointRegistrationDataPort443))]
[IPv6SupportedCondition]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")]
public async Task RegisterIPEndPoint_Port443_Success(IPEndPoint endpoint, string testUrl)
{
if (!CanBindToEndpoint(endpoint.Address, 443))
@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(AddressRegistrationDataIPv6Port5000Default))]
[IPv6SupportedCondition]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")]
public async Task RegisterAddresses_IPv6Port5000Default_Success(string addressInput, string[] testUrls)
{
if (!CanBindToEndpoint(IPAddress.Loopback, 5000) || !CanBindToEndpoint(IPAddress.IPv6Loopback, 5000))
@ -145,7 +145,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(AddressRegistrationDataIPv6Port80))]
[IPv6SupportedCondition]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")]
public async Task RegisterAddresses_IPv6Port80_Success(string addressInput, string[] testUrls)
{
if (!CanBindToEndpoint(IPAddress.Loopback, 80) || !CanBindToEndpoint(IPAddress.IPv6Loopback, 80))
@ -157,7 +157,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
}
[ConditionalTheory]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2179", FlakyOn.Helix.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2179")]
[MemberData(nameof(AddressRegistrationDataIPv6ScopeId))]
[IPv6SupportedCondition]
[IPv6ScopeIdPresentCondition]

2
src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs
View File

@ -108,7 +108,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
}
}
[Theory]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2489", FlakyOn.AzP.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2489")]
[MemberData(nameof(LargeUploadData))]
public async Task LargeUpload(long? maxRequestBufferSize, bool connectionAdapter, bool expectPause)
{

2
src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs
View File

@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
#endif
[ConditionalFact]
[CollectDump]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task TestUnixDomainSocket()
{
var path = Path.GetTempFileName();

2
src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs
View File

@ -142,7 +142,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
}
[Fact]
[Flaky("https://github.com/aspnet/KestrelHttpServer/issues/2282", FlakyOn.AzP.macOS)]
[QuarantinedTest("https://github.com/aspnet/KestrelHttpServer/issues/2282")]
public async Task ConnectionCountingReturnsToZero()
{
const int count = 100;

4
src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs
View File

@ -263,7 +263,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[Fact]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1323", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1323")]
public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnSmallWrite_AbortsConnectionAfterGracePeriod()
{
var mockSystemClock = _serviceContext.MockSystemClock;
@ -316,7 +316,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[Fact]
[Flaky("<no longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnLargeWrite_AbortsConnectionAfterRateTimeout()
{
var mockSystemClock = _serviceContext.MockSystemClock;

2
src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs
View File

@ -264,7 +264,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task MissingAuthority_200Status()
{
var headers = new[]

2
src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs
View File

@ -21,7 +21,7 @@ namespace Interop.FunctionalTests
{
[ConditionalTheory]
[MemberData(nameof(H2SpecTestCases))]
[Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2225", FlakyOn.Helix.All)]
[QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2225")]
public async Task RunIndividualTestCase(H2SpecTestCase testCase)
{
var hostBuilder = new WebHostBuilder()

4
src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs
View File

@ -1024,7 +1024,7 @@ namespace Interop.FunctionalTests
}
[Theory]
[Flaky("https://github.com/dotnet/runtime/issues/860", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/runtime/issues/860")]
[MemberData(nameof(SupportedSchemes))]
public async Task RequestHeaders_MultipleFrames_Accepted(string scheme)
{
@ -1346,7 +1346,7 @@ namespace Interop.FunctionalTests
// Settings_MaxFrameSize_Larger_Client - Not configurable
[Theory]
[Flaky("https://github.com/dotnet/runtime/issues/860", FlakyOn.All)]
[QuarantinedTest("https://github.com/dotnet/runtime/issues/860")]
[MemberData(nameof(SupportedSchemes))]
public async Task Settings_MaxHeaderListSize_Server(string scheme)
{

41
src/Testing/src/FlakyOn.cs
View File

@ -1,41 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Microsoft.AspNetCore.Testing
{
public static class FlakyOn
{
public const string All = "All";
public static class Helix
{
public const string All = QueuePrefix + "All";
public const string Fedora28Amd64 = QueuePrefix + HelixQueues.Fedora28Amd64;
public const string Fedora27Amd64 = QueuePrefix + HelixQueues.Fedora27Amd64;
public const string Redhat7Amd64 = QueuePrefix + HelixQueues.Redhat7Amd64;
public const string Debian9Amd64 = QueuePrefix + HelixQueues.Debian9Amd64;
public const string Debian8Amd64 = QueuePrefix + HelixQueues.Debian8Amd64;
public const string Centos7Amd64 = QueuePrefix + HelixQueues.Centos7Amd64;
public const string Ubuntu1604Amd64 = QueuePrefix + HelixQueues.Ubuntu1604Amd64;
public const string Ubuntu1810Amd64 = QueuePrefix + HelixQueues.Ubuntu1810Amd64;
public const string macOS1012Amd64 = QueuePrefix + HelixQueues.macOS1012Amd64;
public const string Windows10Amd64 = QueuePrefix + HelixQueues.Windows10Amd64;
private const string Prefix = "Helix:";
private const string QueuePrefix = Prefix + "Queue:";
}
public static class AzP
{
public const string All = Prefix + "All";
public const string Windows = OsPrefix + "Windows_NT";
public const string macOS = OsPrefix + "Darwin";
public const string Linux = OsPrefix + "Linux";
private const string Prefix = "AzP:";
private const string OsPrefix = Prefix + "OS:";
}
}
}

92
src/Testing/src/xunit/FlakyAttribute.cs
View File

@ -1,92 +0,0 @@
using System;
using System.Collections.Generic;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Testing
{
/// <summary>
/// Marks a test as "Flaky" so that the build will sequester it and ignore failures.
/// </summary>
/// <remarks>
/// <para>
/// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute
/// properties. Once these traits are applied, build scripts can include/exclude tests based on them.
/// </para>
/// <para>
/// All flakiness-related traits start with <c>Flaky:</c> and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix.
/// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector.
/// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only
/// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks
/// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude
/// flaky tests.
/// </para>
/// </remarks>
/// <example>
/// <code>
/// [Fact]
/// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)]
/// public void FlakyTest()
/// {
/// // Flakiness
/// }
/// </code>
///
/// <para>
/// The above example generates the following facets:
/// </para>
///
/// <list type="bullet">
/// <item>
/// <description><c>Flaky:Helix:Queue:Fedora.28.Amd64.Open</c> = <c>true</c></description>
/// </item>
/// <item>
/// <description><c>Flaky:AzP:OS:Darwin</c> = <c>true</c></description>
/// </item>
/// </list>
///
/// <para>
/// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing <c>-notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true"</c>
/// to <c>xunit.console.exe</c>. Similarly, it can run only flaky tests using <c>-trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true"</c>
/// </para>
/// </example>
[TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(FlakyTraitDiscoverer), "Microsoft.AspNetCore.Testing")]
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
public sealed class FlakyAttribute : Attribute, ITraitAttribute
{
/// <summary>
/// Gets a URL to a GitHub issue tracking this flaky test.
/// </summary>
public string GitHubIssueUrl { get; }
public IReadOnlyList<string> Filters { get; }
/// <summary>
/// Initializes a new instance of the <see cref="FlakyAttribute"/> class with the specified <see cref="GitHubIssueUrl"/> and a list of <see cref="Filters"/>. If no
/// filters are provided, the test is considered flaky in all environments.
/// </summary>
/// <remarks>
/// At least one filter is required.
/// </remarks>
/// <param name="gitHubIssueUrl">The URL to a GitHub issue tracking this flaky test.</param>
/// <param name="firstFilter">The first filter that indicates where the test is flaky. Use a value from <see cref="FlakyOn"/>.</param>
/// <param name="additionalFilters">A list of additional filters that define where this test is flaky. Use values in <see cref="FlakyOn"/>.</param>
public FlakyAttribute(string gitHubIssueUrl, string firstFilter, params string[] additionalFilters)
{
if (string.IsNullOrEmpty(gitHubIssueUrl))
{
throw new ArgumentNullException(nameof(gitHubIssueUrl));
}
if (string.IsNullOrEmpty(firstFilter))
{
throw new ArgumentNullException(nameof(firstFilter));
}
GitHubIssueUrl = gitHubIssueUrl;
var filters = new List<string>();
filters.Add(firstFilter);
filters.AddRange(additionalFilters);
Filters = filters;
}
}
}

38
src/Testing/src/xunit/FlakyTraitDiscoverer.cs
View File

@ -1,38 +0,0 @@
using System;
using System.Collections.Generic;
using Xunit.Abstractions;
using Xunit.Sdk;
// Do not change this namespace without changing the usage in FlakyAttribute
namespace Microsoft.AspNetCore.Testing
{
public class FlakyTraitDiscoverer : ITraitDiscoverer
{
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
{
if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute)
{
return GetTraitsCore(flakyAttribute);
}
else
{
throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection.");
}
}
private IEnumerable<KeyValuePair<string, string>> GetTraitsCore(FlakyAttribute attribute)
{
if (attribute.Filters.Count > 0)
{
foreach (var filter in attribute.Filters)
{
yield return new KeyValuePair<string, string>($"Flaky:{filter}", "true");
}
}
else
{
yield return new KeyValuePair<string, string>($"Flaky:All", "true");
}
}
}
}

57
src/Testing/src/xunit/QuarantinedTestAttribute.cs Normal file
View File

@ -0,0 +1,57 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Testing
{
/// <summary>
/// Marks a test as "Quarantined" so that the build will sequester it and ignore failures.
/// </summary>
/// <remarks>
/// <para>
/// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute
/// properties. Once these traits are applied, build scripts can include/exclude tests based on them.
/// </para>
/// </remarks>
/// <example>
/// <code>
/// [Fact]
/// [QuarantinedTest]
/// public void FlakyTest()
/// {
/// // Flakiness
/// }
/// </code>
///
/// <para>
/// The above example generates the following facet:
/// </para>
///
/// <list type="bullet">
/// <item>
/// <description><c>Quarantined</c> = <c>true</c></description>
/// </item>
/// </list>
/// </example>
[TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(QuarantinedTestTraitDiscoverer), "Microsoft.AspNetCore.Testing")]
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
public sealed class QuarantinedTestAttribute : Attribute, ITraitAttribute
{
/// <summary>
/// Gets an optional reason for the quarantining, such as a link to a GitHub issue URL with more details as to why the test is quarantined.
/// </summary>
public string Reason { get; }
/// <summary>
/// Initializes a new instance of the <see cref="QuarantinedTestAttribute"/> class with an optional <see cref="Reason"/>.
/// </summary>
/// <param name="reason">A reason that this test is quarantined.</param>
public QuarantinedTestAttribute(string reason = null)
{
Reason = reason;
}
}
}

26
src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs Normal file
View File

@ -0,0 +1,26 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using Xunit.Abstractions;
using Xunit.Sdk;
// Do not change this namespace without changing the usage in QuarantinedTestAttribute
namespace Microsoft.AspNetCore.Testing
{
public class QuarantinedTestTraitDiscoverer : ITraitDiscoverer
{
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
{
if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is QuarantinedTestAttribute quarantinedTestAttribute)
{
yield return new KeyValuePair<string, string>("Quarantined", "true");
}
else
{
throw new InvalidOperationException("The 'QuarantinedTest' attribute is only supported via reflection.");
}
}
}
}

99
src/Testing/test/FlakyAttributeTest.cs
View File

@ -1,99 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using Xunit;
namespace Microsoft.AspNetCore.Testing.Tests
{
public class FlakyAttributeTest
{
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.All)]
public void AlwaysFlakyInCI()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
{
throw new Exception("Flaky!");
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.Helix.All)]
public void FlakyInHelixOnly()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")))
{
throw new Exception("Flaky on Helix!");
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.Helix.macOS1012Amd64, FlakyOn.Helix.Fedora28Amd64)]
public void FlakyInSpecificHelixQueue()
{
// Today we don't run Extensions tests on Helix, but this test should light up when we do.
var queueName = Environment.GetEnvironmentVariable("HELIX");
if (!string.IsNullOrEmpty(queueName))
{
var failingQueues = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 };
if (failingQueues.Contains(queueName))
{
throw new Exception($"Flaky on Helix Queue '{queueName}' !");
}
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.AzP.All)]
public void FlakyInAzPOnly()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
{
throw new Exception("Flaky on AzP!");
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.AzP.Windows)]
public void FlakyInAzPWindowsOnly()
{
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Windows_NT"))
{
throw new Exception("Flaky on AzP Windows!");
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.AzP.macOS)]
public void FlakyInAzPmacOSOnly()
{
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Darwin"))
{
throw new Exception("Flaky on AzP macOS!");
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.AzP.Linux)]
public void FlakyInAzPLinuxOnly()
{
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Linux"))
{
throw new Exception("Flaky on AzP Linux!");
}
}
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[Flaky("http://example.com", FlakyOn.AzP.Linux, FlakyOn.AzP.macOS)]
public void FlakyInAzPNonWindowsOnly()
{
var agentOs = Environment.GetEnvironmentVariable("AGENT_OS");
if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, "Darwin"))
{
throw new Exception("Flaky on AzP non-Windows!");
}
}
}
}

22
src/Testing/test/QuarantinedTestAttributeTest.cs Normal file
View File

@ -0,0 +1,22 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using Xunit;
namespace Microsoft.AspNetCore.Testing.Tests
{
public class QuarantinedTestAttributeTest
{
[Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
[QuarantinedTest]
public void AlwaysFlakyInCI()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
{
throw new Exception("Flaky!");
}
}
}
}

26
src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs
View File

@ -17,7 +17,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
public OpenApiAddURLTests(ITestOutputHelper output) : base(output){ }
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url_WithContentDisposition()
{
var project = CreateBasicProject(withOpenApi: false);
@ -50,7 +50,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenAPI_Add_Url_NoContentDisposition()
{
var project = CreateBasicProject(withOpenApi: false);
@ -84,7 +84,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenAPI_Add_Url_NoExtension_AssumesJson()
{
var project = CreateBasicProject(withOpenApi: false);
@ -118,7 +118,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url_NoSegment()
{
var project = CreateBasicProject(withOpenApi: false);
@ -152,7 +152,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url()
{
var project = CreateBasicProject(withOpenApi: false);
@ -185,7 +185,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url_SameName_UniqueFile()
{
var project = CreateBasicProject(withOpenApi: false);
@ -246,7 +246,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url_NSwagCSharp()
{
var project = CreateBasicProject(withOpenApi: false);
@ -279,7 +279,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url_NSwagTypeScript()
{
var project = CreateBasicProject(withOpenApi: false);
@ -312,7 +312,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_Url_OutputFile()
{
var project = CreateBasicProject(withOpenApi: false);
@ -345,7 +345,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenApi_Add_URL_FileAlreadyExists_Fail()
{
var project = CreateBasicProject(withOpenApi: false);
@ -404,7 +404,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public void OpenApi_Add_URL_MultipleTimes_OnlyOneReference()
{
var project = CreateBasicProject(withOpenApi: false);
@ -431,7 +431,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task OpenAPi_Add_URL_InvalidUrl()
{
var project = CreateBasicProject(withOpenApi: false);
@ -459,7 +459,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
Assert.False(File.Exists(jsonFile));
}
[Flaky("<No longer needed, tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
[Fact]
public void OpenApi_Add_URL_ActualResponse()
{

1
src/Tools/dotnet-user-secrets/test/InitCommandTest.cs
View File

@ -57,6 +57,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
[QuarantinedTest]
public void AddsEscapedSpecificSecretIdToProject()
{
const string SecretId = @"<lots of XML invalid values>&";

20
src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs
View File

@ -37,7 +37,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
[Theory]
[InlineData(null)]
[InlineData("")]
[Flaky("<No longer used; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public void Error_MissingId(string id)
{
var project = Path.Combine(_fixture.CreateProject(id), "TestProject.csproj");
@ -83,7 +83,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
[Theory]
[InlineData(true)]
[InlineData(false)]
[Flaky("<No longer used; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public void SetSecrets(bool fromCurrentDirectory)
{
var secrets = new KeyValuePair<string, string>[]
@ -149,19 +149,20 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
[QuarantinedTest]
public void SetSecret_Update_Existing_Secret()
{
var projectPath = _fixture.GetTempSecretProject();
var secretManager = CreateProgram();
secretManager.RunInternal("set", "secret1", "value1", "-p", projectPath);
secretManager.RunInternal("set", "secret1", "value1", "-p", projectPath, "--verbose");
Assert.Contains("Successfully saved secret1 = value1 to the secret store.", _console.GetOutput());
secretManager.RunInternal("set", "secret1", "value2", "-p", projectPath);
secretManager.RunInternal("set", "secret1", "value2", "-p", projectPath, "--verbose");
Assert.Contains("Successfully saved secret1 = value2 to the secret store.", _console.GetOutput());
_console.ClearOutput();
secretManager.RunInternal("list", "-p", projectPath);
secretManager.RunInternal("list", "-p", projectPath, "--verbose");
Assert.Contains("secret1 = value2", _console.GetOutput());
}
@ -186,7 +187,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
[Flaky("<no longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public void Remove_Non_Existing_Secret()
{
var projectPath = _fixture.GetTempSecretProject();
@ -212,7 +213,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
[Flaky("<No longer used; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public void List_Flattens_Nested_Objects()
{
string secretId;
@ -262,15 +263,16 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
[QuarantinedTest]
public void List_Empty_Secrets_File()
{
var projectPath = _fixture.GetTempSecretProject();
var secretManager = CreateProgram();
secretManager.RunInternal("list", "-p", projectPath);
secretManager.RunInternal("list", "-p", projectPath, "--verbose");
Assert.Contains(Resources.Error_No_Secrets_Found, _console.GetOutput());
}
[Flaky("<No longer needed, tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
[Theory]
[InlineData(true)]
[InlineData(false)]

2
src/Tools/dotnet-watch/test/AppWithDepsTests.cs
View File

@ -20,7 +20,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task ChangeFileInDependency()
{
await _app.StartWatcherAsync();

4
src/Tools/dotnet-watch/test/DotNetWatcherTests.cs
View File

@ -23,7 +23,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task RunsWithDotnetWatchEnvVariable()
{
Assert.True(string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_WATCH")), "DOTNET_WATCH cannot be set already when this test is running");
@ -36,7 +36,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task RunsWithIterationEnvVariable()
{
await _app.StartWatcherAsync();

10
src/Tools/dotnet-watch/test/GlobbingAppTests.cs
View File

@ -24,7 +24,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
[Theory]
[InlineData(true)]
[InlineData(false)]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task ChangeCompiledFile(bool usePollingWatcher)
{
_app.UsePollingWatcher = usePollingWatcher;
@ -43,7 +43,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task DeleteCompiledFile()
{
await _app.StartWatcherAsync();
@ -60,7 +60,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task DeleteSourceFolder()
{
await _app.StartWatcherAsync();
@ -77,7 +77,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task RenameCompiledFile()
{
await _app.StartWatcherAsync();
@ -90,7 +90,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task ChangeExcludedFile()
{
await _app.StartWatcherAsync();

4
src/Tools/dotnet-watch/test/NoDepsAppTests.cs
View File

@ -24,7 +24,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task RestartProcessOnFileChange()
{
await _app.StartWatcherAsync(new[] { "--no-exit" });
@ -43,7 +43,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
[Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
[QuarantinedTest]
public async Task RestartProcessThatTerminatesAfterFileChange()
{
await _app.StartWatcherAsync();