From 256045729a864ced6e3998878e2530dceeb3b67e Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Tue, 2 Jun 2020 12:19:28 -0700 Subject: [PATCH] Add `SuccessfulTests` to ensure something runs in every non-Helix run (#22406) * Remove extra `[SkipOnHelix]` attribute - only need the one compiled into Microsoft.AspNetCore.Testing - update the documentation to reflect this - nit: address Markdown warnings that VS Code showed * Add `SuccessfulTests` to ensure something runs in every non-Helix runs - #22241 - cleans up hundreds of warnings but leaves a couple for the Blazor tests assembly - see comments about xUnit runner command line in the new class --- docs/Helix.md | 25 ++++++---- eng/helix/content/RunTests/TestRunner.cs | 2 +- eng/targets/CSharp.Common.props | 1 + ...Microsoft.AspNetCore.Analyzers.Test.csproj | 1 - ...pNetCore.Components.Analyzers.Tests.csproj | 1 - ...Core.Components.Authorization.Tests.csproj | 1 - ...crosoft.AspNetCore.Components.Tests.csproj | 1 - ...t.AspNetCore.Components.Forms.Tests.csproj | 1 - ...oft.AspNetCore.Components.Web.Tests.csproj | 1 - ....Components.WebAssembly.Build.Tests.csproj | 1 - .../test/RuntimeDependenciesResolverTest.cs | 2 +- ...ore.DataProtection.Extensions.Tests.csproj | 1 - .../Microsoft.AspNetCore.Hosting.Tests.csproj | 1 - src/Http/Http/test/BindingAddressTests.cs | 4 +- ...iAuthorization.IdentityServer.Tests.csproj | 1 - .../Microsoft.AspNetCore.Identity.Test.csproj | 1 - .../Diagnostics.EFCore.FunctionalTests.csproj | 2 - ...osoft.AspNetCore.NodeServices.Tests.csproj | 1 - .../test/Mvc.Analyzers.Test.csproj | 1 - .../test/Mvc.Api.Analyzers.Test.csproj | 1 - .../test/ProjectTemplates.Tests.csproj | 1 - ...entication.Negotiate.FunctionalTest.csproj | 4 -- ...tCore.Authentication.Negotiate.Test.csproj | 4 -- ...soft.AspNetCore.Authentication.Test.csproj | 4 -- src/Servers/IIS/IIS/test/FunctionalTest.props | 2 - ...oft.AspNetCore.Server.Kestrel.Tests.csproj | 1 - .../InMemory.FunctionalTests.csproj | 3 +- .../Interop.FunctionalTests.csproj | 1 - .../Libuv.BindTests/Libuv.BindTests.csproj | 3 +- .../Libuv.FunctionalTests.csproj | 1 - .../Sockets.BindTests.csproj | 1 - .../Sockets.FunctionalTests.csproj | 1 - src/Shared/test/SkipOnHelixAttribute.cs | 50 ------------------- src/Shared/test/SuccessfulTests.cs | 39 +++++++++++++++ ...e.DeveloperCertificates.XPlat.Tests.csproj | 1 - .../dotnet-microsoft.openapi.Tests.csproj | 1 - .../test/dotnet-watch.Tests.csproj | 1 - 37 files changed, 60 insertions(+), 108 deletions(-) delete mode 100644 src/Shared/test/SkipOnHelixAttribute.cs create mode 100644 src/Shared/test/SuccessfulTests.cs diff --git a/docs/Helix.md b/docs/Helix.md index 660b0b52f0..d1134500f2 100644 --- a/docs/Helix.md +++ b/docs/Helix.md @@ -1,5 +1,4 @@ -Helix testing in ASP.NET Core -============================== +# Helix testing in ASP.NET Core Helix is the distributed test platform that we use to run tests. We build a helix payload that contains the publish directory of every test project that we want to test send a job with with this payload to a set of queues for the various combinations of OS that we want to test @@ -11,16 +10,17 @@ For more info about helix see: [SDK](https://github.com/dotnet/arcade/blob/maste To run Helix tests for one particular test project: -``` +``` powershell .\eng\scripts\RunHelix.ps1 -Project path\mytestproject.csproj ``` This will restore, and then publish all the test project including some bootstrapping scripts that will install the correct dotnet runtime/sdk before running the test assembly on the helix machine(s), and upload the job to helix. - ## How do I look at the results of a helix run on Azure Pipelines? + There's a link embedded in the build.cmd log of the helix target on Azure Pipelines, near the bottom right that will look something like this: -``` + +``` text 2019-02-07T21:55:48.1516089Z Results will be available from https://mc.dot.net/#/user/aspnetcore/pr~2Faspnet~2Faspnetcore/ci/20190207.34 2019-02-07T21:56:43.2209607Z Job 0dedeef6-210e-4815-89f9-fd07513179fe is completed with 108 finished work items. 2019-02-07T21:56:43.5091018Z Job 4c45a464-9464-4321-906c-2503320066b0 is completed with 108 finished work items. @@ -36,11 +36,13 @@ There's a link embedded in the build.cmd log of the helix target on Azure Pipeli The link will take you to an overview of all the tests with clickable links to the logs and each run broken down by queue. -All of the helix runs for aspnetcore can be found here https://mc.dot.net/#/user/aspnetcore/builds +All of the helix runs for aspnetcore can be found here . ## What do I do if a test fails? + You can simulate how most tests run locally: -``` + +``` powershell dotnet publish cd dotnet vstest My.Tests.dll @@ -49,11 +51,12 @@ dotnet vstest My.Tests.dll If that doesn't help, you can try the Get Repro environment link from mission control and try to debug that way. ## Differences from running tests locally + Most tests that don't just work on helix automatically are ones that depend on the source code being accessible. The helix payloads only contain whatever is in the publish directories, so any thing else that test depends on will need to be included to the payload. This can be accomplished by using the `HelixContent` property like so. -``` +``` msbuild @@ -62,7 +65,7 @@ This can be accomplished by using the `HelixContent` property like so. By default, these files will be included in the root directory. To include these files in a different directory, you can use either the `Link` or `LinkBase` attributes to set the included path. -``` +``` msbuild @@ -70,8 +73,10 @@ By default, these files will be included in the root directory. To include these ``` ## How to skip tests on helix + There are two main ways to opt out of helix + - Skipping the entire test project via `false` in csproj (the default value for this is IsTestProject). -- Skipping an individual test via `[SkipOnHelix("url to github issue")]` which might require including a compile reference to: `` +- Skipping an individual test via `[SkipOnHelix("url to github issue")]`. Make sure to file an issue for any skipped tests and include that in a comment next to either of these diff --git a/eng/helix/content/RunTests/TestRunner.cs b/eng/helix/content/RunTests/TestRunner.cs index 3643df3aac..da4b4c228f 100644 --- a/eng/helix/content/RunTests/TestRunner.cs +++ b/eng/helix/content/RunTests/TestRunner.cs @@ -272,7 +272,7 @@ namespace RunTests // Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md var result = await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet", - commonTestArgs + " --TestCaseFilter:\"Quarantined!=true\"", + commonTestArgs + " --TestCaseFilter:\"Quarantined!=true|Quarantined=false\"", environmentVariables: EnvironmentVariables, outputDataReceived: Console.WriteLine, errorDataReceived: Console.Error.WriteLine, diff --git a/eng/targets/CSharp.Common.props b/eng/targets/CSharp.Common.props index 412aa80c54..c63d4ac7c7 100644 --- a/eng/targets/CSharp.Common.props +++ b/eng/targets/CSharp.Common.props @@ -42,6 +42,7 @@ + diff --git a/src/Analyzers/Analyzers/test/Microsoft.AspNetCore.Analyzers.Test.csproj b/src/Analyzers/Analyzers/test/Microsoft.AspNetCore.Analyzers.Test.csproj index 3d791a7367..8f85e4be8c 100644 --- a/src/Analyzers/Analyzers/test/Microsoft.AspNetCore.Analyzers.Test.csproj +++ b/src/Analyzers/Analyzers/test/Microsoft.AspNetCore.Analyzers.Test.csproj @@ -7,7 +7,6 @@ - diff --git a/src/Components/Analyzers/test/Microsoft.AspNetCore.Components.Analyzers.Tests.csproj b/src/Components/Analyzers/test/Microsoft.AspNetCore.Components.Analyzers.Tests.csproj index 4d1ad9ba52..935e05301e 100644 --- a/src/Components/Analyzers/test/Microsoft.AspNetCore.Components.Analyzers.Tests.csproj +++ b/src/Components/Analyzers/test/Microsoft.AspNetCore.Components.Analyzers.Tests.csproj @@ -16,7 +16,6 @@ - diff --git a/src/Components/Authorization/test/Microsoft.AspNetCore.Components.Authorization.Tests.csproj b/src/Components/Authorization/test/Microsoft.AspNetCore.Components.Authorization.Tests.csproj index 5f7ae8f772..ed1092d444 100644 --- a/src/Components/Authorization/test/Microsoft.AspNetCore.Components.Authorization.Tests.csproj +++ b/src/Components/Authorization/test/Microsoft.AspNetCore.Components.Authorization.Tests.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Components/Components/test/Microsoft.AspNetCore.Components.Tests.csproj b/src/Components/Components/test/Microsoft.AspNetCore.Components.Tests.csproj index fd2a5406c1..8f414c52f8 100644 --- a/src/Components/Components/test/Microsoft.AspNetCore.Components.Tests.csproj +++ b/src/Components/Components/test/Microsoft.AspNetCore.Components.Tests.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Components/Forms/test/Microsoft.AspNetCore.Components.Forms.Tests.csproj b/src/Components/Forms/test/Microsoft.AspNetCore.Components.Forms.Tests.csproj index 6c611379a5..c6a022be09 100644 --- a/src/Components/Forms/test/Microsoft.AspNetCore.Components.Forms.Tests.csproj +++ b/src/Components/Forms/test/Microsoft.AspNetCore.Components.Forms.Tests.csproj @@ -11,7 +11,6 @@ - diff --git a/src/Components/Web/test/Microsoft.AspNetCore.Components.Web.Tests.csproj b/src/Components/Web/test/Microsoft.AspNetCore.Components.Web.Tests.csproj index 5c8a5a0d19..b7697ce8ad 100644 --- a/src/Components/Web/test/Microsoft.AspNetCore.Components.Web.Tests.csproj +++ b/src/Components/Web/test/Microsoft.AspNetCore.Components.Web.Tests.csproj @@ -13,7 +13,6 @@ - diff --git a/src/Components/WebAssembly/Build/test/Microsoft.AspNetCore.Components.WebAssembly.Build.Tests.csproj b/src/Components/WebAssembly/Build/test/Microsoft.AspNetCore.Components.WebAssembly.Build.Tests.csproj index 3f622c9f94..fd25c908f9 100644 --- a/src/Components/WebAssembly/Build/test/Microsoft.AspNetCore.Components.WebAssembly.Build.Tests.csproj +++ b/src/Components/WebAssembly/Build/test/Microsoft.AspNetCore.Components.WebAssembly.Build.Tests.csproj @@ -34,7 +34,6 @@ - diff --git a/src/Components/WebAssembly/Build/test/RuntimeDependenciesResolverTest.cs b/src/Components/WebAssembly/Build/test/RuntimeDependenciesResolverTest.cs index 54df8ed79b..a4c802e321 100644 --- a/src/Components/WebAssembly/Build/test/RuntimeDependenciesResolverTest.cs +++ b/src/Components/WebAssembly/Build/test/RuntimeDependenciesResolverTest.cs @@ -7,7 +7,7 @@ using System.IO; using System.Linq; using System.Reflection; using System.Text; -using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.AspNetCore.Testing; using Xunit; namespace Microsoft.AspNetCore.Components.WebAssembly.Build diff --git a/src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj b/src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj index f184844f0c..6d8f3d313a 100644 --- a/src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj +++ b/src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj @@ -7,7 +7,6 @@ - diff --git a/src/Hosting/Hosting/test/Microsoft.AspNetCore.Hosting.Tests.csproj b/src/Hosting/Hosting/test/Microsoft.AspNetCore.Hosting.Tests.csproj index 0f7f87f097..cf25a1547d 100644 --- a/src/Hosting/Hosting/test/Microsoft.AspNetCore.Hosting.Tests.csproj +++ b/src/Hosting/Hosting/test/Microsoft.AspNetCore.Hosting.Tests.csproj @@ -5,7 +5,6 @@ - diff --git a/src/Http/Http/test/BindingAddressTests.cs b/src/Http/Http/test/BindingAddressTests.cs index 858aafeefb..b130bcd2df 100644 --- a/src/Http/Http/test/BindingAddressTests.cs +++ b/src/Http/Http/test/BindingAddressTests.cs @@ -2,11 +2,9 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; -using System.Text; using Microsoft.AspNetCore.Testing; -using Microsoft.AspNetCore.Testing.xunit; using Xunit; + namespace Microsoft.AspNetCore.Http.Tests { public class BindingAddressTests diff --git a/src/Identity/ApiAuthorization.IdentityServer/test/Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Tests.csproj b/src/Identity/ApiAuthorization.IdentityServer/test/Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Tests.csproj index e50086da71..8b291f4979 100644 --- a/src/Identity/ApiAuthorization.IdentityServer/test/Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Tests.csproj +++ b/src/Identity/ApiAuthorization.IdentityServer/test/Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Tests.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Identity/test/Identity.Test/Microsoft.AspNetCore.Identity.Test.csproj b/src/Identity/test/Identity.Test/Microsoft.AspNetCore.Identity.Test.csproj index ee81519dde..33785fb7ce 100644 --- a/src/Identity/test/Identity.Test/Microsoft.AspNetCore.Identity.Test.csproj +++ b/src/Identity/test/Identity.Test/Microsoft.AspNetCore.Identity.Test.csproj @@ -6,7 +6,6 @@ - diff --git a/src/Middleware/Diagnostics.EntityFrameworkCore/test/FunctionalTests/Diagnostics.EFCore.FunctionalTests.csproj b/src/Middleware/Diagnostics.EntityFrameworkCore/test/FunctionalTests/Diagnostics.EFCore.FunctionalTests.csproj index d784bb29f6..d8c8f75989 100644 --- a/src/Middleware/Diagnostics.EntityFrameworkCore/test/FunctionalTests/Diagnostics.EFCore.FunctionalTests.csproj +++ b/src/Middleware/Diagnostics.EntityFrameworkCore/test/FunctionalTests/Diagnostics.EFCore.FunctionalTests.csproj @@ -15,8 +15,6 @@ - - diff --git a/src/Middleware/NodeServices/test/Microsoft.AspNetCore.NodeServices.Tests.csproj b/src/Middleware/NodeServices/test/Microsoft.AspNetCore.NodeServices.Tests.csproj index f8e0099b04..2d1a5aab26 100644 --- a/src/Middleware/NodeServices/test/Microsoft.AspNetCore.NodeServices.Tests.csproj +++ b/src/Middleware/NodeServices/test/Microsoft.AspNetCore.NodeServices.Tests.csproj @@ -8,7 +8,6 @@ - diff --git a/src/Mvc/Mvc.Analyzers/test/Mvc.Analyzers.Test.csproj b/src/Mvc/Mvc.Analyzers/test/Mvc.Analyzers.Test.csproj index bcbc066c95..0d2fecc272 100644 --- a/src/Mvc/Mvc.Analyzers/test/Mvc.Analyzers.Test.csproj +++ b/src/Mvc/Mvc.Analyzers/test/Mvc.Analyzers.Test.csproj @@ -8,7 +8,6 @@ - diff --git a/src/Mvc/Mvc.Api.Analyzers/test/Mvc.Api.Analyzers.Test.csproj b/src/Mvc/Mvc.Api.Analyzers/test/Mvc.Api.Analyzers.Test.csproj index a37838a6bf..3b4f947e80 100644 --- a/src/Mvc/Mvc.Api.Analyzers/test/Mvc.Api.Analyzers.Test.csproj +++ b/src/Mvc/Mvc.Api.Analyzers/test/Mvc.Api.Analyzers.Test.csproj @@ -8,7 +8,6 @@ - diff --git a/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj b/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj index 569804c90a..63fcb43fe7 100644 --- a/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj +++ b/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj @@ -27,7 +27,6 @@ - diff --git a/src/Security/Authentication/Negotiate/test/Negotiate.FunctionalTest/Microsoft.AspNetCore.Authentication.Negotiate.FunctionalTest.csproj b/src/Security/Authentication/Negotiate/test/Negotiate.FunctionalTest/Microsoft.AspNetCore.Authentication.Negotiate.FunctionalTest.csproj index d0c7e5a9a2..085e3ddfd8 100644 --- a/src/Security/Authentication/Negotiate/test/Negotiate.FunctionalTest/Microsoft.AspNetCore.Authentication.Negotiate.FunctionalTest.csproj +++ b/src/Security/Authentication/Negotiate/test/Negotiate.FunctionalTest/Microsoft.AspNetCore.Authentication.Negotiate.FunctionalTest.csproj @@ -5,10 +5,6 @@ true - - - - diff --git a/src/Security/Authentication/Negotiate/test/Negotiate.Test/Microsoft.AspNetCore.Authentication.Negotiate.Test.csproj b/src/Security/Authentication/Negotiate/test/Negotiate.Test/Microsoft.AspNetCore.Authentication.Negotiate.Test.csproj index 3f483fc9e1..40c623bd11 100644 --- a/src/Security/Authentication/Negotiate/test/Negotiate.Test/Microsoft.AspNetCore.Authentication.Negotiate.Test.csproj +++ b/src/Security/Authentication/Negotiate/test/Negotiate.Test/Microsoft.AspNetCore.Authentication.Negotiate.Test.csproj @@ -4,10 +4,6 @@ $(DefaultNetCoreTargetFramework) - - - - diff --git a/src/Security/Authentication/test/Microsoft.AspNetCore.Authentication.Test.csproj b/src/Security/Authentication/test/Microsoft.AspNetCore.Authentication.Test.csproj index 815d30e027..738515b336 100644 --- a/src/Security/Authentication/test/Microsoft.AspNetCore.Authentication.Test.csproj +++ b/src/Security/Authentication/test/Microsoft.AspNetCore.Authentication.Test.csproj @@ -33,10 +33,6 @@ - - - - diff --git a/src/Servers/IIS/IIS/test/FunctionalTest.props b/src/Servers/IIS/IIS/test/FunctionalTest.props index 8582115239..e701d2d021 100644 --- a/src/Servers/IIS/IIS/test/FunctionalTest.props +++ b/src/Servers/IIS/IIS/test/FunctionalTest.props @@ -5,8 +5,6 @@ - - diff --git a/src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj b/src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj index abf2fb6b3a..571f2cda30 100644 --- a/src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj +++ b/src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj @@ -7,7 +7,6 @@ - diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/InMemory.FunctionalTests.csproj b/src/Servers/Kestrel/test/InMemory.FunctionalTests/InMemory.FunctionalTests.csproj index cad32b08bd..952a0a9f50 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/InMemory.FunctionalTests.csproj +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/InMemory.FunctionalTests.csproj @@ -10,7 +10,6 @@ - @@ -27,4 +26,4 @@ - \ No newline at end of file + diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/Interop.FunctionalTests.csproj b/src/Servers/Kestrel/test/Interop.FunctionalTests/Interop.FunctionalTests.csproj index 9fff0431d9..555d7445a5 100644 --- a/src/Servers/Kestrel/test/Interop.FunctionalTests/Interop.FunctionalTests.csproj +++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/Interop.FunctionalTests.csproj @@ -17,7 +17,6 @@ - diff --git a/src/Servers/Kestrel/test/Libuv.BindTests/Libuv.BindTests.csproj b/src/Servers/Kestrel/test/Libuv.BindTests/Libuv.BindTests.csproj index 79752c12b2..29239ef61c 100644 --- a/src/Servers/Kestrel/test/Libuv.BindTests/Libuv.BindTests.csproj +++ b/src/Servers/Kestrel/test/Libuv.BindTests/Libuv.BindTests.csproj @@ -5,7 +5,7 @@ true Libuv.BindTests - Windows.10.Arm64;Windows.10.Arm64.Open + Windows.10.Arm64;Windows.10.Arm64.Open @@ -14,7 +14,6 @@ - diff --git a/src/Servers/Kestrel/test/Libuv.FunctionalTests/Libuv.FunctionalTests.csproj b/src/Servers/Kestrel/test/Libuv.FunctionalTests/Libuv.FunctionalTests.csproj index 65f62bd641..3358fcfdb9 100644 --- a/src/Servers/Kestrel/test/Libuv.FunctionalTests/Libuv.FunctionalTests.csproj +++ b/src/Servers/Kestrel/test/Libuv.FunctionalTests/Libuv.FunctionalTests.csproj @@ -14,7 +14,6 @@ - diff --git a/src/Servers/Kestrel/test/Sockets.BindTests/Sockets.BindTests.csproj b/src/Servers/Kestrel/test/Sockets.BindTests/Sockets.BindTests.csproj index 3ad3eb297a..0f7576c452 100644 --- a/src/Servers/Kestrel/test/Sockets.BindTests/Sockets.BindTests.csproj +++ b/src/Servers/Kestrel/test/Sockets.BindTests/Sockets.BindTests.csproj @@ -7,7 +7,6 @@ - diff --git a/src/Servers/Kestrel/test/Sockets.FunctionalTests/Sockets.FunctionalTests.csproj b/src/Servers/Kestrel/test/Sockets.FunctionalTests/Sockets.FunctionalTests.csproj index 4bc4f8fbdf..325b301e0b 100644 --- a/src/Servers/Kestrel/test/Sockets.FunctionalTests/Sockets.FunctionalTests.csproj +++ b/src/Servers/Kestrel/test/Sockets.FunctionalTests/Sockets.FunctionalTests.csproj @@ -10,7 +10,6 @@ - diff --git a/src/Shared/test/SkipOnHelixAttribute.cs b/src/Shared/test/SkipOnHelixAttribute.cs deleted file mode 100644 index e1a74467bc..0000000000 --- a/src/Shared/test/SkipOnHelixAttribute.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Linq; - -namespace Microsoft.AspNetCore.Testing.xunit -{ - /// - /// Skip test if running on helix (or a particular helix queue). - /// - [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)] - public class SkipOnHelixAttribute : Attribute, ITestCondition - { - public SkipOnHelixAttribute(string issueUrl) - { - if (string.IsNullOrEmpty(issueUrl)) - { - throw new ArgumentException(); - } - IssueUrl = issueUrl; - } - - public string IssueUrl { get; } - - public bool IsMet - { - get - { - var skip = OnHelix() && (Queues == null || Queues.ToLowerInvariant().Split(";").Contains(GetTargetHelixQueue().ToLowerInvariant())); - return !skip; - } - } - - // Queues that should be skipped on, i.e. "Windows.10.Amd64.ClientRS4.VS2017.Open;OSX.1012.Amd64.Open" - public string Queues { get; set; } - - public string SkipReason - { - get - { - return $"This test is skipped on helix"; - } - } - - public static bool OnHelix() => !string.IsNullOrEmpty(GetTargetHelixQueue()); - - public static string GetTargetHelixQueue() => Environment.GetEnvironmentVariable("helix"); - } -} diff --git a/src/Shared/test/SuccessfulTests.cs b/src/Shared/test/SuccessfulTests.cs new file mode 100644 index 0000000000..9843582b17 --- /dev/null +++ b/src/Shared/test/SuccessfulTests.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Testing; +using Xunit; + +namespace AlwaysTestTests +{ + /// + /// Tests for every test assembly to ensure quarantined and unquarantined runs report at least one test execution. + /// + public class SuccessfulTests + { + /// + /// Test that executes in quarantined runs and always succeeds. + /// + [Fact] + [QuarantinedTest] + public void GuaranteedQuarantinedTest() + { + } + + /// + /// Test that executes in unquarantined runs and always succeeds. + /// + /// + /// applied to ensure test runs even if assembly is quarantined overall. + /// dotnet test --filter, dotnet vstest --testcasefilter, and + /// vstest.console.exe --testcasefilter handle the "no Quarantined=true trait OR Quarantined=false" case + /// e.g. "Quarantined!=true|Quarantined=false. But, xunit.console.exe doesn't have a syntax to + /// make it work (yet?). + /// + [Fact] + [Trait("Quarantined", "false")] + public void GuaranteedUnquarantinedTest() + { + } + } +} diff --git a/src/Tools/FirstRunCertGenerator/test/Microsoft.AspNetCore.DeveloperCertificates.XPlat.Tests.csproj b/src/Tools/FirstRunCertGenerator/test/Microsoft.AspNetCore.DeveloperCertificates.XPlat.Tests.csproj index dd470dbf37..c276d7d020 100644 --- a/src/Tools/FirstRunCertGenerator/test/Microsoft.AspNetCore.DeveloperCertificates.XPlat.Tests.csproj +++ b/src/Tools/FirstRunCertGenerator/test/Microsoft.AspNetCore.DeveloperCertificates.XPlat.Tests.csproj @@ -6,7 +6,6 @@ - diff --git a/src/Tools/Microsoft.dotnet-openapi/test/dotnet-microsoft.openapi.Tests.csproj b/src/Tools/Microsoft.dotnet-openapi/test/dotnet-microsoft.openapi.Tests.csproj index 288c651ed3..1d216328e8 100644 --- a/src/Tools/Microsoft.dotnet-openapi/test/dotnet-microsoft.openapi.Tests.csproj +++ b/src/Tools/Microsoft.dotnet-openapi/test/dotnet-microsoft.openapi.Tests.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj b/src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj index 219f55d6f4..71ebe061da 100644 --- a/src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj +++ b/src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj @@ -9,7 +9,6 @@ -