From c717230b1350ba5ad408de9155f2e784dcf3dbac Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 15 Aug 2019 09:12:53 -0700 Subject: [PATCH] Cleanup to skip/flaky attributes (dotnet/extensions#2186) \n\nCommit migrated from https://github.com/dotnet/extensions/commit/cfef5e07fb893b1c5a94566a0d053290f0c75382 --- src/Testing/src/TestPlatformHelper.cs | 2 +- .../src/xunit/ConditionalFactAttribute.cs | 2 +- .../src/xunit/ConditionalFactDiscoverer.cs | 2 +- .../src/xunit/ConditionalTheoryAttribute.cs | 2 +- .../src/xunit/ConditionalTheoryDiscoverer.cs | 2 +- src/Testing/src/xunit/DockerOnlyAttribute.cs | 2 +- ...vironmentVariableSkipConditionAttribute.cs | 2 +- src/Testing/src/xunit/FlakyAttribute.cs | 2 +- src/Testing/src/xunit/FlakyTestDiscoverer.cs | 2 +- .../xunit/FrameworkSkipConditionAttribute.cs | 2 +- src/Testing/src/xunit/IEnvironmentVariable.cs | 2 +- src/Testing/src/xunit/ITestCondition.cs | 2 +- .../src/xunit/MinimumOsVersionAttribute.cs | 2 +- .../src/xunit/OSSkipConditionAttribute.cs | 2 +- src/Testing/src/xunit/OperatingSystems.cs | 2 +- src/Testing/src/xunit/RuntimeFrameworks.cs | 2 +- src/Testing/src/xunit/SkipOnCIAttribute.cs | 43 ++++++++++++++++ src/Testing/src/xunit/SkipOnHelixAttribute.cs | 50 +++++++++++++++++++ src/Testing/src/xunit/SkippedTestCase.cs | 2 +- src/Testing/src/xunit/TestMethodExtensions.cs | 2 +- src/Testing/src/xunit/WindowsVersions.cs | 2 +- src/Testing/test/ConditionalFactTest.cs | 2 +- src/Testing/test/ConditionalTheoryTest.cs | 2 +- src/Testing/test/DockerTests.cs | 2 +- .../EnvironmentVariableSkipConditionTest.cs | 2 +- src/Testing/test/FlakyAttributeTest.cs | 4 +- .../test/OSSkipConditionAttributeTest.cs | 2 +- src/Testing/test/OSSkipConditionTest.cs | 2 +- src/Testing/test/SkipOnCITests.cs | 22 ++++++++ src/Testing/test/TestPlatformHelperTest.cs | 2 +- 30 files changed, 144 insertions(+), 27 deletions(-) create mode 100644 src/Testing/src/xunit/SkipOnCIAttribute.cs create mode 100644 src/Testing/src/xunit/SkipOnHelixAttribute.cs create mode 100644 src/Testing/test/SkipOnCITests.cs diff --git a/src/Testing/src/TestPlatformHelper.cs b/src/Testing/src/TestPlatformHelper.cs index 1a3f275c7e..2c13e08eb3 100644 --- a/src/Testing/src/TestPlatformHelper.cs +++ b/src/Testing/src/TestPlatformHelper.cs @@ -20,4 +20,4 @@ namespace Microsoft.AspNetCore.Testing public static bool IsMac => RuntimeInformation.IsOSPlatform(OSPlatform.OSX); } -} \ No newline at end of file +} diff --git a/src/Testing/src/xunit/ConditionalFactAttribute.cs b/src/Testing/src/xunit/ConditionalFactAttribute.cs index ce37df2e56..fdc108190a 100644 --- a/src/Testing/src/xunit/ConditionalFactAttribute.cs +++ b/src/Testing/src/xunit/ConditionalFactAttribute.cs @@ -5,7 +5,7 @@ using System; using Xunit; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] [XunitTestCaseDiscoverer("Microsoft.AspNetCore.Testing.xunit." + nameof(ConditionalFactDiscoverer), "Microsoft.AspNetCore.Testing")] diff --git a/src/Testing/src/xunit/ConditionalFactDiscoverer.cs b/src/Testing/src/xunit/ConditionalFactDiscoverer.cs index cf49b29e5a..ce190376fc 100644 --- a/src/Testing/src/xunit/ConditionalFactDiscoverer.cs +++ b/src/Testing/src/xunit/ConditionalFactDiscoverer.cs @@ -4,7 +4,7 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { internal class ConditionalFactDiscoverer : FactDiscoverer { diff --git a/src/Testing/src/xunit/ConditionalTheoryAttribute.cs b/src/Testing/src/xunit/ConditionalTheoryAttribute.cs index fe45f2ffc6..58b460e96e 100644 --- a/src/Testing/src/xunit/ConditionalTheoryAttribute.cs +++ b/src/Testing/src/xunit/ConditionalTheoryAttribute.cs @@ -5,7 +5,7 @@ using System; using Xunit; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] [XunitTestCaseDiscoverer("Microsoft.AspNetCore.Testing.xunit." + nameof(ConditionalTheoryDiscoverer), "Microsoft.AspNetCore.Testing")] diff --git a/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs b/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs index 9e413cd580..c9ee58889a 100644 --- a/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs +++ b/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs @@ -5,7 +5,7 @@ using System.Collections.Generic; using Xunit.Abstractions; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { internal class ConditionalTheoryDiscoverer : TheoryDiscoverer { diff --git a/src/Testing/src/xunit/DockerOnlyAttribute.cs b/src/Testing/src/xunit/DockerOnlyAttribute.cs index d67a35a672..7d809884d6 100644 --- a/src/Testing/src/xunit/DockerOnlyAttribute.cs +++ b/src/Testing/src/xunit/DockerOnlyAttribute.cs @@ -6,7 +6,7 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [AttributeUsage(AttributeTargets.Method, Inherited = true, AllowMultiple = false)] public sealed class DockerOnlyAttribute : Attribute, ITestCondition diff --git a/src/Testing/src/xunit/EnvironmentVariableSkipConditionAttribute.cs b/src/Testing/src/xunit/EnvironmentVariableSkipConditionAttribute.cs index fe215a8e0b..0599e31901 100644 --- a/src/Testing/src/xunit/EnvironmentVariableSkipConditionAttribute.cs +++ b/src/Testing/src/xunit/EnvironmentVariableSkipConditionAttribute.cs @@ -4,7 +4,7 @@ using System; using System.Linq; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { /// /// Skips a test when the value of an environment variable matches any of the supplied values. diff --git a/src/Testing/src/xunit/FlakyAttribute.cs b/src/Testing/src/xunit/FlakyAttribute.cs index ab4450e685..acea96f3cb 100644 --- a/src/Testing/src/xunit/FlakyAttribute.cs +++ b/src/Testing/src/xunit/FlakyAttribute.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { /// /// Marks a test as "Flaky" so that the build will sequester it and ignore failures. diff --git a/src/Testing/src/xunit/FlakyTestDiscoverer.cs b/src/Testing/src/xunit/FlakyTestDiscoverer.cs index 344b9b2378..aea2f9ea5b 100644 --- a/src/Testing/src/xunit/FlakyTestDiscoverer.cs +++ b/src/Testing/src/xunit/FlakyTestDiscoverer.cs @@ -4,7 +4,7 @@ using System.Linq; using Xunit.Abstractions; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public class FlakyTestDiscoverer : ITraitDiscoverer { diff --git a/src/Testing/src/xunit/FrameworkSkipConditionAttribute.cs b/src/Testing/src/xunit/FrameworkSkipConditionAttribute.cs index 168076a434..b7719848a6 100644 --- a/src/Testing/src/xunit/FrameworkSkipConditionAttribute.cs +++ b/src/Testing/src/xunit/FrameworkSkipConditionAttribute.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public class FrameworkSkipConditionAttribute : Attribute, ITestCondition diff --git a/src/Testing/src/xunit/IEnvironmentVariable.cs b/src/Testing/src/xunit/IEnvironmentVariable.cs index 068c210611..ed06ed6505 100644 --- a/src/Testing/src/xunit/IEnvironmentVariable.cs +++ b/src/Testing/src/xunit/IEnvironmentVariable.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { internal interface IEnvironmentVariable { diff --git a/src/Testing/src/xunit/ITestCondition.cs b/src/Testing/src/xunit/ITestCondition.cs index bb6ff1f031..34767b8574 100644 --- a/src/Testing/src/xunit/ITestCondition.cs +++ b/src/Testing/src/xunit/ITestCondition.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public interface ITestCondition { diff --git a/src/Testing/src/xunit/MinimumOsVersionAttribute.cs b/src/Testing/src/xunit/MinimumOsVersionAttribute.cs index 89e3b19556..df4985d338 100644 --- a/src/Testing/src/xunit/MinimumOsVersionAttribute.cs +++ b/src/Testing/src/xunit/MinimumOsVersionAttribute.cs @@ -5,7 +5,7 @@ using System; using System.Runtime.InteropServices; using Microsoft.Win32; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { /// /// Skips a test if the OS is the given type (Windows) and the OS version is less than specified. diff --git a/src/Testing/src/xunit/OSSkipConditionAttribute.cs b/src/Testing/src/xunit/OSSkipConditionAttribute.cs index 9996510718..7655a3b45a 100644 --- a/src/Testing/src/xunit/OSSkipConditionAttribute.cs +++ b/src/Testing/src/xunit/OSSkipConditionAttribute.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] public class OSSkipConditionAttribute : Attribute, ITestCondition diff --git a/src/Testing/src/xunit/OperatingSystems.cs b/src/Testing/src/xunit/OperatingSystems.cs index c575d3e197..2ddacacab9 100644 --- a/src/Testing/src/xunit/OperatingSystems.cs +++ b/src/Testing/src/xunit/OperatingSystems.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [Flags] public enum OperatingSystems diff --git a/src/Testing/src/xunit/RuntimeFrameworks.cs b/src/Testing/src/xunit/RuntimeFrameworks.cs index 2ec5ea7ec1..3a69022b88 100644 --- a/src/Testing/src/xunit/RuntimeFrameworks.cs +++ b/src/Testing/src/xunit/RuntimeFrameworks.cs @@ -3,7 +3,7 @@ using System; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { [Flags] public enum RuntimeFrameworks diff --git a/src/Testing/src/xunit/SkipOnCIAttribute.cs b/src/Testing/src/xunit/SkipOnCIAttribute.cs new file mode 100644 index 0000000000..1ee0b8cde8 --- /dev/null +++ b/src/Testing/src/xunit/SkipOnCIAttribute.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Skip test if running on CI + /// + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)] + public class SkipOnCIAttribute : Attribute, ITestCondition + { + public SkipOnCIAttribute(string issueUrl = "") + { + IssueUrl = issueUrl; + } + + public string IssueUrl { get; } + + public bool IsMet + { + get + { + return !OnCI(); + } + } + + public string SkipReason + { + get + { + return $"This test is skipped on CI"; + } + } + + public static bool OnCI() => OnHelix() || OnAzdo(); + public static bool OnHelix() => !string.IsNullOrEmpty(GetTargetHelixQueue()); + public static string GetTargetHelixQueue() => Environment.GetEnvironmentVariable("helix"); + public static bool OnAzdo() => !string.IsNullOrEmpty(GetIfOnAzdo()); + public static string GetIfOnAzdo() => Environment.GetEnvironmentVariable("AGENT_OS"); + } +} diff --git a/src/Testing/src/xunit/SkipOnHelixAttribute.cs b/src/Testing/src/xunit/SkipOnHelixAttribute.cs new file mode 100644 index 0000000000..85e82c1154 --- /dev/null +++ b/src/Testing/src/xunit/SkipOnHelixAttribute.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Skip test if running on helix (or a particular helix queue). + /// + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)] + public class SkipOnHelixAttribute : Attribute, ITestCondition + { + public SkipOnHelixAttribute(string issueUrl) + { + if (string.IsNullOrEmpty(issueUrl)) + { + throw new ArgumentException(); + } + IssueUrl = issueUrl; + } + + public string IssueUrl { get; } + + public bool IsMet + { + get + { + var skip = OnHelix() && (Queues == null || Queues.ToLowerInvariant().Split(';').Contains(GetTargetHelixQueue().ToLowerInvariant())); + return !skip; + } + } + + // Queues that should be skipped on, i.e. "Windows.10.Amd64.ClientRS4.VS2017.Open;OSX.1012.Amd64.Open" + public string Queues { get; set; } + + public string SkipReason + { + get + { + return $"This test is skipped on helix"; + } + } + + public static bool OnHelix() => !string.IsNullOrEmpty(GetTargetHelixQueue()); + + public static string GetTargetHelixQueue() => Environment.GetEnvironmentVariable("helix"); + } +} diff --git a/src/Testing/src/xunit/SkippedTestCase.cs b/src/Testing/src/xunit/SkippedTestCase.cs index 1c25c507b9..b514c57209 100644 --- a/src/Testing/src/xunit/SkippedTestCase.cs +++ b/src/Testing/src/xunit/SkippedTestCase.cs @@ -5,7 +5,7 @@ using System; using Xunit.Abstractions; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public class SkippedTestCase : XunitTestCase { diff --git a/src/Testing/src/xunit/TestMethodExtensions.cs b/src/Testing/src/xunit/TestMethodExtensions.cs index 5ec3bb4ec3..96dd93eb7c 100644 --- a/src/Testing/src/xunit/TestMethodExtensions.cs +++ b/src/Testing/src/xunit/TestMethodExtensions.cs @@ -5,7 +5,7 @@ using System.Linq; using Xunit.Abstractions; using Xunit.Sdk; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public static class TestMethodExtensions { diff --git a/src/Testing/src/xunit/WindowsVersions.cs b/src/Testing/src/xunit/WindowsVersions.cs index ff8312b363..d89da44de3 100644 --- a/src/Testing/src/xunit/WindowsVersions.cs +++ b/src/Testing/src/xunit/WindowsVersions.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public static class WindowsVersions { diff --git a/src/Testing/test/ConditionalFactTest.cs b/src/Testing/test/ConditionalFactTest.cs index 9c5c6d037d..efc3a16dea 100644 --- a/src/Testing/test/ConditionalFactTest.cs +++ b/src/Testing/test/ConditionalFactTest.cs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.AspNetCore.Testing; using Xunit; namespace Microsoft.AspNetCore.Testing diff --git a/src/Testing/test/ConditionalTheoryTest.cs b/src/Testing/test/ConditionalTheoryTest.cs index d824eb61b4..07cf6a968f 100644 --- a/src/Testing/test/ConditionalTheoryTest.cs +++ b/src/Testing/test/ConditionalTheoryTest.cs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.AspNetCore.Testing; using Xunit; using Xunit.Abstractions; diff --git a/src/Testing/test/DockerTests.cs b/src/Testing/test/DockerTests.cs index c66fdd679c..12735057d3 100644 --- a/src/Testing/test/DockerTests.cs +++ b/src/Testing/test/DockerTests.cs @@ -3,7 +3,7 @@ using System; using System.Runtime.InteropServices; -using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.AspNetCore.Testing; using Xunit; namespace Microsoft.AspNetCore.Testing diff --git a/src/Testing/test/EnvironmentVariableSkipConditionTest.cs b/src/Testing/test/EnvironmentVariableSkipConditionTest.cs index d5e7b6342b..cbc8e9adad 100644 --- a/src/Testing/test/EnvironmentVariableSkipConditionTest.cs +++ b/src/Testing/test/EnvironmentVariableSkipConditionTest.cs @@ -3,7 +3,7 @@ using Xunit; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public class EnvironmentVariableSkipConditionTest { diff --git a/src/Testing/test/FlakyAttributeTest.cs b/src/Testing/test/FlakyAttributeTest.cs index 1b9a122d93..ae06e5cf50 100644 --- a/src/Testing/test/FlakyAttributeTest.cs +++ b/src/Testing/test/FlakyAttributeTest.cs @@ -1,4 +1,6 @@ -using Microsoft.AspNetCore.Testing.xunit; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; using System.Collections.Generic; using Xunit; diff --git a/src/Testing/test/OSSkipConditionAttributeTest.cs b/src/Testing/test/OSSkipConditionAttributeTest.cs index 0120eb7a4c..199af3ab6e 100644 --- a/src/Testing/test/OSSkipConditionAttributeTest.cs +++ b/src/Testing/test/OSSkipConditionAttributeTest.cs @@ -5,7 +5,7 @@ using System; using System.Runtime.InteropServices; using Xunit; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public class OSSkipConditionAttributeTest { diff --git a/src/Testing/test/OSSkipConditionTest.cs b/src/Testing/test/OSSkipConditionTest.cs index 2d76f2c2cd..a7904b1730 100644 --- a/src/Testing/test/OSSkipConditionTest.cs +++ b/src/Testing/test/OSSkipConditionTest.cs @@ -5,7 +5,7 @@ using System; using System.Runtime.InteropServices; using Xunit; -namespace Microsoft.AspNetCore.Testing.xunit +namespace Microsoft.AspNetCore.Testing { public class OSSkipConditionTest { diff --git a/src/Testing/test/SkipOnCITests.cs b/src/Testing/test/SkipOnCITests.cs new file mode 100644 index 0000000000..8df5e73c30 --- /dev/null +++ b/src/Testing/test/SkipOnCITests.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.Testing; +using Xunit; + +namespace Microsoft.AspNetCore.Testing.Tests +{ + public class SkipOnCITests + { + [ConditionalFact] + [SkipOnCI] + public void AlwaysSkipOnCI() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) + { + throw new Exception("Flaky!"); + } + } + } +} diff --git a/src/Testing/test/TestPlatformHelperTest.cs b/src/Testing/test/TestPlatformHelperTest.cs index 8e35e164d5..b1c2fbf2f8 100644 --- a/src/Testing/test/TestPlatformHelperTest.cs +++ b/src/Testing/test/TestPlatformHelperTest.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.AspNetCore.Testing; using Xunit; namespace Microsoft.AspNetCore.Testing