From d9627c80efda3434cf4608e1af70bbff142fb50e Mon Sep 17 00:00:00 2001 From: Andrew Stanton-Nurse Date: Wed, 6 Mar 2019 15:19:11 -0800 Subject: [PATCH] add FlakyAttribute to mark flaky tests (dotnet/extensions#1222) part of aspnet/AspNetCoredotnet/extensions#8237\n\nCommit migrated from https://github.com/dotnet/extensions/commit/42e9a7d712d1b513c32961dca7ad6d0bd33d0fae --- src/Testing/src/AzurePipelines.cs | 17 ++++ src/Testing/src/HelixQueues.cs | 26 ++++++ src/Testing/src/xunit/FlakyAttribute.cs | 75 +++++++++++++++ src/Testing/src/xunit/FlakyTestDiscoverer.cs | 38 ++++++++ src/Testing/test/FlakyAttributeTest.cs | 97 ++++++++++++++++++++ 5 files changed, 253 insertions(+) create mode 100644 src/Testing/src/AzurePipelines.cs create mode 100644 src/Testing/src/HelixQueues.cs create mode 100644 src/Testing/src/xunit/FlakyAttribute.cs create mode 100644 src/Testing/src/xunit/FlakyTestDiscoverer.cs create mode 100644 src/Testing/test/FlakyAttributeTest.cs diff --git a/src/Testing/src/AzurePipelines.cs b/src/Testing/src/AzurePipelines.cs new file mode 100644 index 0000000000..ae1eac3b90 --- /dev/null +++ b/src/Testing/src/AzurePipelines.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Microsoft.AspNetCore.Testing +{ + public static class AzurePipelines + { + public const string All = Prefix + "All"; + public const string Windows = OsPrefix + "Windows_NT"; + public const string macOS = OsPrefix + "Darwin"; + public const string Linux = OsPrefix + "Linux"; + + private const string Prefix = "AzP:"; + private const string OsPrefix = Prefix + "OS:"; + } +} diff --git a/src/Testing/src/HelixQueues.cs b/src/Testing/src/HelixQueues.cs new file mode 100644 index 0000000000..84828b6b83 --- /dev/null +++ b/src/Testing/src/HelixQueues.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Microsoft.AspNetCore.Testing +{ + public static class HelixQueues + { + public const string All = Prefix + "All"; + + public const string Fedora28Amd64 = QueuePrefix + "Fedora.28." + Amd64Suffix; + public const string Fedora27Amd64 = QueuePrefix + "Fedora.27." + Amd64Suffix; + public const string Redhat7Amd64 = QueuePrefix + "Redhat.7." + Amd64Suffix; + public const string Debian9Amd64 = QueuePrefix + "Debian.9." + Amd64Suffix; + public const string Debian8Amd64 = QueuePrefix + "Debian.8." + Amd64Suffix; + public const string Centos7Amd64 = QueuePrefix + "Centos.7." + Amd64Suffix; + public const string Ubuntu1604Amd64 = QueuePrefix + "Ubuntu.1604." + Amd64Suffix; + public const string Ubuntu1810Amd64 = QueuePrefix + "Ubuntu.1810." + Amd64Suffix; + public const string macOS1012Amd64 = QueuePrefix + "OSX.1012." + Amd64Suffix; + public const string Windows10Amd64 = QueuePrefix + "Windows.10.Amd64.ClientRS4.VS2017.Open"; // Doesn't have the default suffix! + + private const string Prefix = "Helix:"; + private const string QueuePrefix = Prefix + "Queue:"; + private const string Amd64Suffix = "Amd64.Open"; + } +} diff --git a/src/Testing/src/xunit/FlakyAttribute.cs b/src/Testing/src/xunit/FlakyAttribute.cs new file mode 100644 index 0000000000..b613a9bf4d --- /dev/null +++ b/src/Testing/src/xunit/FlakyAttribute.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing.xunit +{ + /// + /// Marks a test as "Flaky" so that the build will sequester it and ignore failures. + /// + /// + /// + /// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute + /// properties. Once these traits are applied, build scripts can include/exclude tests based on them. + /// + /// + /// All flakiness-related traits start with Flaky: and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix. + /// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector. + /// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only + /// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks + /// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude + /// flaky tests. + /// + /// + /// + /// + /// [Fact] + /// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)] + /// public void FlakyTest() + /// { + /// // Flakiness + /// } + /// + /// + /// + /// The above example generates the following facets: + /// + /// + /// + /// + /// Flaky:Helix:Queue:Fedora.28.Amd64.Open = true + /// + /// + /// Flaky:AzP:OS:Darwin = true + /// + /// + /// + /// + /// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing -notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true" + /// to xunit.console.exe. Similarly, it can run only flaky tests using -trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true" + /// + /// + [TraitDiscoverer("Microsoft.AspNetCore.Testing.xunit.FlakyTestDiscoverer", "Microsoft.AspNetCore.Testing")] + [AttributeUsage(AttributeTargets.Method)] + public sealed class FlakyAttribute : Attribute, ITraitAttribute + { + /// + /// Gets a URL to a GitHub issue tracking this flaky test. + /// + public string GitHubIssueUrl { get; } + + public IReadOnlyList Filters { get; } + + /// + /// Initializes a new instance of the class with the specified and a list of . If no + /// filters are provided, the test is considered flaky in all environments. + /// + /// The URL to a GitHub issue tracking this flaky test. + /// A list of filters that define where this test is flaky. Use values in and . + public FlakyAttribute(string gitHubIssueUrl, params string[] filters) + { + GitHubIssueUrl = gitHubIssueUrl; + Filters = new List(filters); + } + } +} diff --git a/src/Testing/src/xunit/FlakyTestDiscoverer.cs b/src/Testing/src/xunit/FlakyTestDiscoverer.cs new file mode 100644 index 0000000000..344b9b2378 --- /dev/null +++ b/src/Testing/src/xunit/FlakyTestDiscoverer.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing.xunit +{ + public class FlakyTestDiscoverer : ITraitDiscoverer + { + public IEnumerable> GetTraits(IAttributeInfo traitAttribute) + { + if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute) + { + return GetTraitsCore(flakyAttribute); + } + else + { + throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection."); + } + } + + private IEnumerable> GetTraitsCore(FlakyAttribute attribute) + { + if (attribute.Filters.Count > 0) + { + foreach (var filter in attribute.Filters) + { + yield return new KeyValuePair($"Flaky:{filter}", "true"); + } + } + else + { + yield return new KeyValuePair($"Flaky:All", "true"); + } + } + } +} diff --git a/src/Testing/test/FlakyAttributeTest.cs b/src/Testing/test/FlakyAttributeTest.cs new file mode 100644 index 0000000000..e9accf6274 --- /dev/null +++ b/src/Testing/test/FlakyAttributeTest.cs @@ -0,0 +1,97 @@ +using Microsoft.AspNetCore.Testing.xunit; +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.AspNetCore.Testing.Tests +{ + public class FlakyAttributeTest + { + [Fact] + [Flaky("http://example.com")] + public void AlwaysFlaky() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) + { + throw new Exception("Flaky!"); + } + } + + [Fact] + [Flaky("http://example.com", HelixQueues.All)] + public void FlakyInHelixOnly() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX"))) + { + throw new Exception("Flaky on Helix!"); + } + } + + [Fact] + [Flaky("http://example.com", HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64)] + public void FlakyInSpecificHelixQueue() + { + // Today we don't run Extensions tests on Helix, but this test should light up when we do. + var queueName = Environment.GetEnvironmentVariable("HELIX"); + if (!string.IsNullOrEmpty(queueName)) + { + var failingQueues = new HashSet(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 }; + if (failingQueues.Contains(queueName)) + { + throw new Exception($"Flaky on Helix Queue '{queueName}' !"); + } + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.All)] + public void FlakyInAzPOnly() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) + { + throw new Exception("Flaky on AzP!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.Windows)] + public void FlakyInAzPWindowsOnly() + { + if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Windows)) + { + throw new Exception("Flaky on AzP Windows!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.macOS)] + public void FlakyInAzPmacOSOnly() + { + if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.macOS)) + { + throw new Exception("Flaky on AzP macOS!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.Linux)] + public void FlakyInAzPLinuxOnly() + { + if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Linux)) + { + throw new Exception("Flaky on AzP Linux!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.Linux, AzurePipelines.macOS)] + public void FlakyInAzPNonWindowsOnly() + { + var agentOs = Environment.GetEnvironmentVariable("AGENT_OS"); + if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, AzurePipelines.macOS)) + { + throw new Exception("Flaky on AzP non-Windows!"); + } + } + } +}