huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add FlakyAttribute to mark flaky tests (dotnet/extensions#1222) 

part of aspnet/AspNetCoredotnet/extensions#8237\n\nCommit migrated from 42e9a7d712
This commit is contained in:
Andrew Stanton-Nurse 2019-03-06 15:19:11 -08:00 committed by GitHub
parent b13ea4cd54
commit d9627c80ef
5 changed files with 253 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/Testing/src/AzurePipelines.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Reflection;
namespace Microsoft.AspNetCore.Testing
{
public static class AzurePipelines
{
public const string All = Prefix + "All";
public const string Windows = OsPrefix + "Windows_NT";
public const string macOS = OsPrefix + "Darwin";
public const string Linux = OsPrefix + "Linux";
private const string Prefix = "AzP:";
private const string OsPrefix = Prefix + "OS:";
}
}

26
src/Testing/src/HelixQueues.cs Normal file
View File

@ -0,0 +1,26 @@
using System;
using System.Collections.Generic;
using System.Reflection;
namespace Microsoft.AspNetCore.Testing
{
public static class HelixQueues
{
public const string All = Prefix + "All";
public const string Fedora28Amd64 = QueuePrefix + "Fedora.28." + Amd64Suffix;
public const string Fedora27Amd64 = QueuePrefix + "Fedora.27." + Amd64Suffix;
public const string Redhat7Amd64 = QueuePrefix + "Redhat.7." + Amd64Suffix;
public const string Debian9Amd64 = QueuePrefix + "Debian.9." + Amd64Suffix;
public const string Debian8Amd64 = QueuePrefix + "Debian.8." + Amd64Suffix;
public const string Centos7Amd64 = QueuePrefix + "Centos.7." + Amd64Suffix;
public const string Ubuntu1604Amd64 = QueuePrefix + "Ubuntu.1604." + Amd64Suffix;
public const string Ubuntu1810Amd64 = QueuePrefix + "Ubuntu.1810." + Amd64Suffix;
public const string macOS1012Amd64 = QueuePrefix + "OSX.1012." + Amd64Suffix;
public const string Windows10Amd64 = QueuePrefix + "Windows.10.Amd64.ClientRS4.VS2017.Open"; // Doesn't have the default suffix!
private const string Prefix = "Helix:";
private const string QueuePrefix = Prefix + "Queue:";
private const string Amd64Suffix = "Amd64.Open";
}
}

75
src/Testing/src/xunit/FlakyAttribute.cs Normal file
View File

@ -0,0 +1,75 @@
using System;
using System.Collections.Generic;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Testing.xunit
{
/// <summary>
/// Marks a test as "Flaky" so that the build will sequester it and ignore failures.
/// </summary>
/// <remarks>
/// <para>
/// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute
/// properties. Once these traits are applied, build scripts can include/exclude tests based on them.
/// </para>
/// <para>
/// All flakiness-related traits start with <code>Flaky:</code> and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix.
/// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector.
/// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only
/// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks
/// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude
/// flaky tests.
/// </para>
/// </remarks>
/// <example>
/// <code>
/// [Fact]
/// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)]
/// public void FlakyTest()
/// {
/// // Flakiness
/// }
/// </code>
///
/// <para>
/// The above example generates the following facets:
/// </para>
///
/// <list type="bullet">
/// <item>
/// <description><c>Flaky:Helix:Queue:Fedora.28.Amd64.Open</c> = <c>true</c></description>
/// </item>
/// <item>
/// <description><c>Flaky:AzP:OS:Darwin</c> = <c>true</c></description>
/// </item>
/// </list>
///
/// <para>
/// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing <c>-notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true"</c>
/// to <c>xunit.console.exe</c>. Similarly, it can run only flaky tests using <c>-trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true"</c>
/// </para>
/// </example>
[TraitDiscoverer("Microsoft.AspNetCore.Testing.xunit.FlakyTestDiscoverer", "Microsoft.AspNetCore.Testing")]
[AttributeUsage(AttributeTargets.Method)]
public sealed class FlakyAttribute : Attribute, ITraitAttribute
{
/// <summary>
/// Gets a URL to a GitHub issue tracking this flaky test.
/// </summary>
public string GitHubIssueUrl { get; }
public IReadOnlyList<string> Filters { get; }
/// <summary>
/// Initializes a new instance of the <see cref="FlakyAttribute"/> class with the specified <see cref="GitHubIssueUrl"/> and a list of <see cref="Filters"/>. If no
/// filters are provided, the test is considered flaky in all environments.
/// </summary>
/// <param name="gitHubIssueUrl">The URL to a GitHub issue tracking this flaky test.</param>
/// <param name="filters">A list of filters that define where this test is flaky. Use values in <see cref="AzurePipelines"/> and <see cref="HelixQueues"/>.</param>
public FlakyAttribute(string gitHubIssueUrl, params string[] filters)
{
GitHubIssueUrl = gitHubIssueUrl;
Filters = new List<string>(filters);
}
}
}

38
src/Testing/src/xunit/FlakyTestDiscoverer.cs Normal file
View File

@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Xunit.Abstractions;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Testing.xunit
{
public class FlakyTestDiscoverer : ITraitDiscoverer
{
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
{
if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute)
{
return GetTraitsCore(flakyAttribute);
}
else
{
throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection.");
}
}
private IEnumerable<KeyValuePair<string, string>> GetTraitsCore(FlakyAttribute attribute)
{
if (attribute.Filters.Count > 0)
{
foreach (var filter in attribute.Filters)
{
yield return new KeyValuePair<string, string>($"Flaky:{filter}", "true");
}
}
else
{
yield return new KeyValuePair<string, string>($"Flaky:All", "true");
}
}
}
}

97
src/Testing/test/FlakyAttributeTest.cs Normal file
View File

@ -0,0 +1,97 @@
using Microsoft.AspNetCore.Testing.xunit;
using System;
using System.Collections.Generic;
using Xunit;
namespace Microsoft.AspNetCore.Testing.Tests
{
public class FlakyAttributeTest
{
[Fact]
[Flaky("http://example.com")]
public void AlwaysFlaky()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
{
throw new Exception("Flaky!");
}
}
[Fact]
[Flaky("http://example.com", HelixQueues.All)]
public void FlakyInHelixOnly()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")))
{
throw new Exception("Flaky on Helix!");
}
}
[Fact]
[Flaky("http://example.com", HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64)]
public void FlakyInSpecificHelixQueue()
{
// Today we don't run Extensions tests on Helix, but this test should light up when we do.
var queueName = Environment.GetEnvironmentVariable("HELIX");
if (!string.IsNullOrEmpty(queueName))
{
var failingQueues = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 };
if (failingQueues.Contains(queueName))
{
throw new Exception($"Flaky on Helix Queue '{queueName}' !");
}
}
}
[Fact]
[Flaky("http://example.com", AzurePipelines.All)]
public void FlakyInAzPOnly()
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
{
throw new Exception("Flaky on AzP!");
}
}
[Fact]
[Flaky("http://example.com", AzurePipelines.Windows)]
public void FlakyInAzPWindowsOnly()
{
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Windows))
{
throw new Exception("Flaky on AzP Windows!");
}
}
[Fact]
[Flaky("http://example.com", AzurePipelines.macOS)]
public void FlakyInAzPmacOSOnly()
{
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.macOS))
{
throw new Exception("Flaky on AzP macOS!");
}
}
[Fact]
[Flaky("http://example.com", AzurePipelines.Linux)]
public void FlakyInAzPLinuxOnly()
{
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Linux))
{
throw new Exception("Flaky on AzP Linux!");
}
}
[Fact]
[Flaky("http://example.com", AzurePipelines.Linux, AzurePipelines.macOS)]
public void FlakyInAzPNonWindowsOnly()
{
var agentOs = Environment.GetEnvironmentVariable("AGENT_OS");
if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, AzurePipelines.macOS))
{
throw new Exception("Flaky on AzP non-Windows!");
}
}
}
}