From ddde4faf4f43a2070353fee4d6f3d0c09aaa586d Mon Sep 17 00:00:00 2001 From: Ryan Nowak Date: Mon, 16 Sep 2019 13:33:09 -0700 Subject: [PATCH] Refactor xUnit extensibility Adds our own hook for before/after logic that's more usable, called `ITestMethodLifecycle`. This provides access to a context object including the information about the test and the output helper. This can be implemented by attributes or by the class itself. The goal (and result) of this, is that we have a single *test executor* extensibility point that provides all of the features we need. We should use this everywhere we need features xUnit doesn't have. Adding a new extensibility point (`ITestMethodLifecycle`) allows us to do this without turning all of these features into a giant monolith. --- Also updated our existing extensibility to use this new hook. I did as much cleanup as a could to remove duplication from logging and keep it loosly coupled. I didn't want to tease this apart completely because the scope of this PR is already pretty large. \n\nCommit migrated from https://github.com/dotnet/extensions/commit/1b10284a47a4f58d69f3ee1feb232ecbf77168ee --- src/Testing/src/ITestMethodLifecycle.cs | 23 +++ src/Testing/src/RepeatAttribute.cs | 27 ++++ src/Testing/src/RepeatContext.cs | 27 ++++ src/Testing/src/ShortClassNameAttribute.cs | 17 +++ src/Testing/src/TestContext.cs | 44 ++++++ src/Testing/src/TestFileOutputContext.cs | 140 ++++++++++++++++++ .../src/TestOutputDirectoryAttribute.cs | 20 +++ src/Testing/src/xunit/AspNetTestCaseRunner.cs | 33 +++++ .../src/xunit/AspNetTestClassRunner.cs | 44 ++++++ .../src/xunit/AspNetTestCollectionRunner.cs | 14 +- src/Testing/src/xunit/AspNetTestInvoker.cs | 84 +++++++++++ .../src/xunit/AspNetTestMethodRunner.cs | 73 +++++++++ src/Testing/src/xunit/AspNetTestRunner.cs | 78 ++++++++++ .../src/xunit/AspNetTheoryTestCaseRunner.cs | 33 +++++ src/Testing/test/Properties/AssemblyInfo.cs | 1 + src/Testing/test/RepeatTest.cs | 43 ++++++ src/Testing/test/TestContextTest.cs | 83 +++++++++++ 17 files changed, 781 insertions(+), 3 deletions(-) create mode 100644 src/Testing/src/ITestMethodLifecycle.cs create mode 100644 src/Testing/src/RepeatAttribute.cs create mode 100644 src/Testing/src/RepeatContext.cs create mode 100644 src/Testing/src/ShortClassNameAttribute.cs create mode 100644 src/Testing/src/TestContext.cs create mode 100644 src/Testing/src/TestFileOutputContext.cs create mode 100644 src/Testing/src/TestOutputDirectoryAttribute.cs create mode 100644 src/Testing/src/xunit/AspNetTestCaseRunner.cs create mode 100644 src/Testing/src/xunit/AspNetTestClassRunner.cs create mode 100644 src/Testing/src/xunit/AspNetTestInvoker.cs create mode 100644 src/Testing/src/xunit/AspNetTestMethodRunner.cs create mode 100644 src/Testing/src/xunit/AspNetTestRunner.cs create mode 100644 src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs create mode 100644 src/Testing/test/RepeatTest.cs create mode 100644 src/Testing/test/TestContextTest.cs diff --git a/src/Testing/src/ITestMethodLifecycle.cs b/src/Testing/src/ITestMethodLifecycle.cs new file mode 100644 index 0000000000..d22779b6dd --- /dev/null +++ b/src/Testing/src/ITestMethodLifecycle.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Defines a lifecycle for attributes or classes that want to know about tests starting + /// or ending. Implement this on a test class, or attribute at the method/class/assembly level. + /// + /// + /// Requires defining as the test framework. + /// + public interface ITestMethodLifecycle + { + Task OnTestStartAsync(TestContext context, CancellationToken cancellationToken); + + Task OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken); + } +} diff --git a/src/Testing/src/RepeatAttribute.cs b/src/Testing/src/RepeatAttribute.cs new file mode 100644 index 0000000000..7bf3073734 --- /dev/null +++ b/src/Testing/src/RepeatAttribute.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.ComponentModel; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Runs a test multiple times to stress flaky tests that are believed to be fixed. + /// This can be used on an assembly, class, or method name. Requires using the AspNetCore test framework. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)] + public class RepeatAttribute : Attribute + { + public RepeatAttribute(int runCount = 10) + { + RunCount = runCount; + } + + /// + /// The number of times to run a test. + /// + public int RunCount { get; } + } +} diff --git a/src/Testing/src/RepeatContext.cs b/src/Testing/src/RepeatContext.cs new file mode 100644 index 0000000000..d76a0f177e --- /dev/null +++ b/src/Testing/src/RepeatContext.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; + +namespace Microsoft.AspNetCore.Testing +{ + public class RepeatContext + { + private static AsyncLocal _current = new AsyncLocal(); + + public static RepeatContext Current + { + get => _current.Value; + internal set => _current.Value = value; + } + + public RepeatContext(int limit) + { + Limit = limit; + } + + public int Limit { get; } + + public int CurrentIteration { get; set; } + } +} diff --git a/src/Testing/src/ShortClassNameAttribute.cs b/src/Testing/src/ShortClassNameAttribute.cs new file mode 100644 index 0000000000..6a36575d70 --- /dev/null +++ b/src/Testing/src/ShortClassNameAttribute.cs @@ -0,0 +1,17 @@ +// Copyright(c) .NET Foundation.All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Used to specify that should used the + /// unqualified class name. This is needed when a fully-qualified class name exceeds + /// max path for logging. + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)] + public class ShortClassNameAttribute : Attribute + { + } +} diff --git a/src/Testing/src/TestContext.cs b/src/Testing/src/TestContext.cs new file mode 100644 index 0000000000..a702d71ecf --- /dev/null +++ b/src/Testing/src/TestContext.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using Xunit.Abstractions; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Provides access to contextual information about the running tests. Get access by + /// implementing . + /// + /// + /// Requires defining as the test framework. + /// + public sealed class TestContext + { + private Lazy _files; + + public TestContext( + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] methodArguments, + ITestOutputHelper output) + { + TestClass = testClass; + ConstructorArguments = constructorArguments; + TestMethod = testMethod; + MethodArguments = methodArguments; + Output = output; + + _files = new Lazy(() => new TestFileOutputContext(this)); + } + + public Type TestClass { get; } + public MethodInfo TestMethod { get; } + public object[] ConstructorArguments { get; } + public object[] MethodArguments { get; } + public ITestOutputHelper Output { get; } + public TestFileOutputContext FileOutput => _files.Value; + } +} diff --git a/src/Testing/src/TestFileOutputContext.cs b/src/Testing/src/TestFileOutputContext.cs new file mode 100644 index 0000000000..496a1379fb --- /dev/null +++ b/src/Testing/src/TestFileOutputContext.cs @@ -0,0 +1,140 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Provides access to file storage for the running test. Get access by + /// implementing , and accessing . + /// + /// + /// Requires defining as the test framework. + /// + public sealed class TestFileOutputContext + { + private static char[] InvalidFileChars = new char[] + { + '\"', '<', '>', '|', '\0', + (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10, + (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20, + (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30, + (char)31, ':', '*', '?', '\\', '/', ' ', (char)127 + }; + + private readonly TestContext _parent; + + public TestFileOutputContext(TestContext parent) + { + _parent = parent; + + TestName = GetTestMethodName(parent.TestMethod, parent.MethodArguments); + TestClassName = GetTestClassName(parent.TestClass); + + AssemblyOutputDirectory = GetAssemblyBaseDirectory(_parent.TestClass.Assembly); + if (!string.IsNullOrEmpty(AssemblyOutputDirectory)) + { + TestClassOutputDirectory = Path.Combine(AssemblyOutputDirectory, TestClassName); + } + } + + public string TestName { get; } + + public string TestClassName { get; } + + public string AssemblyOutputDirectory { get; } + + public string TestClassOutputDirectory { get; } + + public string GetUniqueFileName(string prefix, string extension) + { + if (prefix == null) + { + throw new ArgumentNullException(nameof(prefix)); + } + + if (extension != null && !extension.StartsWith(".", StringComparison.Ordinal)) + { + throw new ArgumentException("The extension must start with '.' if one is provided.", nameof(extension)); + } + + var path = Path.Combine(TestClassOutputDirectory, $"{prefix}{extension}"); + + var i = 1; + while (File.Exists(path)) + { + path = Path.Combine(TestClassOutputDirectory, $"{prefix}{i++}{extension}"); + } + + return path; + } + + // Gets the output directory without appending the TFM or assembly name. + public static string GetOutputDirectory(Assembly assembly) + { + var attribute = assembly.GetCustomAttributes().OfType().FirstOrDefault(); + return attribute?.BaseDirectory; + } + + public static string GetAssemblyBaseDirectory(Assembly assembly, string baseDirectory = null) + { + var attribute = assembly.GetCustomAttributes().OfType().FirstOrDefault(); + baseDirectory = baseDirectory ?? attribute?.BaseDirectory; + if (string.IsNullOrEmpty(baseDirectory)) + { + return string.Empty; + } + + return Path.Combine(baseDirectory, assembly.GetName().Name, attribute.TargetFramework); + } + + public static string GetTestClassName(Type type) + { + var shortNameAttribute = + type.GetCustomAttribute() ?? + type.Assembly.GetCustomAttribute(); + var name = shortNameAttribute == null ? type.FullName : type.Name; + + // Try to shorten the class name using the assembly name + var assemblyName = type.Assembly.GetName().Name; + if (name.StartsWith(assemblyName + ".")) + { + name = name.Substring(assemblyName.Length + 1); + } + + return name; + } + + public static string GetTestMethodName(MethodInfo method, object[] arguments) + { + var name = arguments.Aggregate(method.Name, (a, b) => $"{a}-{(b ?? "null")}"); + return RemoveIllegalFileChars(name); + } + + public static string RemoveIllegalFileChars(string s) + { + var sb = new StringBuilder(); + + foreach (var c in s) + { + if (InvalidFileChars.Contains(c)) + { + if (sb.Length > 0 && sb[sb.Length - 1] != '_') + { + sb.Append('_'); + } + } + else + { + sb.Append(c); + } + } + return sb.ToString(); + } + } +} diff --git a/src/Testing/src/TestOutputDirectoryAttribute.cs b/src/Testing/src/TestOutputDirectoryAttribute.cs new file mode 100644 index 0000000000..4ae8cea054 --- /dev/null +++ b/src/Testing/src/TestOutputDirectoryAttribute.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Testing +{ + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false, Inherited = true)] + public class TestOutputDirectoryAttribute : Attribute + { + public TestOutputDirectoryAttribute(string targetFramework, string baseDirectory = null) + { + TargetFramework = targetFramework; + BaseDirectory = baseDirectory; + } + + public string BaseDirectory { get; } + public string TargetFramework { get; } + } +} diff --git a/src/Testing/src/xunit/AspNetTestCaseRunner.cs b/src/Testing/src/xunit/AspNetTestCaseRunner.cs new file mode 100644 index 0000000000..42773db212 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestCaseRunner.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestCaseRunner : XunitTestCaseRunner + { + public AspNetTestCaseRunner( + IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + object[] testMethodArguments, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus, aggregator, cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + { + return new AspNetTestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestClassRunner.cs b/src/Testing/src/xunit/AspNetTestClassRunner.cs new file mode 100644 index 0000000000..bbefa37427 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestClassRunner.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestClassRunner : XunitTestClassRunner + { + public AspNetTestClassRunner( + ITestClass testClass, + IReflectionTypeInfo @class, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ITestCaseOrderer testCaseOrderer, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource, + IDictionary collectionFixtureMappings) + : base(testClass, @class, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource, collectionFixtureMappings) + { + } + + protected override Task RunTestMethodAsync(ITestMethod testMethod, IReflectionMethodInfo method, IEnumerable testCases, object[] constructorArguments) + { + var runner = new AspNetTestMethodRunner( + testMethod, + Class, + method, + testCases, + DiagnosticMessageSink, + MessageBus, + new ExceptionAggregator(Aggregator), + CancellationTokenSource, + constructorArguments); + return runner.RunAsync(); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestCollectionRunner.cs b/src/Testing/src/xunit/AspNetTestCollectionRunner.cs index 264cf769c6..522cbd4624 100644 --- a/src/Testing/src/xunit/AspNetTestCollectionRunner.cs +++ b/src/Testing/src/xunit/AspNetTestCollectionRunner.cs @@ -59,9 +59,17 @@ namespace Microsoft.AspNetCore.Testing protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable testCases) { - var caste = testCases.ToArray(); - var type = caste.First().GetType(); - return base.RunTestClassAsync(testClass, @class, testCases); + var runner = new AspNetTestClassRunner( + testClass, + @class, + testCases, + DiagnosticMessageSink, + MessageBus, + TestCaseOrderer, + new ExceptionAggregator(Aggregator), + CancellationTokenSource, + CollectionFixtureMappings); + return runner.RunAsync(); } } } diff --git a/src/Testing/src/xunit/AspNetTestInvoker.cs b/src/Testing/src/xunit/AspNetTestInvoker.cs new file mode 100644 index 0000000000..a764db6622 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestInvoker.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestInvoker : XunitTestInvoker + { + public AspNetTestInvoker( + ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, beforeAfterAttributes, aggregator, cancellationTokenSource) + { + } + + protected override async Task InvokeTestMethodAsync(object testClassInstance) + { + var output = new TestOutputHelper(); + output.Initialize(MessageBus, Test); + + var context = new TestContext(TestClass, ConstructorArguments, TestMethod, TestMethodArguments, output); + var lifecycleHooks = GetLifecycleHooks(testClassInstance, TestClass, TestMethod); + + await Aggregator.RunAsync(async () => + { + foreach (var lifecycleHook in lifecycleHooks) + { + await lifecycleHook.OnTestStartAsync(context, CancellationTokenSource.Token); + } + }); + + var time = await base.InvokeTestMethodAsync(testClassInstance); + + await Aggregator.RunAsync(async () => + { + var exception = Aggregator.HasExceptions ? Aggregator.ToException() : null; + foreach (var lifecycleHook in lifecycleHooks) + { + await lifecycleHook.OnTestEndAsync(context, exception, CancellationTokenSource.Token); + } + }); + + return time; + } + + private static IEnumerable GetLifecycleHooks(object testClassInstance, Type testClass, MethodInfo testMethod) + { + foreach (var attribute in testMethod.GetCustomAttributes(inherit: true).OfType()) + { + yield return attribute; + } + + if (testClassInstance is ITestMethodLifecycle instance) + { + yield return instance; + } + + foreach (var attribute in testClass.GetCustomAttributes(inherit: true).OfType()) + { + yield return attribute; + } + + foreach (var attribute in testClass.Assembly.GetCustomAttributes(inherit: true).OfType()) + { + yield return attribute; + } + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestMethodRunner.cs b/src/Testing/src/xunit/AspNetTestMethodRunner.cs new file mode 100644 index 0000000000..e238d0769d --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestMethodRunner.cs @@ -0,0 +1,73 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing.xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestMethodRunner : XunitTestMethodRunner + { + private readonly object[] _constructorArguments; + private readonly IMessageSink _diagnosticMessageSink; + + public AspNetTestMethodRunner( + ITestMethod testMethod, + IReflectionTypeInfo @class, + IReflectionMethodInfo method, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource, + object[] constructorArguments) + : base(testMethod, @class, method, testCases, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, constructorArguments) + { + _diagnosticMessageSink = diagnosticMessageSink; + _constructorArguments = constructorArguments; + } + + protected override Task RunTestCaseAsync(IXunitTestCase testCase) + { + if (testCase.GetType() == typeof(XunitTestCase)) + { + // If we get here this is a 'regular' test case, not something that represents a skipped test. + // + // We can take control of it's invocation thusly. + var runner = new AspNetTestCaseRunner( + testCase, + testCase.DisplayName, + testCase.SkipReason, + _constructorArguments, + testCase.TestMethodArguments, + MessageBus, + new ExceptionAggregator(Aggregator), + CancellationTokenSource); + return runner.RunAsync(); + } + + if (testCase.GetType() == typeof(XunitTheoryTestCase)) + { + // If we get here this is a 'regular' theory test case, not something that represents a skipped test. + // + // We can take control of it's invocation thusly. + var runner = new AspNetTheoryTestCaseRunner( + testCase, + testCase.DisplayName, + testCase.SkipReason, + _constructorArguments, + _diagnosticMessageSink, + MessageBus, + new ExceptionAggregator(Aggregator), + CancellationTokenSource); + return runner.RunAsync(); + } + + return base.RunTestCaseAsync(testCase); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestRunner.cs b/src/Testing/src/xunit/AspNetTestRunner.cs new file mode 100644 index 0000000000..2786a866b4 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestRunner.cs @@ -0,0 +1,78 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestRunner : XunitTestRunner + { + public AspNetTestRunner( + ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + string skipReason, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) + { + } + + protected override async Task InvokeTestMethodAsync(ExceptionAggregator aggregator) + { + var repeatAttribute = GetRepeatAttribute(TestMethod); + if (repeatAttribute == null) + { + return await InvokeTestMethodCoreAsync(aggregator); + } + + var repeatContext = new RepeatContext(repeatAttribute.RunCount); + RepeatContext.Current = repeatContext; + + var timeTaken = 0.0M; + for (repeatContext.CurrentIteration = 0; repeatContext.CurrentIteration < repeatContext.Limit; repeatContext.CurrentIteration++) + { + timeTaken = await InvokeTestMethodCoreAsync(aggregator); + if (aggregator.HasExceptions) + { + return timeTaken; + } + } + + return timeTaken; + } + + private Task InvokeTestMethodCoreAsync(ExceptionAggregator aggregator) + { + var invoker = new AspNetTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource); + return invoker.RunAsync(); + } + + private RepeatAttribute GetRepeatAttribute(MethodInfo methodInfo) + { + var attributeCandidate = methodInfo.GetCustomAttribute(); + if (attributeCandidate != null) + { + return attributeCandidate; + } + + attributeCandidate = methodInfo.DeclaringType.GetCustomAttribute(); + if (attributeCandidate != null) + { + return attributeCandidate; + } + + return methodInfo.DeclaringType.Assembly.GetCustomAttribute(); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs b/src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs new file mode 100644 index 0000000000..a09a17cf69 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing.xunit +{ + internal class AspNetTheoryTestCaseRunner : XunitTheoryTestCaseRunner + { + public AspNetTheoryTestCaseRunner( + IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCase, displayName, skipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + { + return new AspNetTestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource); + } + } +} diff --git a/src/Testing/test/Properties/AssemblyInfo.cs b/src/Testing/test/Properties/AssemblyInfo.cs index 4dba8d157e..d585b5ed95 100644 --- a/src/Testing/test/Properties/AssemblyInfo.cs +++ b/src/Testing/test/Properties/AssemblyInfo.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Testing; using Xunit; +[assembly: Repeat(1)] [assembly: AssemblyFixture(typeof(TestAssemblyFixture))] [assembly: TestFramework("Microsoft.AspNetCore.Testing.AspNetTestFramework", "Microsoft.AspNetCore.Testing")] diff --git a/src/Testing/test/RepeatTest.cs b/src/Testing/test/RepeatTest.cs new file mode 100644 index 0000000000..0d995fad59 --- /dev/null +++ b/src/Testing/test/RepeatTest.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace Microsoft.AspNetCore.Testing +{ + [Repeat] + public class RepeatTest + { + public static int _runCount = 0; + + [Fact] + [Repeat(5)] + public void RepeatLimitIsSetCorrectly() + { + Assert.Equal(5, RepeatContext.Current.Limit); + } + + [Fact] + [Repeat(5)] + public void RepeatRunsTestSpecifiedNumberOfTimes() + { + Assert.Equal(RepeatContext.Current.CurrentIteration, _runCount); + _runCount++; + } + + [Fact] + public void RepeatCanBeSetOnClass() + { + Assert.Equal(10, RepeatContext.Current.Limit); + } + } + + public class LoggedTestXunitRepeatAssemblyTests + { + [Fact] + public void RepeatCanBeSetOnAssembly() + { + Assert.Equal(1, RepeatContext.Current.Limit); + } + } +} diff --git a/src/Testing/test/TestContextTest.cs b/src/Testing/test/TestContextTest.cs new file mode 100644 index 0000000000..944d706477 --- /dev/null +++ b/src/Testing/test/TestContextTest.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.AspNetCore.Testing +{ + public class TestContextTest : ITestMethodLifecycle + { + public TestContext Context { get; private set; } + + [Fact] + public void FullName_IsUsed_ByDefault() + { + Assert.Equal(GetType().FullName, Context.FileOutput.TestClassName); + } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + Context = context; + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} + +namespace Microsoft.AspNetCore.Testing.Tests +{ + public class TestContextNameShorteningTest : ITestMethodLifecycle + { + public TestContext Context { get; private set; } + + [Fact] + public void NameIsShortenedWhenAssemblyNameIsAPrefix() + { + Assert.Equal(GetType().Name, Context.FileOutput.TestClassName); + } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + Context = context; + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} + +namespace Microsoft.AspNetCore.Testing +{ + [ShortClassName] + public class TestContextTestClassShortNameAttributeTest : ITestMethodLifecycle + { + public TestContext Context { get; private set; } + + [Fact] + public void ShortClassNameUsedWhenShortClassNameAttributeSpecified() + { + Assert.Equal(GetType().Name, Context.FileOutput.TestClassName); + } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + Context = context; + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +}