diff --git a/src/JSInterop/Microsoft.JSInterop/src/DotNetObjectReference.cs b/src/JSInterop/Microsoft.JSInterop/src/DotNetObjectReference.cs index 989d8062bb..53217936d6 100644 --- a/src/JSInterop/Microsoft.JSInterop/src/DotNetObjectReference.cs +++ b/src/JSInterop/Microsoft.JSInterop/src/DotNetObjectReference.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; + namespace Microsoft.JSInterop { /// @@ -15,6 +17,11 @@ namespace Microsoft.JSInterop /// An instance of . public static DotNetObjectReference Create(TValue value) where TValue : class { + if (value is null) + { + throw new ArgumentNullException(nameof(value)); + } + return new DotNetObjectReference(value); } } diff --git a/src/Testing/src/ITestMethodLifecycle.cs b/src/Testing/src/ITestMethodLifecycle.cs new file mode 100644 index 0000000000..d22779b6dd --- /dev/null +++ b/src/Testing/src/ITestMethodLifecycle.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Defines a lifecycle for attributes or classes that want to know about tests starting + /// or ending. Implement this on a test class, or attribute at the method/class/assembly level. + /// + /// + /// Requires defining as the test framework. + /// + public interface ITestMethodLifecycle + { + Task OnTestStartAsync(TestContext context, CancellationToken cancellationToken); + + Task OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken); + } +} diff --git a/src/Testing/src/RepeatAttribute.cs b/src/Testing/src/RepeatAttribute.cs new file mode 100644 index 0000000000..7bf3073734 --- /dev/null +++ b/src/Testing/src/RepeatAttribute.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.ComponentModel; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Runs a test multiple times to stress flaky tests that are believed to be fixed. + /// This can be used on an assembly, class, or method name. Requires using the AspNetCore test framework. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)] + public class RepeatAttribute : Attribute + { + public RepeatAttribute(int runCount = 10) + { + RunCount = runCount; + } + + /// + /// The number of times to run a test. + /// + public int RunCount { get; } + } +} diff --git a/src/Testing/src/RepeatContext.cs b/src/Testing/src/RepeatContext.cs new file mode 100644 index 0000000000..d76a0f177e --- /dev/null +++ b/src/Testing/src/RepeatContext.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; + +namespace Microsoft.AspNetCore.Testing +{ + public class RepeatContext + { + private static AsyncLocal _current = new AsyncLocal(); + + public static RepeatContext Current + { + get => _current.Value; + internal set => _current.Value = value; + } + + public RepeatContext(int limit) + { + Limit = limit; + } + + public int Limit { get; } + + public int CurrentIteration { get; set; } + } +} diff --git a/src/Testing/src/ShortClassNameAttribute.cs b/src/Testing/src/ShortClassNameAttribute.cs new file mode 100644 index 0000000000..6a36575d70 --- /dev/null +++ b/src/Testing/src/ShortClassNameAttribute.cs @@ -0,0 +1,17 @@ +// Copyright(c) .NET Foundation.All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Used to specify that should used the + /// unqualified class name. This is needed when a fully-qualified class name exceeds + /// max path for logging. + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)] + public class ShortClassNameAttribute : Attribute + { + } +} diff --git a/src/Testing/src/TestContext.cs b/src/Testing/src/TestContext.cs new file mode 100644 index 0000000000..a702d71ecf --- /dev/null +++ b/src/Testing/src/TestContext.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using Xunit.Abstractions; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Provides access to contextual information about the running tests. Get access by + /// implementing . + /// + /// + /// Requires defining as the test framework. + /// + public sealed class TestContext + { + private Lazy _files; + + public TestContext( + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] methodArguments, + ITestOutputHelper output) + { + TestClass = testClass; + ConstructorArguments = constructorArguments; + TestMethod = testMethod; + MethodArguments = methodArguments; + Output = output; + + _files = new Lazy(() => new TestFileOutputContext(this)); + } + + public Type TestClass { get; } + public MethodInfo TestMethod { get; } + public object[] ConstructorArguments { get; } + public object[] MethodArguments { get; } + public ITestOutputHelper Output { get; } + public TestFileOutputContext FileOutput => _files.Value; + } +} diff --git a/src/Testing/src/TestFileOutputContext.cs b/src/Testing/src/TestFileOutputContext.cs new file mode 100644 index 0000000000..496a1379fb --- /dev/null +++ b/src/Testing/src/TestFileOutputContext.cs @@ -0,0 +1,140 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +namespace Microsoft.AspNetCore.Testing +{ + /// + /// Provides access to file storage for the running test. Get access by + /// implementing , and accessing . + /// + /// + /// Requires defining as the test framework. + /// + public sealed class TestFileOutputContext + { + private static char[] InvalidFileChars = new char[] + { + '\"', '<', '>', '|', '\0', + (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10, + (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20, + (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30, + (char)31, ':', '*', '?', '\\', '/', ' ', (char)127 + }; + + private readonly TestContext _parent; + + public TestFileOutputContext(TestContext parent) + { + _parent = parent; + + TestName = GetTestMethodName(parent.TestMethod, parent.MethodArguments); + TestClassName = GetTestClassName(parent.TestClass); + + AssemblyOutputDirectory = GetAssemblyBaseDirectory(_parent.TestClass.Assembly); + if (!string.IsNullOrEmpty(AssemblyOutputDirectory)) + { + TestClassOutputDirectory = Path.Combine(AssemblyOutputDirectory, TestClassName); + } + } + + public string TestName { get; } + + public string TestClassName { get; } + + public string AssemblyOutputDirectory { get; } + + public string TestClassOutputDirectory { get; } + + public string GetUniqueFileName(string prefix, string extension) + { + if (prefix == null) + { + throw new ArgumentNullException(nameof(prefix)); + } + + if (extension != null && !extension.StartsWith(".", StringComparison.Ordinal)) + { + throw new ArgumentException("The extension must start with '.' if one is provided.", nameof(extension)); + } + + var path = Path.Combine(TestClassOutputDirectory, $"{prefix}{extension}"); + + var i = 1; + while (File.Exists(path)) + { + path = Path.Combine(TestClassOutputDirectory, $"{prefix}{i++}{extension}"); + } + + return path; + } + + // Gets the output directory without appending the TFM or assembly name. + public static string GetOutputDirectory(Assembly assembly) + { + var attribute = assembly.GetCustomAttributes().OfType().FirstOrDefault(); + return attribute?.BaseDirectory; + } + + public static string GetAssemblyBaseDirectory(Assembly assembly, string baseDirectory = null) + { + var attribute = assembly.GetCustomAttributes().OfType().FirstOrDefault(); + baseDirectory = baseDirectory ?? attribute?.BaseDirectory; + if (string.IsNullOrEmpty(baseDirectory)) + { + return string.Empty; + } + + return Path.Combine(baseDirectory, assembly.GetName().Name, attribute.TargetFramework); + } + + public static string GetTestClassName(Type type) + { + var shortNameAttribute = + type.GetCustomAttribute() ?? + type.Assembly.GetCustomAttribute(); + var name = shortNameAttribute == null ? type.FullName : type.Name; + + // Try to shorten the class name using the assembly name + var assemblyName = type.Assembly.GetName().Name; + if (name.StartsWith(assemblyName + ".")) + { + name = name.Substring(assemblyName.Length + 1); + } + + return name; + } + + public static string GetTestMethodName(MethodInfo method, object[] arguments) + { + var name = arguments.Aggregate(method.Name, (a, b) => $"{a}-{(b ?? "null")}"); + return RemoveIllegalFileChars(name); + } + + public static string RemoveIllegalFileChars(string s) + { + var sb = new StringBuilder(); + + foreach (var c in s) + { + if (InvalidFileChars.Contains(c)) + { + if (sb.Length > 0 && sb[sb.Length - 1] != '_') + { + sb.Append('_'); + } + } + else + { + sb.Append(c); + } + } + return sb.ToString(); + } + } +} diff --git a/src/Testing/src/TestOutputDirectoryAttribute.cs b/src/Testing/src/TestOutputDirectoryAttribute.cs new file mode 100644 index 0000000000..4ae8cea054 --- /dev/null +++ b/src/Testing/src/TestOutputDirectoryAttribute.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Testing +{ + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false, Inherited = true)] + public class TestOutputDirectoryAttribute : Attribute + { + public TestOutputDirectoryAttribute(string targetFramework, string baseDirectory = null) + { + TargetFramework = targetFramework; + BaseDirectory = baseDirectory; + } + + public string BaseDirectory { get; } + public string TargetFramework { get; } + } +} diff --git a/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs b/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs new file mode 100644 index 0000000000..48fbbbfa3b --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + public class AspNetTestAssemblyRunner : XunitTestAssemblyRunner + { + private readonly Dictionary _assemblyFixtureMappings = new Dictionary(); + + public AspNetTestAssemblyRunner( + ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + { + } + + protected override async Task AfterTestAssemblyStartingAsync() + { + await base.AfterTestAssemblyStartingAsync(); + + // Find all the AssemblyFixtureAttributes on the test assembly + Aggregator.Run(() => + { + var fixturesAttributes = ((IReflectionAssemblyInfo)TestAssembly.Assembly) + .Assembly + .GetCustomAttributes(typeof(AssemblyFixtureAttribute), false) + .Cast() + .ToList(); + + // Instantiate all the fixtures + foreach (var fixtureAttribute in fixturesAttributes) + { + var ctorWithDiagnostics = fixtureAttribute.FixtureType.GetConstructor(new[] { typeof(IMessageSink) }); + if (ctorWithDiagnostics != null) + { + _assemblyFixtureMappings[fixtureAttribute.FixtureType] = Activator.CreateInstance(fixtureAttribute.FixtureType, DiagnosticMessageSink); + } + else + { + _assemblyFixtureMappings[fixtureAttribute.FixtureType] = Activator.CreateInstance(fixtureAttribute.FixtureType); + } + } + }); + } + + protected override Task BeforeTestAssemblyFinishedAsync() + { + // Dispose fixtures + foreach (var disposable in _assemblyFixtureMappings.Values.OfType()) + { + Aggregator.Run(disposable.Dispose); + } + + return base.BeforeTestAssemblyFinishedAsync(); + } + + protected override Task RunTestCollectionAsync( + IMessageBus messageBus, + ITestCollection testCollection, + IEnumerable testCases, + CancellationTokenSource cancellationTokenSource) + => new AspNetTestCollectionRunner( + _assemblyFixtureMappings, + testCollection, + testCases, + DiagnosticMessageSink, + messageBus, + TestCaseOrderer, + new ExceptionAggregator(Aggregator), + cancellationTokenSource).RunAsync(); + } +} diff --git a/src/Testing/src/xunit/AspNetTestCaseRunner.cs b/src/Testing/src/xunit/AspNetTestCaseRunner.cs new file mode 100644 index 0000000000..42773db212 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestCaseRunner.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestCaseRunner : XunitTestCaseRunner + { + public AspNetTestCaseRunner( + IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + object[] testMethodArguments, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus, aggregator, cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + { + return new AspNetTestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestClassRunner.cs b/src/Testing/src/xunit/AspNetTestClassRunner.cs new file mode 100644 index 0000000000..bbefa37427 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestClassRunner.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestClassRunner : XunitTestClassRunner + { + public AspNetTestClassRunner( + ITestClass testClass, + IReflectionTypeInfo @class, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ITestCaseOrderer testCaseOrderer, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource, + IDictionary collectionFixtureMappings) + : base(testClass, @class, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource, collectionFixtureMappings) + { + } + + protected override Task RunTestMethodAsync(ITestMethod testMethod, IReflectionMethodInfo method, IEnumerable testCases, object[] constructorArguments) + { + var runner = new AspNetTestMethodRunner( + testMethod, + Class, + method, + testCases, + DiagnosticMessageSink, + MessageBus, + new ExceptionAggregator(Aggregator), + CancellationTokenSource, + constructorArguments); + return runner.RunAsync(); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestCollectionRunner.cs b/src/Testing/src/xunit/AspNetTestCollectionRunner.cs new file mode 100644 index 0000000000..522cbd4624 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestCollectionRunner.cs @@ -0,0 +1,75 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + public class AspNetTestCollectionRunner : XunitTestCollectionRunner + { + private readonly IDictionary _assemblyFixtureMappings; + private readonly IMessageSink _diagnosticMessageSink; + + public AspNetTestCollectionRunner( + Dictionary assemblyFixtureMappings, + ITestCollection testCollection, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ITestCaseOrderer testCaseOrderer, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource) + { + _assemblyFixtureMappings = assemblyFixtureMappings; + _diagnosticMessageSink = diagnosticMessageSink; + } + + protected override async Task AfterTestCollectionStartingAsync() + { + await base.AfterTestCollectionStartingAsync(); + + // note: We pass the assembly fixtures into the runner as ICollectionFixture<> - this seems to work OK without any + // drawbacks. It's reasonable that we could add IAssemblyFixture<> and related plumbing if it ever became required. + // + // The reason for assembly fixture is when we want to start/stop something as the project scope - tests can only be + // in one test collection at a time. + foreach (var mapping in _assemblyFixtureMappings) + { + CollectionFixtureMappings.Add(mapping.Key, mapping.Value); + } + } + + protected override Task BeforeTestCollectionFinishedAsync() + { + // We need to remove the assembly fixtures so they won't get disposed. + foreach (var mapping in _assemblyFixtureMappings) + { + CollectionFixtureMappings.Remove(mapping.Key); + } + + return base.BeforeTestCollectionFinishedAsync(); + } + + protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable testCases) + { + var runner = new AspNetTestClassRunner( + testClass, + @class, + testCases, + DiagnosticMessageSink, + MessageBus, + TestCaseOrderer, + new ExceptionAggregator(Aggregator), + CancellationTokenSource, + CollectionFixtureMappings); + return runner.RunAsync(); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestFramework.cs b/src/Testing/src/xunit/AspNetTestFramework.cs new file mode 100644 index 0000000000..0a2dc1b21f --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestFramework.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + public class AspNetTestFramework : XunitTestFramework + { + public AspNetTestFramework(IMessageSink messageSink) + : base(messageSink) + { + } + + protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + => new AspNetTestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink); + } +} diff --git a/src/Testing/src/xunit/AspNetTestFrameworkExecutor.cs b/src/Testing/src/xunit/AspNetTestFrameworkExecutor.cs new file mode 100644 index 0000000000..b34f0b715e --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestFrameworkExecutor.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + public class AspNetTestFrameworkExecutor : XunitTestFrameworkExecutor + { + public AspNetTestFrameworkExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) + : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) + { + } + + protected override async void RunTestCases(IEnumerable testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + { + using (var assemblyRunner = new AspNetTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) + { + await assemblyRunner.RunAsync(); + } + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestInvoker.cs b/src/Testing/src/xunit/AspNetTestInvoker.cs new file mode 100644 index 0000000000..a764db6622 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestInvoker.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestInvoker : XunitTestInvoker + { + public AspNetTestInvoker( + ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, beforeAfterAttributes, aggregator, cancellationTokenSource) + { + } + + protected override async Task InvokeTestMethodAsync(object testClassInstance) + { + var output = new TestOutputHelper(); + output.Initialize(MessageBus, Test); + + var context = new TestContext(TestClass, ConstructorArguments, TestMethod, TestMethodArguments, output); + var lifecycleHooks = GetLifecycleHooks(testClassInstance, TestClass, TestMethod); + + await Aggregator.RunAsync(async () => + { + foreach (var lifecycleHook in lifecycleHooks) + { + await lifecycleHook.OnTestStartAsync(context, CancellationTokenSource.Token); + } + }); + + var time = await base.InvokeTestMethodAsync(testClassInstance); + + await Aggregator.RunAsync(async () => + { + var exception = Aggregator.HasExceptions ? Aggregator.ToException() : null; + foreach (var lifecycleHook in lifecycleHooks) + { + await lifecycleHook.OnTestEndAsync(context, exception, CancellationTokenSource.Token); + } + }); + + return time; + } + + private static IEnumerable GetLifecycleHooks(object testClassInstance, Type testClass, MethodInfo testMethod) + { + foreach (var attribute in testMethod.GetCustomAttributes(inherit: true).OfType()) + { + yield return attribute; + } + + if (testClassInstance is ITestMethodLifecycle instance) + { + yield return instance; + } + + foreach (var attribute in testClass.GetCustomAttributes(inherit: true).OfType()) + { + yield return attribute; + } + + foreach (var attribute in testClass.Assembly.GetCustomAttributes(inherit: true).OfType()) + { + yield return attribute; + } + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestMethodRunner.cs b/src/Testing/src/xunit/AspNetTestMethodRunner.cs new file mode 100644 index 0000000000..e238d0769d --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestMethodRunner.cs @@ -0,0 +1,73 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing.xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestMethodRunner : XunitTestMethodRunner + { + private readonly object[] _constructorArguments; + private readonly IMessageSink _diagnosticMessageSink; + + public AspNetTestMethodRunner( + ITestMethod testMethod, + IReflectionTypeInfo @class, + IReflectionMethodInfo method, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource, + object[] constructorArguments) + : base(testMethod, @class, method, testCases, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, constructorArguments) + { + _diagnosticMessageSink = diagnosticMessageSink; + _constructorArguments = constructorArguments; + } + + protected override Task RunTestCaseAsync(IXunitTestCase testCase) + { + if (testCase.GetType() == typeof(XunitTestCase)) + { + // If we get here this is a 'regular' test case, not something that represents a skipped test. + // + // We can take control of it's invocation thusly. + var runner = new AspNetTestCaseRunner( + testCase, + testCase.DisplayName, + testCase.SkipReason, + _constructorArguments, + testCase.TestMethodArguments, + MessageBus, + new ExceptionAggregator(Aggregator), + CancellationTokenSource); + return runner.RunAsync(); + } + + if (testCase.GetType() == typeof(XunitTheoryTestCase)) + { + // If we get here this is a 'regular' theory test case, not something that represents a skipped test. + // + // We can take control of it's invocation thusly. + var runner = new AspNetTheoryTestCaseRunner( + testCase, + testCase.DisplayName, + testCase.SkipReason, + _constructorArguments, + _diagnosticMessageSink, + MessageBus, + new ExceptionAggregator(Aggregator), + CancellationTokenSource); + return runner.RunAsync(); + } + + return base.RunTestCaseAsync(testCase); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTestRunner.cs b/src/Testing/src/xunit/AspNetTestRunner.cs new file mode 100644 index 0000000000..2786a866b4 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTestRunner.cs @@ -0,0 +1,78 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + internal class AspNetTestRunner : XunitTestRunner + { + public AspNetTestRunner( + ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + string skipReason, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) + { + } + + protected override async Task InvokeTestMethodAsync(ExceptionAggregator aggregator) + { + var repeatAttribute = GetRepeatAttribute(TestMethod); + if (repeatAttribute == null) + { + return await InvokeTestMethodCoreAsync(aggregator); + } + + var repeatContext = new RepeatContext(repeatAttribute.RunCount); + RepeatContext.Current = repeatContext; + + var timeTaken = 0.0M; + for (repeatContext.CurrentIteration = 0; repeatContext.CurrentIteration < repeatContext.Limit; repeatContext.CurrentIteration++) + { + timeTaken = await InvokeTestMethodCoreAsync(aggregator); + if (aggregator.HasExceptions) + { + return timeTaken; + } + } + + return timeTaken; + } + + private Task InvokeTestMethodCoreAsync(ExceptionAggregator aggregator) + { + var invoker = new AspNetTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource); + return invoker.RunAsync(); + } + + private RepeatAttribute GetRepeatAttribute(MethodInfo methodInfo) + { + var attributeCandidate = methodInfo.GetCustomAttribute(); + if (attributeCandidate != null) + { + return attributeCandidate; + } + + attributeCandidate = methodInfo.DeclaringType.GetCustomAttribute(); + if (attributeCandidate != null) + { + return attributeCandidate; + } + + return methodInfo.DeclaringType.Assembly.GetCustomAttribute(); + } + } +} diff --git a/src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs b/src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs new file mode 100644 index 0000000000..a09a17cf69 --- /dev/null +++ b/src/Testing/src/xunit/AspNetTheoryTestCaseRunner.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing.xunit +{ + internal class AspNetTheoryTestCaseRunner : XunitTheoryTestCaseRunner + { + public AspNetTheoryTestCaseRunner( + IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCase, displayName, skipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + { + return new AspNetTestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource); + } + } +} diff --git a/src/Testing/src/xunit/AssemblyFixtureAttribute.cs b/src/Testing/src/xunit/AssemblyFixtureAttribute.cs new file mode 100644 index 0000000000..c3b9eba31d --- /dev/null +++ b/src/Testing/src/xunit/AssemblyFixtureAttribute.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Testing +{ + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] + public class AssemblyFixtureAttribute : Attribute + { + public AssemblyFixtureAttribute(Type fixtureType) + { + FixtureType = fixtureType; + } + + public Type FixtureType { get; private set; } + } +} diff --git a/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs b/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs index 764b613b16..e7f655a5be 100644 --- a/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs +++ b/src/Testing/src/xunit/ConditionalTheoryDiscoverer.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using Xunit.Abstractions; using Xunit.Sdk; @@ -63,5 +64,24 @@ namespace Microsoft.AspNetCore.Testing base.CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) : base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); } + + protected override IEnumerable CreateTestCasesForSkippedDataRow( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, + IAttributeInfo theoryAttribute, + object[] dataRow, + string skipReason) + { + return new[] + { + new WORKAROUND_SkippedDataRowTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, skipReason, dataRow), + }; + } + + [Obsolete] + protected override IXunitTestCase CreateTestCaseForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason) + { + return new WORKAROUND_SkippedDataRowTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, skipReason, dataRow); + } } } diff --git a/src/Testing/src/xunit/FlakyAttribute.cs b/src/Testing/src/xunit/FlakyAttribute.cs index 411435b74a..daceb964fd 100644 --- a/src/Testing/src/xunit/FlakyAttribute.cs +++ b/src/Testing/src/xunit/FlakyAttribute.cs @@ -49,7 +49,7 @@ namespace Microsoft.AspNetCore.Testing /// to xunit.console.exe. Similarly, it can run only flaky tests using -trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true" /// /// - [TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(FlakyTestDiscoverer), "Microsoft.AspNetCore.Testing")] + [TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(FlakyTraitDiscoverer), "Microsoft.AspNetCore.Testing")] [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] public sealed class FlakyAttribute : Attribute, ITraitAttribute { diff --git a/src/Testing/src/xunit/FlakyTestDiscoverer.cs b/src/Testing/src/xunit/FlakyTraitDiscoverer.cs similarity index 95% rename from src/Testing/src/xunit/FlakyTestDiscoverer.cs rename to src/Testing/src/xunit/FlakyTraitDiscoverer.cs index 0c481e1c41..4e6bc27b1b 100644 --- a/src/Testing/src/xunit/FlakyTestDiscoverer.cs +++ b/src/Testing/src/xunit/FlakyTraitDiscoverer.cs @@ -6,7 +6,7 @@ using Xunit.Sdk; // Do not change this namespace without changing the usage in FlakyAttribute namespace Microsoft.AspNetCore.Testing { - public class FlakyTestDiscoverer : ITraitDiscoverer + public class FlakyTraitDiscoverer : ITraitDiscoverer { public IEnumerable> GetTraits(IAttributeInfo traitAttribute) { diff --git a/src/Testing/src/xunit/SkippedTestCase.cs b/src/Testing/src/xunit/SkippedTestCase.cs index b514c57209..0fdf166f2b 100644 --- a/src/Testing/src/xunit/SkippedTestCase.cs +++ b/src/Testing/src/xunit/SkippedTestCase.cs @@ -2,6 +2,8 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Threading; +using System.Threading.Tasks; using Xunit.Abstractions; using Xunit.Sdk; @@ -33,8 +35,11 @@ namespace Microsoft.AspNetCore.Testing public override void Deserialize(IXunitSerializationInfo data) { - base.Deserialize(data); _skipReason = data.GetValue(nameof(_skipReason)); + + // We need to call base after reading our value, because Deserialize will call + // into GetSkipReason. + base.Deserialize(data); } public override void Serialize(IXunitSerializationInfo data) diff --git a/src/Testing/src/xunit/WORKAROUND_SkippedDataRowTestCase.cs b/src/Testing/src/xunit/WORKAROUND_SkippedDataRowTestCase.cs new file mode 100644 index 0000000000..a86f5645bf --- /dev/null +++ b/src/Testing/src/xunit/WORKAROUND_SkippedDataRowTestCase.cs @@ -0,0 +1,80 @@ +using System; +using System.ComponentModel; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing +{ + // This is a workaround for https://github.com/xunit/xunit/issues/1782 - as such, this code is a copy-paste + // from xUnit with the exception of fixing the bug. + // + // This will only work with [ConditionalTheory]. + internal class WORKAROUND_SkippedDataRowTestCase : XunitTestCase + { + string skipReason; + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] + public WORKAROUND_SkippedDataRowTestCase() { } + + /// + /// Initializes a new instance of the class. + /// + /// The message sink used to send diagnostic messages + /// Default method display to use (when not customized). + /// The test method this test case belongs to. + /// The reason that this test case will be skipped + /// The arguments for the test method. + [Obsolete("Please call the constructor which takes TestMethodDisplayOptions")] + public WORKAROUND_SkippedDataRowTestCase(IMessageSink diagnosticMessageSink, + TestMethodDisplay defaultMethodDisplay, + ITestMethod testMethod, + string skipReason, + object[] testMethodArguments = null) + : this(diagnosticMessageSink, defaultMethodDisplay, TestMethodDisplayOptions.None, testMethod, skipReason, testMethodArguments) { } + + /// + /// Initializes a new instance of the class. + /// + /// The message sink used to send diagnostic messages + /// Default method display to use (when not customized). + /// Default method display options to use (when not customized). + /// The test method this test case belongs to. + /// The reason that this test case will be skipped + /// The arguments for the test method. + public WORKAROUND_SkippedDataRowTestCase(IMessageSink diagnosticMessageSink, + TestMethodDisplay defaultMethodDisplay, + TestMethodDisplayOptions defaultMethodDisplayOptions, + ITestMethod testMethod, + string skipReason, + object[] testMethodArguments = null) + : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) + { + this.skipReason = skipReason; + } + + /// + public override void Deserialize(IXunitSerializationInfo data) + { + // SkipReason has to be read before we call base.Deserialize, this is the workaround. + this.skipReason = data.GetValue("SkipReason"); + + base.Deserialize(data); + } + + /// + protected override string GetSkipReason(IAttributeInfo factAttribute) + { + return skipReason; + } + + /// + public override void Serialize(IXunitSerializationInfo data) + { + base.Serialize(data); + + data.AddValue("SkipReason", skipReason); + } + } +} diff --git a/src/Testing/test/AssemblyFixtureTest.cs b/src/Testing/test/AssemblyFixtureTest.cs new file mode 100644 index 0000000000..a5fa73019a --- /dev/null +++ b/src/Testing/test/AssemblyFixtureTest.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace Microsoft.AspNetCore.Testing +{ + // We include a collection and assembly fixture to verify that they both still work. + [Collection("MyCollection")] + [TestCaseOrderer("Microsoft.AspNetCore.Testing.AlphabeticalOrderer", "Microsoft.AspNetCore.Testing.Tests")] + public class AssemblyFixtureTest + { + public AssemblyFixtureTest(TestAssemblyFixture assemblyFixture, TestCollectionFixture collectionFixture) + { + AssemblyFixture = assemblyFixture; + CollectionFixture = collectionFixture; + } + + public TestAssemblyFixture AssemblyFixture { get; } + public TestCollectionFixture CollectionFixture { get; } + + [Fact] + public void A() + { + Assert.NotNull(AssemblyFixture); + Assert.Equal(0, AssemblyFixture.Count); + + Assert.NotNull(CollectionFixture); + Assert.Equal(0, CollectionFixture.Count); + + AssemblyFixture.Count++; + CollectionFixture.Count++; + } + + [Fact] + public void B() + { + Assert.Equal(1, AssemblyFixture.Count); + Assert.Equal(1, CollectionFixture.Count); + } + } + + [CollectionDefinition("MyCollection", DisableParallelization = true)] + public class MyCollection : ICollectionFixture + { + } +} diff --git a/src/Testing/test/Properties/AssemblyInfo.cs b/src/Testing/test/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..d585b5ed95 --- /dev/null +++ b/src/Testing/test/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +using Microsoft.AspNetCore.Testing; +using Xunit; + +[assembly: Repeat(1)] +[assembly: AssemblyFixture(typeof(TestAssemblyFixture))] +[assembly: TestFramework("Microsoft.AspNetCore.Testing.AspNetTestFramework", "Microsoft.AspNetCore.Testing")] diff --git a/src/Testing/test/RepeatTest.cs b/src/Testing/test/RepeatTest.cs new file mode 100644 index 0000000000..0d995fad59 --- /dev/null +++ b/src/Testing/test/RepeatTest.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace Microsoft.AspNetCore.Testing +{ + [Repeat] + public class RepeatTest + { + public static int _runCount = 0; + + [Fact] + [Repeat(5)] + public void RepeatLimitIsSetCorrectly() + { + Assert.Equal(5, RepeatContext.Current.Limit); + } + + [Fact] + [Repeat(5)] + public void RepeatRunsTestSpecifiedNumberOfTimes() + { + Assert.Equal(RepeatContext.Current.CurrentIteration, _runCount); + _runCount++; + } + + [Fact] + public void RepeatCanBeSetOnClass() + { + Assert.Equal(10, RepeatContext.Current.Limit); + } + } + + public class LoggedTestXunitRepeatAssemblyTests + { + [Fact] + public void RepeatCanBeSetOnAssembly() + { + Assert.Equal(1, RepeatContext.Current.Limit); + } + } +} diff --git a/src/Testing/test/TestAssemblyFixture.cs b/src/Testing/test/TestAssemblyFixture.cs new file mode 100644 index 0000000000..44308160bd --- /dev/null +++ b/src/Testing/test/TestAssemblyFixture.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.AspNetCore.Testing +{ + public class TestAssemblyFixture + { + public int Count { get; set; } + } +} diff --git a/src/Testing/test/TestCollectionFixture.cs b/src/Testing/test/TestCollectionFixture.cs new file mode 100644 index 0000000000..b9aed01e41 --- /dev/null +++ b/src/Testing/test/TestCollectionFixture.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.AspNetCore.Testing +{ + public class TestCollectionFixture + { + public int Count { get; set; } + } +} diff --git a/src/Testing/test/TestContextTest.cs b/src/Testing/test/TestContextTest.cs new file mode 100644 index 0000000000..944d706477 --- /dev/null +++ b/src/Testing/test/TestContextTest.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.AspNetCore.Testing +{ + public class TestContextTest : ITestMethodLifecycle + { + public TestContext Context { get; private set; } + + [Fact] + public void FullName_IsUsed_ByDefault() + { + Assert.Equal(GetType().FullName, Context.FileOutput.TestClassName); + } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + Context = context; + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} + +namespace Microsoft.AspNetCore.Testing.Tests +{ + public class TestContextNameShorteningTest : ITestMethodLifecycle + { + public TestContext Context { get; private set; } + + [Fact] + public void NameIsShortenedWhenAssemblyNameIsAPrefix() + { + Assert.Equal(GetType().Name, Context.FileOutput.TestClassName); + } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + Context = context; + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} + +namespace Microsoft.AspNetCore.Testing +{ + [ShortClassName] + public class TestContextTestClassShortNameAttributeTest : ITestMethodLifecycle + { + public TestContext Context { get; private set; } + + [Fact] + public void ShortClassNameUsedWhenShortClassNameAttributeSpecified() + { + Assert.Equal(GetType().Name, Context.FileOutput.TestClassName); + } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + Context = context; + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +}