From a8c8ddbb451f81cf88024f1925067df2ddc365ff Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 6 Nov 2018 13:11:45 -0800 Subject: [PATCH 01/13] Reorganize source code in preparation to move into aspnet/Extensions Prior to reorganization, this source code was found in https://github.com/aspnet/Logging/tree/8270c545224e8734d7297e54edef5c584ee82f01 --- src/Testing/src/AssemblyTestLog.cs | 305 ++++++++++++++++++ ...Microsoft.Extensions.Logging.Testing.props | 8 + src/Testing/test/AssemblyTestLogTests.cs | 207 ++++++++++++ src/Testing/test/LoggedTestXunitTests.cs | 142 ++++++++ src/Testing/test/TestTestOutputHelper.cs | 36 +++ 5 files changed, 698 insertions(+) create mode 100644 src/Testing/src/AssemblyTestLog.cs create mode 100644 src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props create mode 100644 src/Testing/test/AssemblyTestLogTests.cs create mode 100644 src/Testing/test/LoggedTestXunitTests.cs create mode 100644 src/Testing/test/TestTestOutputHelper.cs diff --git a/src/Testing/src/AssemblyTestLog.cs b/src/Testing/src/AssemblyTestLog.cs new file mode 100644 index 0000000000..97a67b11fa --- /dev/null +++ b/src/Testing/src/AssemblyTestLog.cs @@ -0,0 +1,305 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Serilog; +using Serilog.Extensions.Logging; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging.Testing +{ + public class AssemblyTestLog : IDisposable + { + public static readonly string OutputDirectoryEnvironmentVariableName = "ASPNETCORE_TEST_LOG_DIR"; + private static readonly string MaxPathLengthEnvironmentVariableName = "ASPNETCORE_TEST_LOG_MAXPATH"; + private static readonly string LogFileExtension = ".log"; + private static readonly int MaxPathLength = GetMaxPathLength(); + private static char[] InvalidFileChars = new char[] + { + '\"', '<', '>', '|', '\0', + (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10, + (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20, + (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30, + (char)31, ':', '*', '?', '\\', '/', ' ', (char)127 + }; + + private static readonly object _lock = new object(); + private static readonly Dictionary _logs = new Dictionary(); + + private readonly ILoggerFactory _globalLoggerFactory; + private readonly ILogger _globalLogger; + private readonly string _baseDirectory; + private readonly string _assemblyName; + private readonly IServiceProvider _serviceProvider; + + private static int GetMaxPathLength() + { + var maxPathString = Environment.GetEnvironmentVariable(MaxPathLengthEnvironmentVariableName); + var defaultMaxPath = 245; + return string.IsNullOrEmpty(maxPathString) ? defaultMaxPath : int.Parse(maxPathString); + } + + private AssemblyTestLog(ILoggerFactory globalLoggerFactory, ILogger globalLogger, string baseDirectory, string assemblyName, IServiceProvider serviceProvider) + { + _globalLoggerFactory = globalLoggerFactory; + _globalLogger = globalLogger; + _baseDirectory = baseDirectory; + _assemblyName = assemblyName; + _serviceProvider = serviceProvider; + } + + public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, [CallerMemberName] string testName = null) => + StartTestLog(output, className, out loggerFactory, LogLevel.Debug, testName); + + public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, [CallerMemberName] string testName = null) => + StartTestLog(output, className, out loggerFactory, minLogLevel, out var _, testName); + + internal IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, out string resolvedTestName, [CallerMemberName] string testName = null) + { + var serviceProvider = CreateLoggerServices(output, className, minLogLevel, out resolvedTestName, testName); + var factory = serviceProvider.GetRequiredService(); + loggerFactory = factory; + var logger = loggerFactory.CreateLogger("TestLifetime"); + + var stopwatch = Stopwatch.StartNew(); + + var scope = logger.BeginScope("Test: {testName}", testName); + + _globalLogger.LogInformation("Starting test {testName}", testName); + logger.LogInformation("Starting test {testName}", testName); + + return new Disposable(() => + { + stopwatch.Stop(); + _globalLogger.LogInformation("Finished test {testName} in {duration}s", testName, stopwatch.Elapsed.TotalSeconds); + logger.LogInformation("Finished test {testName} in {duration}s", testName, stopwatch.Elapsed.TotalSeconds); + scope.Dispose(); + factory.Dispose(); + (serviceProvider as IDisposable)?.Dispose(); + }); + } + + public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, [CallerMemberName] string testName = null) => + CreateLoggerFactory(output, className, LogLevel.Trace, testName); + + public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, LogLevel minLogLevel, [CallerMemberName] string testName = null) + { + return CreateLoggerServices(output, className, minLogLevel, out var _, testName).GetRequiredService(); + } + + public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string className, LogLevel minLogLevel, out string normalizedTestName, [CallerMemberName] string testName = null) + { + normalizedTestName = string.Empty; + + // Try to shorten the class name using the assembly name + if (className.StartsWith(_assemblyName + ".")) + { + className = className.Substring(_assemblyName.Length + 1); + } + + SerilogLoggerProvider serilogLoggerProvider = null; + if (!string.IsNullOrEmpty(_baseDirectory)) + { + var testOutputDirectory = Path.Combine(GetAssemblyBaseDirectory(_assemblyName, _baseDirectory), className); + testName = RemoveIllegalFileChars(testName); + + if (testOutputDirectory.Length + testName.Length + LogFileExtension.Length >= MaxPathLength) + { + _globalLogger.LogWarning($"Test name {testName} is too long. Please shorten test name."); + + // Shorten the test name by removing the middle portion of the testname + var testNameLength = MaxPathLength - testOutputDirectory.Length - LogFileExtension.Length; + + if (testNameLength <= 0) + { + throw new InvalidOperationException("Output file path could not be constructed due to max path length restrictions. Please shorten test assembly, class or method names."); + } + + testName = testName.Substring(0, testNameLength / 2) + testName.Substring(testName.Length - testNameLength / 2, testNameLength / 2); + + _globalLogger.LogWarning($"To prevent long paths test name was shortened to {testName}."); + } + + var testOutputFile = Path.Combine(testOutputDirectory, $"{testName}{LogFileExtension}"); + + if (File.Exists(testOutputFile)) + { + _globalLogger.LogWarning($"Output log file {testOutputFile} already exists. Please try to keep log file names unique."); + + for (var i = 0; i < 1000; i++) + { + testOutputFile = Path.Combine(testOutputDirectory, $"{testName}.{i}{LogFileExtension}"); + + if (!File.Exists(testOutputFile)) + { + _globalLogger.LogWarning($"To resolve log file collision, the enumerated file {testOutputFile} will be used."); + testName = $"{testName}.{i}"; + break; + } + } + } + + normalizedTestName = testName; + serilogLoggerProvider = ConfigureFileLogging(testOutputFile); + } + + var serviceCollection = new ServiceCollection(); + serviceCollection.AddLogging(builder => + { + builder.SetMinimumLevel(minLogLevel); + + if (output != null) + { + builder.AddXunit(output, minLogLevel); + } + + if (serilogLoggerProvider != null) + { + // Use a factory so that the container will dispose it + builder.Services.AddSingleton(_ => serilogLoggerProvider); + } + }); + + return serviceCollection.BuildServiceProvider(); + } + + public static AssemblyTestLog Create(string assemblyName, string baseDirectory) + { + SerilogLoggerProvider serilogLoggerProvider = null; + var globalLogDirectory = GetAssemblyBaseDirectory(assemblyName, baseDirectory); + if (!string.IsNullOrEmpty(globalLogDirectory)) + { + var globalLogFileName = Path.Combine(globalLogDirectory, "global.log"); + serilogLoggerProvider = ConfigureFileLogging(globalLogFileName); + } + + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddLogging(builder => + { + // Global logging, when it's written, is expected to be outputted. So set the log level to minimum. + builder.SetMinimumLevel(LogLevel.Trace); + + if (serilogLoggerProvider != null) + { + // Use a factory so that the container will dispose it + builder.Services.AddSingleton(_ => serilogLoggerProvider); + } + }); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + var loggerFactory = serviceProvider.GetRequiredService(); + + var logger = loggerFactory.CreateLogger("GlobalTestLog"); + logger.LogInformation($"Global Test Logging initialized. Set the '{OutputDirectoryEnvironmentVariableName}' Environment Variable in order to create log files on disk."); + return new AssemblyTestLog(loggerFactory, logger, baseDirectory, assemblyName, serviceProvider); + } + + public static AssemblyTestLog ForAssembly(Assembly assembly) + { + lock (_lock) + { + if (!_logs.TryGetValue(assembly, out var log)) + { + var assemblyName = assembly.GetName().Name; + var baseDirectory = Environment.GetEnvironmentVariable(OutputDirectoryEnvironmentVariableName); + log = Create(assemblyName, baseDirectory); + _logs[assembly] = log; + + // Try to clear previous logs + var assemblyBaseDirectory = GetAssemblyBaseDirectory(assemblyName, baseDirectory); + if (Directory.Exists(assemblyBaseDirectory)) + { + try + { + Directory.Delete(assemblyBaseDirectory, recursive: true); + } + catch {} + } + } + return log; + } + } + + private static string GetAssemblyBaseDirectory(string assemblyName, string baseDirectory) + { + if (!string.IsNullOrEmpty(baseDirectory)) + { + return Path.Combine(baseDirectory, assemblyName, RuntimeInformation.FrameworkDescription.TrimStart('.')); + } + return string.Empty; + } + + private static SerilogLoggerProvider ConfigureFileLogging(string fileName) + { + var dir = Path.GetDirectoryName(fileName); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + if (File.Exists(fileName)) + { + File.Delete(fileName); + } + + var serilogger = new LoggerConfiguration() + .Enrich.FromLogContext() + .MinimumLevel.Verbose() + .WriteTo.File(fileName, outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{SourceContext}] [{Level}] {Message}{NewLine}{Exception}", flushToDiskInterval: TimeSpan.FromSeconds(1), shared: true) + .CreateLogger(); + return new SerilogLoggerProvider(serilogger, dispose: true); + } + + private static string RemoveIllegalFileChars(string s) + { + var sb = new StringBuilder(); + + foreach (var c in s) + { + if (InvalidFileChars.Contains(c)) + { + if (sb.Length > 0 && sb[sb.Length - 1] != '_') + { + sb.Append('_'); + } + } + else + { + sb.Append(c); + } + } + return sb.ToString(); + } + + public void Dispose() + { + (_serviceProvider as IDisposable)?.Dispose(); + _globalLoggerFactory.Dispose(); + } + + private class Disposable : IDisposable + { + private Action _action; + + public Disposable(Action action) + { + _action = action; + } + + public void Dispose() + { + _action(); + } + } + } +} diff --git a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props new file mode 100644 index 0000000000..f98e3e13b5 --- /dev/null +++ b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props @@ -0,0 +1,8 @@ + + + + <_Parameter1>Microsoft.Extensions.Logging.Testing.LoggedTestFramework + <_Parameter2>Microsoft.Extensions.Logging.Testing + + + \ No newline at end of file diff --git a/src/Testing/test/AssemblyTestLogTests.cs b/src/Testing/test/AssemblyTestLogTests.cs new file mode 100644 index 0000000000..0efadb4367 --- /dev/null +++ b/src/Testing/test/AssemblyTestLogTests.cs @@ -0,0 +1,207 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.Logging.Testing.Tests +{ + public class AssemblyTestLogTests : LoggedTest + { + private static readonly Assembly ThisAssembly = typeof(AssemblyTestLog).GetTypeInfo().Assembly; + + [Fact] + public void FullClassNameUsedWhenShortClassNameAttributeNotSpecified() + { + Assert.Equal(GetType().FullName, ResolvedTestClassName); + } + + [Fact] + public void ForAssembly_ReturnsSameInstanceForSameAssembly() + { + Assert.Same( + AssemblyTestLog.ForAssembly(ThisAssembly), + AssemblyTestLog.ForAssembly(ThisAssembly)); + } + + [Fact] + public void TestLogWritesToITestOutputHelper() + { + var output = new TestTestOutputHelper(); + var assemblyLog = AssemblyTestLog.Create("NonExistant.Test.Assembly", baseDirectory: null); + + using (assemblyLog.StartTestLog(output, "NonExistant.Test.Class", out var loggerFactory)) + { + var logger = loggerFactory.CreateLogger("TestLogger"); + logger.LogInformation("Information!"); + + // Trace is disabled by default + logger.LogTrace("Trace!"); + } + + Assert.Equal(@"[TIMESTAMP] TestLifetime Information: Starting test TestLogWritesToITestOutputHelper +[TIMESTAMP] TestLogger Information: Information! +[TIMESTAMP] TestLifetime Information: Finished test TestLogWritesToITestOutputHelper in DURATION +", MakeConsistent(output.Output), ignoreLineEndingDifferences: true); + } + + [Fact] + private Task TestLogEscapesIllegalFileNames() => + RunTestLogFunctionalTest((tempDir) => + { + var illegalTestName = "Testing-https://localhost:5000"; + var escapedTestName = "Testing-https_localhost_5000"; + using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", baseDirectory: tempDir)) + using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, resolvedTestName: out var resolvedTestname, testName: illegalTestName)) + { + Assert.Equal(escapedTestName, resolvedTestname); + } + }); + + [Fact] + public Task TestLogWritesToGlobalLogFile() => + RunTestLogFunctionalTest((tempDir) => + { + // Because this test writes to a file, it is a functional test and should be logged + // but it's also testing the test logging facility. So this is pretty meta ;) + var logger = LoggerFactory.CreateLogger("Test"); + + using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", tempDir)) + { + logger.LogInformation("Created test log in {baseDirectory}", tempDir); + + using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) + { + var testLogger = testLoggerFactory.CreateLogger("TestLogger"); + testLogger.LogInformation("Information!"); + testLogger.LogTrace("Trace!"); + } + } + + logger.LogInformation("Finished test log in {baseDirectory}", tempDir); + + var globalLogPath = Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "global.log"); + var testLog = Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass", $"FakeTestName.log"); + + Assert.True(File.Exists(globalLogPath), $"Expected global log file {globalLogPath} to exist"); + Assert.True(File.Exists(testLog), $"Expected test log file {testLog} to exist"); + + var globalLogContent = MakeConsistent(File.ReadAllText(globalLogPath)); + var testLogContent = MakeConsistent(File.ReadAllText(testLog)); + + Assert.Equal(@"[GlobalTestLog] [Information] Global Test Logging initialized. Set the 'ASPNETCORE_TEST_LOG_DIR' Environment Variable in order to create log files on disk. +[GlobalTestLog] [Information] Starting test ""FakeTestName"" +[GlobalTestLog] [Information] Finished test ""FakeTestName"" in DURATION +", globalLogContent, ignoreLineEndingDifferences: true); + Assert.Equal(@"[TestLifetime] [Information] Starting test ""FakeTestName"" +[TestLogger] [Information] Information! +[TestLogger] [Verbose] Trace! +[TestLifetime] [Information] Finished test ""FakeTestName"" in DURATION +", testLogContent, ignoreLineEndingDifferences: true); + }); + + [Fact] + public Task TestLogTruncatesTestNameToAvoidLongPaths() => + RunTestLogFunctionalTest((tempDir) => + { + var longTestName = new string('0', 50) + new string('1', 50) + new string('2', 50) + new string('3', 50) + new string('4', 50); + var logger = LoggerFactory.CreateLogger("Test"); + using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", tempDir)) + { + logger.LogInformation("Created test log in {baseDirectory}", tempDir); + + using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: longTestName)) + { + testLoggerFactory.CreateLogger("TestLogger").LogInformation("Information!"); + } + } + logger.LogInformation("Finished test log in {baseDirectory}", tempDir); + + var testLogFiles = new DirectoryInfo(Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass")).EnumerateFiles(); + var testLog = Assert.Single(testLogFiles); + var testFileName = Path.GetFileNameWithoutExtension(testLog.Name); + + // The first half of the file comes from the beginning of the test name passed to the logger + Assert.Equal(longTestName.Substring(0, testFileName.Length / 2), testFileName.Substring(0, testFileName.Length / 2)); + // The last half of the file comes from the ending of the test name passed to the logger + Assert.Equal(longTestName.Substring(longTestName.Length - testFileName.Length / 2, testFileName.Length / 2), testFileName.Substring(testFileName.Length - testFileName.Length / 2, testFileName.Length / 2)); + }); + + [Fact] + public Task TestLogEnumerateFilenamesToAvoidCollisions() => + RunTestLogFunctionalTest((tempDir) => + { + var logger = LoggerFactory.CreateLogger("Test"); + using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", tempDir)) + { + logger.LogInformation("Created test log in {baseDirectory}", tempDir); + + for (var i = 0; i < 10; i++) + { + using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) + { + testLoggerFactory.CreateLogger("TestLogger").LogInformation("Information!"); + } + } + } + logger.LogInformation("Finished test log in {baseDirectory}", tempDir); + + // The first log file exists + Assert.True(File.Exists(Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass", $"FakeTestName.log"))); + + // Subsequent files exist + for (var i = 0; i < 9; i++) + { + Assert.True(File.Exists(Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass", $"FakeTestName.{i}.log"))); + } + }); + + private static readonly Regex TimestampRegex = new Regex(@"\d+-\d+-\d+T\d+:\d+:\d+"); + private static readonly Regex DurationRegex = new Regex(@"[^ ]+s$"); + + private async Task RunTestLogFunctionalTest(Action action, [CallerMemberName] string testName = null) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"TestLogging_{Guid.NewGuid().ToString("N")}"); + try + { + action(tempDir); + } + finally + { + if (Directory.Exists(tempDir)) + { + try + { + Directory.Delete(tempDir, recursive: true); + } + catch + { + await Task.Delay(100); + Directory.Delete(tempDir, recursive: true); + } + } + } + } + + private static string MakeConsistent(string input) + { + return string.Join(Environment.NewLine, input.Split(new[] { Environment.NewLine }, StringSplitOptions.None) + .Select(line => + { + var strippedPrefix = line.IndexOf("[") >= 0 ? line.Substring(line.IndexOf("[")) : line; + + var strippedDuration = + DurationRegex.Replace(strippedPrefix, "DURATION"); + var strippedTimestamp = TimestampRegex.Replace(strippedDuration, "TIMESTAMP"); + return strippedTimestamp; + })); + } + } +} diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs new file mode 100644 index 0000000000..31fd6d631f --- /dev/null +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -0,0 +1,142 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.Extensions.DependencyInjection; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging.Testing.Tests +{ + [ShortClassName] + public class LoggedTestXunitTests : TestLoggedTest + { + private readonly ITestOutputHelper _output; + + public LoggedTestXunitTests(ITestOutputHelper output) + { + _output = output; + } + + [Fact] + public void ShortClassNameUsedWhenShortClassNameAttributeSpecified() + { + Assert.Equal(GetType().Name, ResolvedTestClassName); + } + + [Fact] + public void LoggedTestTestOutputHelperSameInstanceAsInjectedConstructorArg() + { + Assert.Same(_output, TestOutputHelper); + } + + [Fact] + public void LoggedFactInitializesLoggedTestProperties() + { + Assert.NotNull(Logger); + Assert.NotNull(LoggerFactory); + Assert.NotNull(TestSink); + Assert.NotNull(TestOutputHelper); + } + + [Theory] + [InlineData("Hello world")] + public void LoggedTheoryInitializesLoggedTestProperties(string argument) + { + Assert.NotNull(Logger); + Assert.NotNull(LoggerFactory); + Assert.NotNull(TestSink); + Assert.NotNull(TestOutputHelper); + // Use the test argument + Assert.NotNull(argument); + } + + [ConditionalFact] + public void ConditionalLoggedFactGetsInitializedLoggerFactory() + { + Assert.NotNull(Logger); + Assert.NotNull(LoggerFactory); + Assert.NotNull(TestSink); + Assert.NotNull(TestOutputHelper); + } + + [ConditionalTheory] + [InlineData("Hello world")] + public void LoggedConditionalTheoryInitializesLoggedTestProperties(string argument) + { + Assert.NotNull(Logger); + Assert.NotNull(LoggerFactory); + Assert.NotNull(TestSink); + Assert.NotNull(TestOutputHelper); + // Use the test argument + Assert.NotNull(argument); + } + + [Fact] + [LogLevel(LogLevel.Information)] + public void LoggedFactFilteredByLogLevel() + { + Logger.LogInformation("Information"); + Logger.LogDebug("Debug"); + + var message = Assert.Single(TestSink.Writes); + Assert.Equal(LogLevel.Information, message.LogLevel); + Assert.Equal("Information", message.Formatter(message.State, null)); + } + + [Theory] + [InlineData("Hello world")] + [LogLevel(LogLevel.Information)] + public void LoggedTheoryFilteredByLogLevel(string argument) + { + Logger.LogInformation("Information"); + Logger.LogDebug("Debug"); + + var message = Assert.Single(TestSink.Writes); + Assert.Equal(LogLevel.Information, message.LogLevel); + Assert.Equal("Information", message.Formatter(message.State, null)); + + // Use the test argument + Assert.NotNull(argument); + } + + [Fact] + public void AddTestLoggingUpdatedWhenLoggerFactoryIsSet() + { + var loggerFactory = new LoggerFactory(); + var serviceCollection = new ServiceCollection(); + + LoggerFactory = loggerFactory; + AddTestLogging(serviceCollection); + + Assert.Same(loggerFactory, serviceCollection.BuildServiceProvider().GetRequiredService()); + } + + [ConditionalTheory] + [EnvironmentVariableSkipCondition("ASPNETCORE_TEST_LOG_DIR", "")] // The test name is only generated when logging is enabled via the environment variable + [InlineData(null)] + public void LoggedTheoryNullArgumentsAreEscaped(string argument) + { + Assert.NotNull(LoggerFactory); + Assert.Equal($"{nameof(LoggedTheoryNullArgumentsAreEscaped)}_null", ResolvedTestMethodName); + // Use the test argument + Assert.Null(argument); + } + + [Fact] + public void AdditionalSetupInvoked() + { + Assert.True(SetupInvoked); + } + } + + public class TestLoggedTest : LoggedTest + { + public bool SetupInvoked { get; private set; } = false; + + public override void AdditionalSetup() + { + SetupInvoked = true; + } + } +} diff --git a/src/Testing/test/TestTestOutputHelper.cs b/src/Testing/test/TestTestOutputHelper.cs new file mode 100644 index 0000000000..7043fe4ed2 --- /dev/null +++ b/src/Testing/test/TestTestOutputHelper.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Text; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging.Testing.Tests +{ + public class TestTestOutputHelper : ITestOutputHelper + { + private StringBuilder _output = new StringBuilder(); + + public bool Throw { get; set; } + + public string Output => _output.ToString(); + + public void WriteLine(string message) + { + if (Throw) + { + throw new Exception("Boom!"); + } + _output.AppendLine(message); + } + + public void WriteLine(string format, params object[] args) + { + if (Throw) + { + throw new Exception("Boom!"); + } + _output.AppendLine(string.Format(format, args)); + } + } +} From fd100ade9e885b4a8adced9414434cde0bf35b7a Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 6 Nov 2018 15:28:23 -0800 Subject: [PATCH 02/13] Reorganize source code in preparation to move into aspnet/Extensions Prior to reorganization, this source code was found in https://github.com/aspnet/Logging/tree/5381f42ded1f41a3b0960bba799aed53da411401 --- src/Testing/src/AssemblyTestLog.cs | 119 +++++++++++------- src/Testing/src/LoggedTest/ILoggedTest.cs | 23 ++++ src/Testing/src/LoggedTest/LoggedTest.cs | 24 ++++ src/Testing/src/LoggedTest/LoggedTestBase.cs | 82 ++++++++++++ .../src/TestFrameworkFileLoggerAttribute.cs | 20 +++ ...Microsoft.Extensions.Logging.Testing.props | 27 +++- src/Testing/test/AssemblyTestLogTests.cs | 71 ++++++----- src/Testing/test/LoggedTestXunitTests.cs | 20 ++- 8 files changed, 304 insertions(+), 82 deletions(-) create mode 100644 src/Testing/src/LoggedTest/ILoggedTest.cs create mode 100644 src/Testing/src/LoggedTest/LoggedTest.cs create mode 100644 src/Testing/src/LoggedTest/LoggedTestBase.cs create mode 100644 src/Testing/src/TestFrameworkFileLoggerAttribute.cs diff --git a/src/Testing/src/AssemblyTestLog.cs b/src/Testing/src/AssemblyTestLog.cs index 97a67b11fa..e84df52554 100644 --- a/src/Testing/src/AssemblyTestLog.cs +++ b/src/Testing/src/AssemblyTestLog.cs @@ -8,10 +8,11 @@ using System.IO; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Text; using Microsoft.Extensions.DependencyInjection; using Serilog; +using Serilog.Core; +using Serilog.Events; using Serilog.Extensions.Logging; using Xunit.Abstractions; @@ -19,7 +20,6 @@ namespace Microsoft.Extensions.Logging.Testing { public class AssemblyTestLog : IDisposable { - public static readonly string OutputDirectoryEnvironmentVariableName = "ASPNETCORE_TEST_LOG_DIR"; private static readonly string MaxPathLengthEnvironmentVariableName = "ASPNETCORE_TEST_LOG_MAXPATH"; private static readonly string LogFileExtension = ".log"; private static readonly int MaxPathLength = GetMaxPathLength(); @@ -38,7 +38,7 @@ namespace Microsoft.Extensions.Logging.Testing private readonly ILoggerFactory _globalLoggerFactory; private readonly ILogger _globalLogger; private readonly string _baseDirectory; - private readonly string _assemblyName; + private readonly Assembly _assembly; private readonly IServiceProvider _serviceProvider; private static int GetMaxPathLength() @@ -48,12 +48,12 @@ namespace Microsoft.Extensions.Logging.Testing return string.IsNullOrEmpty(maxPathString) ? defaultMaxPath : int.Parse(maxPathString); } - private AssemblyTestLog(ILoggerFactory globalLoggerFactory, ILogger globalLogger, string baseDirectory, string assemblyName, IServiceProvider serviceProvider) + private AssemblyTestLog(ILoggerFactory globalLoggerFactory, ILogger globalLogger, string baseDirectory, Assembly assembly, IServiceProvider serviceProvider) { _globalLoggerFactory = globalLoggerFactory; _globalLogger = globalLogger; _baseDirectory = baseDirectory; - _assemblyName = assemblyName; + _assembly = assembly; _serviceProvider = serviceProvider; } @@ -61,11 +61,12 @@ namespace Microsoft.Extensions.Logging.Testing StartTestLog(output, className, out loggerFactory, LogLevel.Debug, testName); public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, [CallerMemberName] string testName = null) => - StartTestLog(output, className, out loggerFactory, minLogLevel, out var _, testName); + StartTestLog(output, className, out loggerFactory, minLogLevel, out var _, out var _, testName); - internal IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, out string resolvedTestName, [CallerMemberName] string testName = null) + internal IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, out string resolvedTestName, out string logOutputDirectory, [CallerMemberName] string testName = null) { - var serviceProvider = CreateLoggerServices(output, className, minLogLevel, out resolvedTestName, testName); + var logStart = DateTimeOffset.UtcNow; + var serviceProvider = CreateLoggerServices(output, className, minLogLevel, out resolvedTestName, out logOutputDirectory, testName, logStart); var factory = serviceProvider.GetRequiredService(); loggerFactory = factory; var logger = loggerFactory.CreateLogger("TestLifetime"); @@ -75,7 +76,7 @@ namespace Microsoft.Extensions.Logging.Testing var scope = logger.BeginScope("Test: {testName}", testName); _globalLogger.LogInformation("Starting test {testName}", testName); - logger.LogInformation("Starting test {testName}", testName); + logger.LogInformation("Starting test {testName} at {logStart}", testName, logStart.ToString("s")); return new Disposable(() => { @@ -88,36 +89,39 @@ namespace Microsoft.Extensions.Logging.Testing }); } - public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, [CallerMemberName] string testName = null) => - CreateLoggerFactory(output, className, LogLevel.Trace, testName); + public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, [CallerMemberName] string testName = null, DateTimeOffset? logStart = null) + => CreateLoggerFactory(output, className, LogLevel.Trace, testName, logStart); - public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, LogLevel minLogLevel, [CallerMemberName] string testName = null) - { - return CreateLoggerServices(output, className, minLogLevel, out var _, testName).GetRequiredService(); - } + public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, LogLevel minLogLevel, [CallerMemberName] string testName = null, DateTimeOffset? logStart = null) + => CreateLoggerServices(output, className, minLogLevel, out var _, out var _, testName, logStart).GetRequiredService(); - public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string className, LogLevel minLogLevel, out string normalizedTestName, [CallerMemberName] string testName = null) + public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string className, LogLevel minLogLevel, out string normalizedTestName, [CallerMemberName] string testName = null, DateTimeOffset? logStart = null) + => CreateLoggerServices(output, className, minLogLevel, out normalizedTestName, out var _, testName, logStart); + + public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string className, LogLevel minLogLevel, out string normalizedTestName, out string logOutputDirectory, [CallerMemberName] string testName = null, DateTimeOffset? logStart = null) { normalizedTestName = string.Empty; + logOutputDirectory = string.Empty; + var assemblyName = _assembly.GetName().Name; // Try to shorten the class name using the assembly name - if (className.StartsWith(_assemblyName + ".")) + if (className.StartsWith(assemblyName + ".")) { - className = className.Substring(_assemblyName.Length + 1); + className = className.Substring(assemblyName.Length + 1); } SerilogLoggerProvider serilogLoggerProvider = null; if (!string.IsNullOrEmpty(_baseDirectory)) { - var testOutputDirectory = Path.Combine(GetAssemblyBaseDirectory(_assemblyName, _baseDirectory), className); + logOutputDirectory = Path.Combine(GetAssemblyBaseDirectory(_baseDirectory, _assembly), className); testName = RemoveIllegalFileChars(testName); - if (testOutputDirectory.Length + testName.Length + LogFileExtension.Length >= MaxPathLength) + if (logOutputDirectory.Length + testName.Length + LogFileExtension.Length >= MaxPathLength) { _globalLogger.LogWarning($"Test name {testName} is too long. Please shorten test name."); // Shorten the test name by removing the middle portion of the testname - var testNameLength = MaxPathLength - testOutputDirectory.Length - LogFileExtension.Length; + var testNameLength = MaxPathLength - logOutputDirectory.Length - LogFileExtension.Length; if (testNameLength <= 0) { @@ -129,7 +133,7 @@ namespace Microsoft.Extensions.Logging.Testing _globalLogger.LogWarning($"To prevent long paths test name was shortened to {testName}."); } - var testOutputFile = Path.Combine(testOutputDirectory, $"{testName}{LogFileExtension}"); + var testOutputFile = Path.Combine(logOutputDirectory, $"{testName}{LogFileExtension}"); if (File.Exists(testOutputFile)) { @@ -137,7 +141,7 @@ namespace Microsoft.Extensions.Logging.Testing for (var i = 0; i < 1000; i++) { - testOutputFile = Path.Combine(testOutputDirectory, $"{testName}.{i}{LogFileExtension}"); + testOutputFile = Path.Combine(logOutputDirectory, $"{testName}.{i}{LogFileExtension}"); if (!File.Exists(testOutputFile)) { @@ -149,7 +153,7 @@ namespace Microsoft.Extensions.Logging.Testing } normalizedTestName = testName; - serilogLoggerProvider = ConfigureFileLogging(testOutputFile); + serilogLoggerProvider = ConfigureFileLogging(testOutputFile, logStart); } var serviceCollection = new ServiceCollection(); @@ -159,7 +163,7 @@ namespace Microsoft.Extensions.Logging.Testing if (output != null) { - builder.AddXunit(output, minLogLevel); + builder.AddXunit(output, minLogLevel, logStart); } if (serilogLoggerProvider != null) @@ -172,14 +176,19 @@ namespace Microsoft.Extensions.Logging.Testing return serviceCollection.BuildServiceProvider(); } + // For back compat public static AssemblyTestLog Create(string assemblyName, string baseDirectory) + => Create(Assembly.Load(new AssemblyName(assemblyName)), baseDirectory); + + public static AssemblyTestLog Create(Assembly assembly, string baseDirectory) { + var logStart = DateTimeOffset.UtcNow; SerilogLoggerProvider serilogLoggerProvider = null; - var globalLogDirectory = GetAssemblyBaseDirectory(assemblyName, baseDirectory); + var globalLogDirectory = GetAssemblyBaseDirectory(baseDirectory, assembly); if (!string.IsNullOrEmpty(globalLogDirectory)) { var globalLogFileName = Path.Combine(globalLogDirectory, "global.log"); - serilogLoggerProvider = ConfigureFileLogging(globalLogFileName); + serilogLoggerProvider = ConfigureFileLogging(globalLogFileName, logStart); } var serviceCollection = new ServiceCollection(); @@ -200,8 +209,11 @@ namespace Microsoft.Extensions.Logging.Testing var loggerFactory = serviceProvider.GetRequiredService(); var logger = loggerFactory.CreateLogger("GlobalTestLog"); - logger.LogInformation($"Global Test Logging initialized. Set the '{OutputDirectoryEnvironmentVariableName}' Environment Variable in order to create log files on disk."); - return new AssemblyTestLog(loggerFactory, logger, baseDirectory, assemblyName, serviceProvider); + logger.LogInformation("Global Test Logging initialized at {logStart}. " + + "Configure the output directory via 'LoggingTestingFileLoggingDirectory' MSBuild property " + + "or set 'LoggingTestingDisableFileLogging' to 'true' to disable file logging.", + logStart.ToString("s")); + return new AssemblyTestLog(loggerFactory, logger, baseDirectory, assembly, serviceProvider); } public static AssemblyTestLog ForAssembly(Assembly assembly) @@ -210,13 +222,13 @@ namespace Microsoft.Extensions.Logging.Testing { if (!_logs.TryGetValue(assembly, out var log)) { - var assemblyName = assembly.GetName().Name; - var baseDirectory = Environment.GetEnvironmentVariable(OutputDirectoryEnvironmentVariableName); - log = Create(assemblyName, baseDirectory); + var baseDirectory = GetFileLoggerAttribute(assembly).BaseDirectory; + + log = Create(assembly, baseDirectory); _logs[assembly] = log; // Try to clear previous logs - var assemblyBaseDirectory = GetAssemblyBaseDirectory(assemblyName, baseDirectory); + var assemblyBaseDirectory = GetAssemblyBaseDirectory(baseDirectory, assembly); if (Directory.Exists(assemblyBaseDirectory)) { try @@ -230,16 +242,18 @@ namespace Microsoft.Extensions.Logging.Testing } } - private static string GetAssemblyBaseDirectory(string assemblyName, string baseDirectory) - { - if (!string.IsNullOrEmpty(baseDirectory)) - { - return Path.Combine(baseDirectory, assemblyName, RuntimeInformation.FrameworkDescription.TrimStart('.')); - } - return string.Empty; - } + private static string GetAssemblyBaseDirectory(string baseDirectory, Assembly assembly) + => string.IsNullOrEmpty(baseDirectory) + ? string.Empty + : Path.Combine(baseDirectory, assembly.GetName().Name, GetFileLoggerAttribute(assembly).TFM); - private static SerilogLoggerProvider ConfigureFileLogging(string fileName) + private static TestFrameworkFileLoggerAttribute GetFileLoggerAttribute(Assembly assembly) + => assembly.GetCustomAttribute() + ?? throw new InvalidOperationException($"No {nameof(TestFrameworkFileLoggerAttribute)} found on the assembly {assembly.GetName().Name}. " + + "The attribute is added via msbuild properties of the Microsoft.Extensions.Logging.Testing. " + + "Please ensure the msbuild property is imported or a direct reference to Microsoft.Extensions.Logging.Testing is added."); + + private static SerilogLoggerProvider ConfigureFileLogging(string fileName, DateTimeOffset? logStart) { var dir = Path.GetDirectoryName(fileName); if (!Directory.Exists(dir)) @@ -254,8 +268,9 @@ namespace Microsoft.Extensions.Logging.Testing var serilogger = new LoggerConfiguration() .Enrich.FromLogContext() + .Enrich.With(new AssemblyLogTimestampOffsetEnricher(logStart)) .MinimumLevel.Verbose() - .WriteTo.File(fileName, outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{SourceContext}] [{Level}] {Message}{NewLine}{Exception}", flushToDiskInterval: TimeSpan.FromSeconds(1), shared: true) + .WriteTo.File(fileName, outputTemplate: "[{TimestampOffset}] [{SourceContext}] [{Level}] {Message:l}{NewLine}{Exception}", flushToDiskInterval: TimeSpan.FromSeconds(1), shared: true) .CreateLogger(); return new SerilogLoggerProvider(serilogger, dispose: true); } @@ -287,6 +302,24 @@ namespace Microsoft.Extensions.Logging.Testing _globalLoggerFactory.Dispose(); } + private class AssemblyLogTimestampOffsetEnricher : ILogEventEnricher + { + private DateTimeOffset? _logStart; + + public AssemblyLogTimestampOffsetEnricher(DateTimeOffset? logStart) + { + _logStart = logStart; + } + + public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + => logEvent.AddPropertyIfAbsent( + propertyFactory.CreateProperty( + "TimestampOffset", + _logStart.HasValue + ? $"{(DateTimeOffset.UtcNow - _logStart.Value).TotalSeconds.ToString("N3")}s" + : DateTimeOffset.UtcNow.ToString("s"))); + } + private class Disposable : IDisposable { private Action _action; diff --git a/src/Testing/src/LoggedTest/ILoggedTest.cs b/src/Testing/src/LoggedTest/ILoggedTest.cs new file mode 100644 index 0000000000..a563cbdaf9 --- /dev/null +++ b/src/Testing/src/LoggedTest/ILoggedTest.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging.Testing +{ + public interface ILoggedTest : IDisposable + { + ILogger Logger { get; } + + ILoggerFactory LoggerFactory { get; } + + ITestOutputHelper TestOutputHelper { get; } + + // For back compat + IDisposable StartLog(out ILoggerFactory loggerFactory, LogLevel minLogLevel, string testName); + + void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper); + } +} diff --git a/src/Testing/src/LoggedTest/LoggedTest.cs b/src/Testing/src/LoggedTest/LoggedTest.cs new file mode 100644 index 0000000000..64a9adec06 --- /dev/null +++ b/src/Testing/src/LoggedTest/LoggedTest.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging.Testing +{ + public class LoggedTest : LoggedTestBase + { + // Obsolete but keeping for back compat + public LoggedTest(ITestOutputHelper output = null) : base (output) { } + + public ITestSink TestSink { get; set; } + + public override void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) + { + base.Initialize(methodInfo, testMethodArguments, testOutputHelper); + + TestSink = new TestSink(); + LoggerFactory.AddProvider(new TestLoggerProvider(TestSink)); + } + } +} diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs new file mode 100644 index 0000000000..f714a632a4 --- /dev/null +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging.Testing +{ + public class LoggedTestBase : ILoggedTest + { + private IDisposable _testLog; + + // Obsolete but keeping for back compat + public LoggedTestBase(ITestOutputHelper output = null) + { + TestOutputHelper = output; + } + + // Internal for testing + internal string ResolvedTestClassName { get; set; } + + internal RetryContext RetryContext { get; set; } + + public string ResolvedLogOutputDirectory { get; set; } + + public string ResolvedTestMethodName { get; set; } + + public ILogger Logger { get; set; } + + public ILoggerFactory LoggerFactory { get; set; } + + public ITestOutputHelper TestOutputHelper { get; set; } + + public void AddTestLogging(IServiceCollection services) => services.AddSingleton(LoggerFactory); + + // For back compat + public IDisposable StartLog(out ILoggerFactory loggerFactory, [CallerMemberName] string testName = null) => StartLog(out loggerFactory, LogLevel.Debug, testName); + + // For back compat + public IDisposable StartLog(out ILoggerFactory loggerFactory, LogLevel minLogLevel, [CallerMemberName] string testName = null) + { + return AssemblyTestLog.ForAssembly(GetType().GetTypeInfo().Assembly).StartTestLog(TestOutputHelper, GetType().FullName, out loggerFactory, minLogLevel, testName); + } + + public virtual void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) + { + TestOutputHelper = testOutputHelper; + + var classType = GetType(); + var logLevelAttribute = methodInfo.GetCustomAttribute(); + var testName = testMethodArguments.Aggregate(methodInfo.Name, (a, b) => $"{a}-{(b ?? "null")}"); + + var useShortClassName = methodInfo.DeclaringType.GetCustomAttribute() + ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); + // internal for testing + ResolvedTestClassName = useShortClassName == null ? classType.FullName : classType.Name; + + _testLog = AssemblyTestLog + .ForAssembly(classType.GetTypeInfo().Assembly) + .StartTestLog( + TestOutputHelper, + ResolvedTestClassName, + out var loggerFactory, + logLevelAttribute?.LogLevel ?? LogLevel.Debug, + out var resolvedTestName, + out var logOutputDirectory, + testName); + + ResolvedLogOutputDirectory = logOutputDirectory; + ResolvedTestMethodName = resolvedTestName; + + LoggerFactory = loggerFactory; + Logger = loggerFactory.CreateLogger(classType); + } + + public virtual void Dispose() => _testLog.Dispose(); + } +} diff --git a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs new file mode 100644 index 0000000000..32d8f30584 --- /dev/null +++ b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Extensions.Logging.Testing +{ + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] + public class TestFrameworkFileLoggerAttribute : Attribute + { + public TestFrameworkFileLoggerAttribute(string tfm, string baseDirectory = null) + { + TFM = tfm; + BaseDirectory = baseDirectory; + } + + public string TFM { get; } + public string BaseDirectory { get; } + } +} \ No newline at end of file diff --git a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props index f98e3e13b5..0d2585146c 100644 --- a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props +++ b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props @@ -1,8 +1,23 @@  - - - <_Parameter1>Microsoft.Extensions.Logging.Testing.LoggedTestFramework - <_Parameter2>Microsoft.Extensions.Logging.Testing - - + + + $(ASPNETCORE_TEST_LOG_DIR) + $(RepositoryRoot)artifacts\logs\ + + + + + + <_Parameter1>Microsoft.Extensions.Logging.Testing.LoggedTestFramework + <_Parameter2>Microsoft.Extensions.Logging.Testing + + + + <_Parameter1>$(TargetFramework) + <_Parameter2 Condition="'$(LoggingTestingDisableFileLogging)' != 'true'">$(LoggingTestingFileLoggingDirectory) + + + \ No newline at end of file diff --git a/src/Testing/test/AssemblyTestLogTests.cs b/src/Testing/test/AssemblyTestLogTests.cs index 0efadb4367..20f597defc 100644 --- a/src/Testing/test/AssemblyTestLogTests.cs +++ b/src/Testing/test/AssemblyTestLogTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading.Tasks; using Xunit; @@ -15,7 +14,9 @@ namespace Microsoft.Extensions.Logging.Testing.Tests { public class AssemblyTestLogTests : LoggedTest { - private static readonly Assembly ThisAssembly = typeof(AssemblyTestLog).GetTypeInfo().Assembly; + private static readonly Assembly ThisAssembly = typeof(AssemblyTestLogTests).GetTypeInfo().Assembly; + private static readonly string ThisAssemblyName = ThisAssembly.GetName().Name; + private static readonly string TFM = new DirectoryInfo(AppContext.BaseDirectory).Name; [Fact] public void FullClassNameUsedWhenShortClassNameAttributeNotSpecified() @@ -35,7 +36,7 @@ namespace Microsoft.Extensions.Logging.Testing.Tests public void TestLogWritesToITestOutputHelper() { var output = new TestTestOutputHelper(); - var assemblyLog = AssemblyTestLog.Create("NonExistant.Test.Assembly", baseDirectory: null); + var assemblyLog = AssemblyTestLog.Create(ThisAssemblyName, baseDirectory: null); using (assemblyLog.StartTestLog(output, "NonExistant.Test.Class", out var loggerFactory)) { @@ -46,20 +47,23 @@ namespace Microsoft.Extensions.Logging.Testing.Tests logger.LogTrace("Trace!"); } - Assert.Equal(@"[TIMESTAMP] TestLifetime Information: Starting test TestLogWritesToITestOutputHelper -[TIMESTAMP] TestLogger Information: Information! -[TIMESTAMP] TestLifetime Information: Finished test TestLogWritesToITestOutputHelper in DURATION -", MakeConsistent(output.Output), ignoreLineEndingDifferences: true); + var testLogContent = MakeConsistent(output.Output); + + Assert.Equal( +@"[OFFSET] TestLifetime Information: Starting test TestLogWritesToITestOutputHelper at TIMESTAMP +[OFFSET] TestLogger Information: Information! +[OFFSET] TestLifetime Information: Finished test TestLogWritesToITestOutputHelper in DURATION +", testLogContent, ignoreLineEndingDifferences: true); } [Fact] private Task TestLogEscapesIllegalFileNames() => RunTestLogFunctionalTest((tempDir) => { - var illegalTestName = "Testing-https://localhost:5000"; - var escapedTestName = "Testing-https_localhost_5000"; - using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", baseDirectory: tempDir)) - using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, resolvedTestName: out var resolvedTestname, testName: illegalTestName)) + var illegalTestName = "T:e/s//t"; + var escapedTestName = "T_e_s_t"; + using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssemblyName, baseDirectory: tempDir)) + using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, resolvedTestName: out var resolvedTestname, out var _, testName: illegalTestName)) { Assert.Equal(escapedTestName, resolvedTestname); } @@ -73,11 +77,11 @@ namespace Microsoft.Extensions.Logging.Testing.Tests // but it's also testing the test logging facility. So this is pretty meta ;) var logger = LoggerFactory.CreateLogger("Test"); - using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", tempDir)) + using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssemblyName, tempDir)) { logger.LogInformation("Created test log in {baseDirectory}", tempDir); - using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) + using (testAssemblyLog.StartTestLog(output: null, className: $"{ThisAssemblyName}.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) { var testLogger = testLoggerFactory.CreateLogger("TestLogger"); testLogger.LogInformation("Information!"); @@ -87,8 +91,8 @@ namespace Microsoft.Extensions.Logging.Testing.Tests logger.LogInformation("Finished test log in {baseDirectory}", tempDir); - var globalLogPath = Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "global.log"); - var testLog = Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass", $"FakeTestName.log"); + var globalLogPath = Path.Combine(tempDir, ThisAssemblyName, TFM, "global.log"); + var testLog = Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass", "FakeTestName.log"); Assert.True(File.Exists(globalLogPath), $"Expected global log file {globalLogPath} to exist"); Assert.True(File.Exists(testLog), $"Expected test log file {testLog} to exist"); @@ -96,14 +100,16 @@ namespace Microsoft.Extensions.Logging.Testing.Tests var globalLogContent = MakeConsistent(File.ReadAllText(globalLogPath)); var testLogContent = MakeConsistent(File.ReadAllText(testLog)); - Assert.Equal(@"[GlobalTestLog] [Information] Global Test Logging initialized. Set the 'ASPNETCORE_TEST_LOG_DIR' Environment Variable in order to create log files on disk. -[GlobalTestLog] [Information] Starting test ""FakeTestName"" -[GlobalTestLog] [Information] Finished test ""FakeTestName"" in DURATION + Assert.Equal( +@"[OFFSET] [GlobalTestLog] [Information] Global Test Logging initialized at TIMESTAMP. Configure the output directory via 'LoggingTestingFileLoggingDirectory' MSBuild property or set 'LoggingTestingDisableFileLogging' to 'true' to disable file logging. +[OFFSET] [GlobalTestLog] [Information] Starting test FakeTestName +[OFFSET] [GlobalTestLog] [Information] Finished test FakeTestName in DURATION ", globalLogContent, ignoreLineEndingDifferences: true); - Assert.Equal(@"[TestLifetime] [Information] Starting test ""FakeTestName"" -[TestLogger] [Information] Information! -[TestLogger] [Verbose] Trace! -[TestLifetime] [Information] Finished test ""FakeTestName"" in DURATION + Assert.Equal( +@"[OFFSET] [TestLifetime] [Information] Starting test FakeTestName at TIMESTAMP +[OFFSET] [TestLogger] [Information] Information! +[OFFSET] [TestLogger] [Verbose] Trace! +[OFFSET] [TestLifetime] [Information] Finished test FakeTestName in DURATION ", testLogContent, ignoreLineEndingDifferences: true); }); @@ -113,18 +119,18 @@ namespace Microsoft.Extensions.Logging.Testing.Tests { var longTestName = new string('0', 50) + new string('1', 50) + new string('2', 50) + new string('3', 50) + new string('4', 50); var logger = LoggerFactory.CreateLogger("Test"); - using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", tempDir)) + using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssemblyName, tempDir)) { logger.LogInformation("Created test log in {baseDirectory}", tempDir); - using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: longTestName)) + using (testAssemblyLog.StartTestLog(output: null, className: $"{ThisAssemblyName}.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: longTestName)) { testLoggerFactory.CreateLogger("TestLogger").LogInformation("Information!"); } } logger.LogInformation("Finished test log in {baseDirectory}", tempDir); - var testLogFiles = new DirectoryInfo(Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass")).EnumerateFiles(); + var testLogFiles = new DirectoryInfo(Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass")).EnumerateFiles(); var testLog = Assert.Single(testLogFiles); var testFileName = Path.GetFileNameWithoutExtension(testLog.Name); @@ -139,13 +145,13 @@ namespace Microsoft.Extensions.Logging.Testing.Tests RunTestLogFunctionalTest((tempDir) => { var logger = LoggerFactory.CreateLogger("Test"); - using (var testAssemblyLog = AssemblyTestLog.Create("FakeTestAssembly", tempDir)) + using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssemblyName, tempDir)) { logger.LogInformation("Created test log in {baseDirectory}", tempDir); for (var i = 0; i < 10; i++) { - using (testAssemblyLog.StartTestLog(output: null, className: "FakeTestAssembly.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) + using (testAssemblyLog.StartTestLog(output: null, className: $"{ThisAssemblyName}.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) { testLoggerFactory.CreateLogger("TestLogger").LogInformation("Information!"); } @@ -154,16 +160,17 @@ namespace Microsoft.Extensions.Logging.Testing.Tests logger.LogInformation("Finished test log in {baseDirectory}", tempDir); // The first log file exists - Assert.True(File.Exists(Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass", $"FakeTestName.log"))); + Assert.True(File.Exists(Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass", "FakeTestName.log"))); // Subsequent files exist for (var i = 0; i < 9; i++) { - Assert.True(File.Exists(Path.Combine(tempDir, "FakeTestAssembly", RuntimeInformation.FrameworkDescription.TrimStart('.'), "FakeTestClass", $"FakeTestName.{i}.log"))); + Assert.True(File.Exists(Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass", $"FakeTestName.{i}.log"))); } }); private static readonly Regex TimestampRegex = new Regex(@"\d+-\d+-\d+T\d+:\d+:\d+"); + private static readonly Regex TimestampOffsetRegex = new Regex(@"\d+\.\d+s"); private static readonly Regex DurationRegex = new Regex(@"[^ ]+s$"); private async Task RunTestLogFunctionalTest(Action action, [CallerMemberName] string testName = null) @@ -197,10 +204,10 @@ namespace Microsoft.Extensions.Logging.Testing.Tests { var strippedPrefix = line.IndexOf("[") >= 0 ? line.Substring(line.IndexOf("[")) : line; - var strippedDuration = - DurationRegex.Replace(strippedPrefix, "DURATION"); + var strippedDuration = DurationRegex.Replace(strippedPrefix, "DURATION"); var strippedTimestamp = TimestampRegex.Replace(strippedDuration, "TIMESTAMP"); - return strippedTimestamp; + var strippedTimestampOffset = TimestampOffsetRegex.Replace(strippedTimestamp, "OFFSET"); + return strippedTimestampOffset; })); } } diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs index 31fd6d631f..d1d8581193 100644 --- a/src/Testing/test/LoggedTestXunitTests.cs +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Reflection; using Microsoft.AspNetCore.Testing.xunit; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -130,12 +131,29 @@ namespace Microsoft.Extensions.Logging.Testing.Tests } } + public class LoggedTestXunitInitializationTests : TestLoggedTest + { + [Fact] + public void ITestOutputHelperInitializedByDefault() + { + Assert.True(ITestOutputHelperIsInitialized); + } + } + public class TestLoggedTest : LoggedTest { public bool SetupInvoked { get; private set; } = false; + public bool ITestOutputHelperIsInitialized { get; private set; } = false; - public override void AdditionalSetup() + public override void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) { + base.Initialize(methodInfo, testMethodArguments, testOutputHelper); + + try + { + TestOutputHelper.WriteLine("Test"); + ITestOutputHelperIsInitialized = true; + } catch { } SetupInvoked = true; } } From e7cca176e589b9102cf43365f2bca3d356fb29aa Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 6 Nov 2018 16:58:30 -0800 Subject: [PATCH 03/13] Reorganize source code in preparation to move into aspnet/Extensions Prior to reorganization, this source code was found in https://github.com/aspnet/Logging/tree/f7d8e4e0537eaab54dcf28c2b148b82688a3d62d --- src/Testing/src/LoggedTest/LoggedTestBase.cs | 4 +- src/Testing/test/LoggedTestXunitTests.cs | 46 +++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs index f714a632a4..c3a4e78931 100644 --- a/src/Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -51,7 +51,9 @@ namespace Microsoft.Extensions.Logging.Testing TestOutputHelper = testOutputHelper; var classType = GetType(); - var logLevelAttribute = methodInfo.GetCustomAttribute(); + var logLevelAttribute = methodInfo.GetCustomAttribute() + ?? methodInfo.DeclaringType.GetCustomAttribute() + ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); var testName = testMethodArguments.Aggregate(methodInfo.Name, (a, b) => $"{a}-{(b ?? "null")}"); var useShortClassName = methodInfo.DeclaringType.GetCustomAttribute() diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs index d1d8581193..507453a242 100644 --- a/src/Testing/test/LoggedTestXunitTests.cs +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Linq; using System.Reflection; using Microsoft.AspNetCore.Testing.xunit; using Microsoft.Extensions.DependencyInjection; @@ -9,6 +10,7 @@ using Xunit.Abstractions; namespace Microsoft.Extensions.Logging.Testing.Tests { + [LogLevel(LogLevel.Debug)] [ShortClassName] public class LoggedTestXunitTests : TestLoggedTest { @@ -75,7 +77,7 @@ namespace Microsoft.Extensions.Logging.Testing.Tests [Fact] [LogLevel(LogLevel.Information)] - public void LoggedFactFilteredByLogLevel() + public void LoggedFactFilteredByMethodLogLevel() { Logger.LogInformation("Information"); Logger.LogDebug("Debug"); @@ -85,6 +87,17 @@ namespace Microsoft.Extensions.Logging.Testing.Tests Assert.Equal("Information", message.Formatter(message.State, null)); } + [Fact] + public void LoggedFactFilteredByClassLogLevel() + { + Logger.LogDebug("Debug"); + Logger.LogTrace("Trace"); + + var message = Assert.Single(TestSink.Writes); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal("Debug", message.Formatter(message.State, null)); + } + [Theory] [InlineData("Hello world")] [LogLevel(LogLevel.Information)] @@ -129,6 +142,37 @@ namespace Microsoft.Extensions.Logging.Testing.Tests { Assert.True(SetupInvoked); } + + [Fact] + public void MessageWrittenEventInvoked() + { + WriteContext context = null; + TestSink.MessageLogged += ctx => context = ctx; + Logger.LogInformation("Information"); + Assert.Equal(TestSink.Writes.Single(), context); + } + + [Fact] + public void ScopeStartedEventInvoked() + { + BeginScopeContext context = null; + TestSink.ScopeStarted += ctx => context = ctx; + using (Logger.BeginScope("Scope")) {} + Assert.Equal(TestSink.Scopes.Single(), context); + } + } + + public class LoggedTestXunitLogLevelTests : LoggedTest + { + [Fact] + public void LoggedFactFilteredByAssemblyLogLevel() + { + Logger.LogTrace("Trace"); + + var message = Assert.Single(TestSink.Writes); + Assert.Equal(LogLevel.Trace, message.LogLevel); + Assert.Equal("Trace", message.Formatter(message.State, null)); + } } public class LoggedTestXunitInitializationTests : TestLoggedTest From 688914bb70063b97dcb2facdbc1ab91980406ece Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Mon, 17 Dec 2018 09:18:26 -0800 Subject: [PATCH 04/13] Capture LoggedTest.Initialize exception and re-trow in Dispose (#770) --- src/Testing/src/LoggedTest/LoggedTestBase.cs | 64 ++++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs index c3a4e78931..72de6a87c3 100644 --- a/src/Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -5,6 +5,7 @@ using System; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.DependencyInjection; using Xunit.Abstractions; @@ -12,6 +13,8 @@ namespace Microsoft.Extensions.Logging.Testing { public class LoggedTestBase : ILoggedTest { + private ExceptionDispatchInfo _initializationException; + private IDisposable _testLog; // Obsolete but keeping for back compat @@ -48,37 +51,48 @@ namespace Microsoft.Extensions.Logging.Testing public virtual void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) { - TestOutputHelper = testOutputHelper; + try + { + TestOutputHelper = testOutputHelper; - var classType = GetType(); - var logLevelAttribute = methodInfo.GetCustomAttribute() - ?? methodInfo.DeclaringType.GetCustomAttribute() - ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); - var testName = testMethodArguments.Aggregate(methodInfo.Name, (a, b) => $"{a}-{(b ?? "null")}"); + var classType = GetType(); + var logLevelAttribute = methodInfo.GetCustomAttribute() + ?? methodInfo.DeclaringType.GetCustomAttribute() + ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); + var testName = testMethodArguments.Aggregate(methodInfo.Name, (a, b) => $"{a}-{(b ?? "null")}"); - var useShortClassName = methodInfo.DeclaringType.GetCustomAttribute() - ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); - // internal for testing - ResolvedTestClassName = useShortClassName == null ? classType.FullName : classType.Name; + var useShortClassName = methodInfo.DeclaringType.GetCustomAttribute() + ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); + // internal for testing + ResolvedTestClassName = useShortClassName == null ? classType.FullName : classType.Name; - _testLog = AssemblyTestLog - .ForAssembly(classType.GetTypeInfo().Assembly) - .StartTestLog( - TestOutputHelper, - ResolvedTestClassName, - out var loggerFactory, - logLevelAttribute?.LogLevel ?? LogLevel.Debug, - out var resolvedTestName, - out var logOutputDirectory, - testName); + _testLog = AssemblyTestLog + .ForAssembly(classType.GetTypeInfo().Assembly) + .StartTestLog( + TestOutputHelper, + ResolvedTestClassName, + out var loggerFactory, + logLevelAttribute?.LogLevel ?? LogLevel.Debug, + out var resolvedTestName, + out var logOutputDirectory, + testName); - ResolvedLogOutputDirectory = logOutputDirectory; - ResolvedTestMethodName = resolvedTestName; + ResolvedLogOutputDirectory = logOutputDirectory; + ResolvedTestMethodName = resolvedTestName; - LoggerFactory = loggerFactory; - Logger = loggerFactory.CreateLogger(classType); + LoggerFactory = loggerFactory; + Logger = loggerFactory.CreateLogger(classType); + } + catch (Exception e) + { + _initializationException = ExceptionDispatchInfo.Capture(e); + } } - public virtual void Dispose() => _testLog.Dispose(); + public virtual void Dispose() + { + _initializationException?.Throw(); + _testLog.Dispose(); + } } } From b51be2c25045ba50909822f971425da764fed2fd Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 7 Jan 2019 13:42:22 -0500 Subject: [PATCH 05/13] Add attribute to allow LoggedTest to collect dump on failure (#905) * Add attribute for collecting dumps for failed LoggedTest --- src/Testing/src/CollectDumpAttribute.cs | 18 +++++ .../DumpCollector/DumpCollector.Windows.cs | 76 +++++++++++++++++++ .../src/DumpCollector/DumpCollector.cs | 20 +++++ 3 files changed, 114 insertions(+) create mode 100644 src/Testing/src/CollectDumpAttribute.cs create mode 100644 src/Testing/src/DumpCollector/DumpCollector.Windows.cs create mode 100644 src/Testing/src/DumpCollector/DumpCollector.cs diff --git a/src/Testing/src/CollectDumpAttribute.cs b/src/Testing/src/CollectDumpAttribute.cs new file mode 100644 index 0000000000..8a6aa84bac --- /dev/null +++ b/src/Testing/src/CollectDumpAttribute.cs @@ -0,0 +1,18 @@ +// Copyright(c) .NET Foundation.All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Extensions.Logging.Testing +{ + /// + /// Capture the memory dump upon test failure. + /// + /// + /// This currently only works in Windows environments + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class CollectDumpAttribute : Attribute + { + } +} diff --git a/src/Testing/src/DumpCollector/DumpCollector.Windows.cs b/src/Testing/src/DumpCollector/DumpCollector.Windows.cs new file mode 100644 index 0000000000..8d4168c20c --- /dev/null +++ b/src/Testing/src/DumpCollector/DumpCollector.Windows.cs @@ -0,0 +1,76 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; + +namespace Microsoft.Extensions.Logging.Testing +{ + public static partial class DumpCollector + { + private static class Windows + { + internal static void Collect(Process process, string outputFile) + { + // Open the file for writing + using (var stream = new FileStream(outputFile, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + { + // Dump the process! + var exceptionInfo = new NativeMethods.MINIDUMP_EXCEPTION_INFORMATION(); + if (!NativeMethods.MiniDumpWriteDump(process.Handle, (uint)process.Id, stream.SafeFileHandle, NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemory, ref exceptionInfo, IntPtr.Zero, IntPtr.Zero)) + { + var err = Marshal.GetHRForLastWin32Error(); + Marshal.ThrowExceptionForHR(err); + } + } + } + + private static class NativeMethods + { + [DllImport("Dbghelp.dll")] + public static extern bool MiniDumpWriteDump(IntPtr hProcess, uint ProcessId, SafeFileHandle hFile, MINIDUMP_TYPE DumpType, ref MINIDUMP_EXCEPTION_INFORMATION ExceptionParam, IntPtr UserStreamParam, IntPtr CallbackParam); + + [StructLayout(LayoutKind.Sequential, Pack = 4)] + public struct MINIDUMP_EXCEPTION_INFORMATION + { + public uint ThreadId; + public IntPtr ExceptionPointers; + public int ClientPointers; + } + + [Flags] + public enum MINIDUMP_TYPE : uint + { + MiniDumpNormal = 0, + MiniDumpWithDataSegs = 1 << 0, + MiniDumpWithFullMemory = 1 << 1, + MiniDumpWithHandleData = 1 << 2, + MiniDumpFilterMemory = 1 << 3, + MiniDumpScanMemory = 1 << 4, + MiniDumpWithUnloadedModules = 1 << 5, + MiniDumpWithIndirectlyReferencedMemory = 1 << 6, + MiniDumpFilterModulePaths = 1 << 7, + MiniDumpWithProcessThreadData = 1 << 8, + MiniDumpWithPrivateReadWriteMemory = 1 << 9, + MiniDumpWithoutOptionalData = 1 << 10, + MiniDumpWithFullMemoryInfo = 1 << 11, + MiniDumpWithThreadInfo = 1 << 12, + MiniDumpWithCodeSegs = 1 << 13, + MiniDumpWithoutAuxiliaryState = 1 << 14, + MiniDumpWithFullAuxiliaryState = 1 << 15, + MiniDumpWithPrivateWriteCopyMemory = 1 << 16, + MiniDumpIgnoreInaccessibleMemory = 1 << 17, + MiniDumpWithTokenInformation = 1 << 18, + MiniDumpWithModuleHeaders = 1 << 19, + MiniDumpFilterTriage = 1 << 20, + MiniDumpWithAvxXStateContext = 1 << 21, + MiniDumpWithIptTrace = 1 << 22, + MiniDumpValidTypeFlags = (-1) ^ ((~1) << 22) + } + } + } + } +} diff --git a/src/Testing/src/DumpCollector/DumpCollector.cs b/src/Testing/src/DumpCollector/DumpCollector.cs new file mode 100644 index 0000000000..67043ed827 --- /dev/null +++ b/src/Testing/src/DumpCollector/DumpCollector.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace Microsoft.Extensions.Logging.Testing +{ + public static partial class DumpCollector + { + public static void Collect(Process process, string fileName) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Windows.Collect(process, fileName); + } + // No implementations yet for macOS and Linux + } + } +} From c43d3b823e37de499c3113351575ed47366638d6 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 29 Jan 2019 18:34:54 -0800 Subject: [PATCH 06/13] Cleanup conversion to Arcade (#1014) * Remove obsolete targets, properties, and scripts * Replace IsProductComponent with IsShipping * Undo bad merge to version.props * Update documentation, and put workarounds into a common file * Replace usages of RepositoryRoot with RepoRoot * Remove API baselines * Remove unnecessary restore feeds and split workarounds into two files * Enable PR checks on all branches, and disable autocancel --- .../src/build/Microsoft.Extensions.Logging.Testing.props | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props index 0d2585146c..c503c32d40 100644 --- a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props +++ b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props @@ -1,8 +1,9 @@  + $(RepositoryRoot) $(ASPNETCORE_TEST_LOG_DIR) - $(RepositoryRoot)artifacts\logs\ + $(RepoRoot)artifacts\log\ - \ No newline at end of file + From 262262569a3d6f204330ce38b801ede9b7bf77d0 Mon Sep 17 00:00:00 2001 From: Andrew Stanton-Nurse Date: Wed, 6 Mar 2019 15:19:11 -0800 Subject: [PATCH 07/13] add FlakyAttribute to mark flaky tests (#1222) part of aspnet/AspNetCore#8237 --- src/Testing/src/LoggedTest/LoggedTestBase.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs index 72de6a87c3..492de61cb6 100644 --- a/src/Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -91,6 +91,14 @@ namespace Microsoft.Extensions.Logging.Testing public virtual void Dispose() { + if(_testLog == null) + { + // It seems like sometimes the MSBuild goop that adds the test framework can end up in a bad state and not actually add it + // Not sure yet why that happens but the exception isn't clear so I'm adding this error so we can detect it better. + // -anurse + throw new InvalidOperationException("LoggedTest base class was used but nothing initialized it! The test framework may not be enabled. Try cleaning your 'obj' directory."); + } + _initializationException?.Throw(); _testLog.Dispose(); } From f57e591af3e3fd0500fb1e51b5be8a4fb5a3562a Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 9 Apr 2019 14:03:12 -0700 Subject: [PATCH 08/13] Add Repeat attribute (#1375) --- src/Testing/src/LoggedTest/LoggedTestBase.cs | 2 +- src/Testing/test/LoggedTestXunitTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs index 492de61cb6..94cdf82257 100644 --- a/src/Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -26,7 +26,7 @@ namespace Microsoft.Extensions.Logging.Testing // Internal for testing internal string ResolvedTestClassName { get; set; } - internal RetryContext RetryContext { get; set; } + internal RepeatContext RepeatContext { get; set; } public string ResolvedLogOutputDirectory { get; set; } diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs index 507453a242..520ffaaa9e 100644 --- a/src/Testing/test/LoggedTestXunitTests.cs +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Linq; From ce392fa4f7a2e6b8eb186d64e0e6eb7d07c3a2c4 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 15 Aug 2019 09:12:53 -0700 Subject: [PATCH 09/13] Cleanup to skip/flaky attributes (#2186) --- src/Testing/test/LoggedTestXunitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs index 520ffaaa9e..ab9ee746c3 100644 --- a/src/Testing/test/LoggedTestXunitTests.cs +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Reflection; -using Microsoft.AspNetCore.Testing.xunit; +using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.DependencyInjection; using Xunit; using Xunit.Abstractions; From 221985c254773e10da453c4d4b1381677b275a71 Mon Sep 17 00:00:00 2001 From: Ryan Nowak Date: Mon, 16 Sep 2019 13:33:09 -0700 Subject: [PATCH 10/13] Refactor xUnit extensibility Adds our own hook for before/after logic that's more usable, called `ITestMethodLifecycle`. This provides access to a context object including the information about the test and the output helper. This can be implemented by attributes or by the class itself. The goal (and result) of this, is that we have a single *test executor* extensibility point that provides all of the features we need. We should use this everywhere we need features xUnit doesn't have. Adding a new extensibility point (`ITestMethodLifecycle`) allows us to do this without turning all of these features into a giant monolith. --- Also updated our existing extensibility to use this new hook. I did as much cleanup as a could to remove duplication from logging and keep it loosly coupled. I didn't want to tease this apart completely because the scope of this PR is already pretty large. --- src/Testing/src/AssemblyTestLog.cs | 55 ++++--------------- src/Testing/src/CollectDumpAttribute.cs | 23 +++++++- src/Testing/src/LoggedTest/ILoggedTest.cs | 3 +- src/Testing/src/LoggedTest/LoggedTest.cs | 5 +- src/Testing/src/LoggedTest/LoggedTestBase.cs | 41 +++++++++----- .../src/TestFrameworkFileLoggerAttribute.cs | 13 ++--- ...Microsoft.Extensions.Logging.Testing.props | 4 +- src/Testing/test/AssemblyTestLogTests.cs | 8 +-- src/Testing/test/LoggedTestXunitTests.cs | 16 +----- 9 files changed, 76 insertions(+), 92 deletions(-) diff --git a/src/Testing/src/AssemblyTestLog.cs b/src/Testing/src/AssemblyTestLog.cs index e84df52554..3c598b67d5 100644 --- a/src/Testing/src/AssemblyTestLog.cs +++ b/src/Testing/src/AssemblyTestLog.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; +using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Core; @@ -23,14 +24,6 @@ namespace Microsoft.Extensions.Logging.Testing private static readonly string MaxPathLengthEnvironmentVariableName = "ASPNETCORE_TEST_LOG_MAXPATH"; private static readonly string LogFileExtension = ".log"; private static readonly int MaxPathLength = GetMaxPathLength(); - private static char[] InvalidFileChars = new char[] - { - '\"', '<', '>', '|', '\0', - (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10, - (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20, - (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30, - (char)31, ':', '*', '?', '\\', '/', ' ', (char)127 - }; private static readonly object _lock = new object(); private static readonly Dictionary _logs = new Dictionary(); @@ -113,8 +106,8 @@ namespace Microsoft.Extensions.Logging.Testing SerilogLoggerProvider serilogLoggerProvider = null; if (!string.IsNullOrEmpty(_baseDirectory)) { - logOutputDirectory = Path.Combine(GetAssemblyBaseDirectory(_baseDirectory, _assembly), className); - testName = RemoveIllegalFileChars(testName); + logOutputDirectory = Path.Combine(_baseDirectory, className); + testName = TestFileOutputContext.RemoveIllegalFileChars(testName); if (logOutputDirectory.Length + testName.Length + LogFileExtension.Length >= MaxPathLength) { @@ -184,10 +177,10 @@ namespace Microsoft.Extensions.Logging.Testing { var logStart = DateTimeOffset.UtcNow; SerilogLoggerProvider serilogLoggerProvider = null; - var globalLogDirectory = GetAssemblyBaseDirectory(baseDirectory, assembly); - if (!string.IsNullOrEmpty(globalLogDirectory)) + if (!string.IsNullOrEmpty(baseDirectory)) { - var globalLogFileName = Path.Combine(globalLogDirectory, "global.log"); + baseDirectory = TestFileOutputContext.GetAssemblyBaseDirectory(assembly, baseDirectory); + var globalLogFileName = Path.Combine(baseDirectory, "global.log"); serilogLoggerProvider = ConfigureFileLogging(globalLogFileName, logStart); } @@ -222,31 +215,26 @@ namespace Microsoft.Extensions.Logging.Testing { if (!_logs.TryGetValue(assembly, out var log)) { - var baseDirectory = GetFileLoggerAttribute(assembly).BaseDirectory; + var baseDirectory = TestFileOutputContext.GetOutputDirectory(assembly); log = Create(assembly, baseDirectory); _logs[assembly] = log; - // Try to clear previous logs - var assemblyBaseDirectory = GetAssemblyBaseDirectory(baseDirectory, assembly); - if (Directory.Exists(assemblyBaseDirectory)) + // Try to clear previous logs, continue if it fails. + var assemblyBaseDirectory = TestFileOutputContext.GetAssemblyBaseDirectory(assembly); + if (!string.IsNullOrEmpty(assemblyBaseDirectory)) { try { Directory.Delete(assemblyBaseDirectory, recursive: true); } - catch {} + catch { } } } return log; } } - private static string GetAssemblyBaseDirectory(string baseDirectory, Assembly assembly) - => string.IsNullOrEmpty(baseDirectory) - ? string.Empty - : Path.Combine(baseDirectory, assembly.GetName().Name, GetFileLoggerAttribute(assembly).TFM); - private static TestFrameworkFileLoggerAttribute GetFileLoggerAttribute(Assembly assembly) => assembly.GetCustomAttribute() ?? throw new InvalidOperationException($"No {nameof(TestFrameworkFileLoggerAttribute)} found on the assembly {assembly.GetName().Name}. " @@ -275,27 +263,6 @@ namespace Microsoft.Extensions.Logging.Testing return new SerilogLoggerProvider(serilogger, dispose: true); } - private static string RemoveIllegalFileChars(string s) - { - var sb = new StringBuilder(); - - foreach (var c in s) - { - if (InvalidFileChars.Contains(c)) - { - if (sb.Length > 0 && sb[sb.Length - 1] != '_') - { - sb.Append('_'); - } - } - else - { - sb.Append(c); - } - } - return sb.ToString(); - } - public void Dispose() { (_serviceProvider as IDisposable)?.Dispose(); diff --git a/src/Testing/src/CollectDumpAttribute.cs b/src/Testing/src/CollectDumpAttribute.cs index 8a6aa84bac..012a5c8fa1 100644 --- a/src/Testing/src/CollectDumpAttribute.cs +++ b/src/Testing/src/CollectDumpAttribute.cs @@ -2,6 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing; namespace Microsoft.Extensions.Logging.Testing { @@ -12,7 +17,23 @@ namespace Microsoft.Extensions.Logging.Testing /// This currently only works in Windows environments /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class CollectDumpAttribute : Attribute + public class CollectDumpAttribute : Attribute, ITestMethodLifecycle { + public Task OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public Task OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + if (exception != null) + { + var path = Path.Combine(context.FileOutput.TestClassOutputDirectory, context.FileOutput.GetUniqueFileName(context.FileOutput.TestName, ".dmp")); + var process = Process.GetCurrentProcess(); + DumpCollector.Collect(process, path); + } + + return Task.CompletedTask; + } } } diff --git a/src/Testing/src/LoggedTest/ILoggedTest.cs b/src/Testing/src/LoggedTest/ILoggedTest.cs index a563cbdaf9..a906ae84a2 100644 --- a/src/Testing/src/LoggedTest/ILoggedTest.cs +++ b/src/Testing/src/LoggedTest/ILoggedTest.cs @@ -3,6 +3,7 @@ using System; using System.Reflection; +using Microsoft.AspNetCore.Testing; using Xunit.Abstractions; namespace Microsoft.Extensions.Logging.Testing @@ -18,6 +19,6 @@ namespace Microsoft.Extensions.Logging.Testing // For back compat IDisposable StartLog(out ILoggerFactory loggerFactory, LogLevel minLogLevel, string testName); - void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper); + void Initialize(TestContext context, MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper); } } diff --git a/src/Testing/src/LoggedTest/LoggedTest.cs b/src/Testing/src/LoggedTest/LoggedTest.cs index 64a9adec06..d108ffb7e8 100644 --- a/src/Testing/src/LoggedTest/LoggedTest.cs +++ b/src/Testing/src/LoggedTest/LoggedTest.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Reflection; +using Microsoft.AspNetCore.Testing; using Xunit.Abstractions; namespace Microsoft.Extensions.Logging.Testing @@ -13,9 +14,9 @@ namespace Microsoft.Extensions.Logging.Testing public ITestSink TestSink { get; set; } - public override void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) + public override void Initialize(TestContext context, MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) { - base.Initialize(methodInfo, testMethodArguments, testOutputHelper); + base.Initialize(context, methodInfo, testMethodArguments, testOutputHelper); TestSink = new TestSink(); LoggerFactory.AddProvider(new TestLoggerProvider(TestSink)); diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs index 94cdf82257..324b855319 100644 --- a/src/Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -6,12 +6,16 @@ using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.DependencyInjection; +using Serilog; using Xunit.Abstractions; namespace Microsoft.Extensions.Logging.Testing { - public class LoggedTestBase : ILoggedTest + public class LoggedTestBase : ILoggedTest, ITestMethodLifecycle { private ExceptionDispatchInfo _initializationException; @@ -23,11 +27,11 @@ namespace Microsoft.Extensions.Logging.Testing TestOutputHelper = output; } + protected TestContext Context { get; private set; } + // Internal for testing internal string ResolvedTestClassName { get; set; } - internal RepeatContext RepeatContext { get; set; } - public string ResolvedLogOutputDirectory { get; set; } public string ResolvedTestMethodName { get; set; } @@ -49,7 +53,7 @@ namespace Microsoft.Extensions.Logging.Testing return AssemblyTestLog.ForAssembly(GetType().GetTypeInfo().Assembly).StartTestLog(TestOutputHelper, GetType().FullName, out loggerFactory, minLogLevel, testName); } - public virtual void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) + public virtual void Initialize(TestContext context, MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) { try { @@ -59,25 +63,22 @@ namespace Microsoft.Extensions.Logging.Testing var logLevelAttribute = methodInfo.GetCustomAttribute() ?? methodInfo.DeclaringType.GetCustomAttribute() ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); - var testName = testMethodArguments.Aggregate(methodInfo.Name, (a, b) => $"{a}-{(b ?? "null")}"); - var useShortClassName = methodInfo.DeclaringType.GetCustomAttribute() - ?? methodInfo.DeclaringType.Assembly.GetCustomAttribute(); // internal for testing - ResolvedTestClassName = useShortClassName == null ? classType.FullName : classType.Name; + ResolvedTestClassName = context.FileOutput.TestClassName; _testLog = AssemblyTestLog .ForAssembly(classType.GetTypeInfo().Assembly) .StartTestLog( TestOutputHelper, - ResolvedTestClassName, + context.FileOutput.TestClassName, out var loggerFactory, logLevelAttribute?.LogLevel ?? LogLevel.Debug, out var resolvedTestName, - out var logOutputDirectory, - testName); + out var logDirectory, + context.FileOutput.TestName); - ResolvedLogOutputDirectory = logOutputDirectory; + ResolvedLogOutputDirectory = logDirectory; ResolvedTestMethodName = resolvedTestName; LoggerFactory = loggerFactory; @@ -91,7 +92,7 @@ namespace Microsoft.Extensions.Logging.Testing public virtual void Dispose() { - if(_testLog == null) + if (_testLog == null) { // It seems like sometimes the MSBuild goop that adds the test framework can end up in a bad state and not actually add it // Not sure yet why that happens but the exception isn't clear so I'm adding this error so we can detect it better. @@ -102,5 +103,19 @@ namespace Microsoft.Extensions.Logging.Testing _initializationException?.Throw(); _testLog.Dispose(); } + + Task ITestMethodLifecycle.OnTestStartAsync(TestContext context, CancellationToken cancellationToken) + { + + Context = context; + + Initialize(context, context.TestMethod, context.MethodArguments, context.Output); + return Task.CompletedTask; + } + + Task ITestMethodLifecycle.OnTestEndAsync(TestContext context, Exception exception, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } } } diff --git a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs index 32d8f30584..025a5a9bd8 100644 --- a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs +++ b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs @@ -1,20 +1,17 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using Microsoft.AspNetCore.Testing; namespace Microsoft.Extensions.Logging.Testing { [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] - public class TestFrameworkFileLoggerAttribute : Attribute + public class TestFrameworkFileLoggerAttribute : TestOutputDirectoryAttribute { public TestFrameworkFileLoggerAttribute(string tfm, string baseDirectory = null) + : base(tfm, baseDirectory) { - TFM = tfm; - BaseDirectory = baseDirectory; } - - public string TFM { get; } - public string BaseDirectory { get; } } -} \ No newline at end of file +} diff --git a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props index c503c32d40..3895cb5471 100644 --- a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props +++ b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props @@ -11,8 +11,8 @@ Condition="'$(GenerateLoggingTestingAssemblyAttributes)' != 'false'"> - <_Parameter1>Microsoft.Extensions.Logging.Testing.LoggedTestFramework - <_Parameter2>Microsoft.Extensions.Logging.Testing + <_Parameter1>Microsoft.AspNetCore.Testing.AspNetTestFramework + <_Parameter2>Microsoft.AspNetCore.Testing diff --git a/src/Testing/test/AssemblyTestLogTests.cs b/src/Testing/test/AssemblyTestLogTests.cs index 20f597defc..6d7ae5139a 100644 --- a/src/Testing/test/AssemblyTestLogTests.cs +++ b/src/Testing/test/AssemblyTestLogTests.cs @@ -18,12 +18,6 @@ namespace Microsoft.Extensions.Logging.Testing.Tests private static readonly string ThisAssemblyName = ThisAssembly.GetName().Name; private static readonly string TFM = new DirectoryInfo(AppContext.BaseDirectory).Name; - [Fact] - public void FullClassNameUsedWhenShortClassNameAttributeNotSpecified() - { - Assert.Equal(GetType().FullName, ResolvedTestClassName); - } - [Fact] public void ForAssembly_ReturnsSameInstanceForSameAssembly() { @@ -57,7 +51,7 @@ namespace Microsoft.Extensions.Logging.Testing.Tests } [Fact] - private Task TestLogEscapesIllegalFileNames() => + public Task TestLogEscapesIllegalFileNames() => RunTestLogFunctionalTest((tempDir) => { var illegalTestName = "T:e/s//t"; diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs index ab9ee746c3..d8454023a2 100644 --- a/src/Testing/test/LoggedTestXunitTests.cs +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -21,18 +21,6 @@ namespace Microsoft.Extensions.Logging.Testing.Tests _output = output; } - [Fact] - public void ShortClassNameUsedWhenShortClassNameAttributeSpecified() - { - Assert.Equal(GetType().Name, ResolvedTestClassName); - } - - [Fact] - public void LoggedTestTestOutputHelperSameInstanceAsInjectedConstructorArg() - { - Assert.Same(_output, TestOutputHelper); - } - [Fact] public void LoggedFactInitializesLoggedTestProperties() { @@ -189,9 +177,9 @@ namespace Microsoft.Extensions.Logging.Testing.Tests public bool SetupInvoked { get; private set; } = false; public bool ITestOutputHelperIsInitialized { get; private set; } = false; - public override void Initialize(MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) + public override void Initialize(TestContext context, MethodInfo methodInfo, object[] testMethodArguments, ITestOutputHelper testOutputHelper) { - base.Initialize(methodInfo, testMethodArguments, testOutputHelper); + base.Initialize(context, methodInfo, testMethodArguments, testOutputHelper); try { From b6e5fd86713f1a96f55027e76ca54897bf28aa2a Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 6 Jan 2020 11:40:44 -0800 Subject: [PATCH 11/13] Preserve functional test logs on CI (#2819) * Add option to preserve function test logs * Upload test logs as artifacts * Preserve binlogs * Add target to ensure all functional test logs preserved --- src/Testing/src/AssemblyTestLog.cs | 2 +- src/Testing/src/TestFrameworkFileLoggerAttribute.cs | 4 ++-- .../build/Microsoft.Extensions.Logging.Testing.props | 10 ++++++++-- src/Testing/test/AssemblyTestLogTests.cs | 12 ++++++++++++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/Testing/src/AssemblyTestLog.cs b/src/Testing/src/AssemblyTestLog.cs index 3c598b67d5..611b853fac 100644 --- a/src/Testing/src/AssemblyTestLog.cs +++ b/src/Testing/src/AssemblyTestLog.cs @@ -222,7 +222,7 @@ namespace Microsoft.Extensions.Logging.Testing // Try to clear previous logs, continue if it fails. var assemblyBaseDirectory = TestFileOutputContext.GetAssemblyBaseDirectory(assembly); - if (!string.IsNullOrEmpty(assemblyBaseDirectory)) + if (!string.IsNullOrEmpty(assemblyBaseDirectory) && !TestFileOutputContext.GetPreserveExistingLogsInOutput(assembly)) { try { diff --git a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs index 025a5a9bd8..1059fa76f2 100644 --- a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs +++ b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs @@ -9,8 +9,8 @@ namespace Microsoft.Extensions.Logging.Testing [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] public class TestFrameworkFileLoggerAttribute : TestOutputDirectoryAttribute { - public TestFrameworkFileLoggerAttribute(string tfm, string baseDirectory = null) - : base(tfm, baseDirectory) + public TestFrameworkFileLoggerAttribute(string preserveExistingLogsInOutput, string tfm, string baseDirectory = null) + : base(preserveExistingLogsInOutput, tfm, baseDirectory) { } } diff --git a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props index 3895cb5471..167efb3f82 100644 --- a/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props +++ b/src/Testing/src/build/Microsoft.Extensions.Logging.Testing.props @@ -9,6 +9,11 @@ + + true + false + + <_Parameter1>Microsoft.AspNetCore.Testing.AspNetTestFramework @@ -16,8 +21,9 @@ - <_Parameter1>$(TargetFramework) - <_Parameter2 Condition="'$(LoggingTestingDisableFileLogging)' != 'true'">$(LoggingTestingFileLoggingDirectory) + <_Parameter1>$(PreserveExistingLogsInOutput) + <_Parameter2>$(TargetFramework) + <_Parameter3 Condition="'$(LoggingTestingDisableFileLogging)' != 'true'">$(LoggingTestingFileLoggingDirectory) diff --git a/src/Testing/test/AssemblyTestLogTests.cs b/src/Testing/test/AssemblyTestLogTests.cs index 6d7ae5139a..dbefa4ccd2 100644 --- a/src/Testing/test/AssemblyTestLogTests.cs +++ b/src/Testing/test/AssemblyTestLogTests.cs @@ -8,6 +8,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing; using Xunit; namespace Microsoft.Extensions.Logging.Testing.Tests @@ -18,6 +19,17 @@ namespace Microsoft.Extensions.Logging.Testing.Tests private static readonly string ThisAssemblyName = ThisAssembly.GetName().Name; private static readonly string TFM = new DirectoryInfo(AppContext.BaseDirectory).Name; + [Fact] + public void FunctionalLogs_LogsPreservedFromNonFlakyRun() + { + } + + [Fact] + [Flaky("http://example.com", FlakyOn.All)] + public void FunctionalLogs_LogsPreservedFromFlakyRun() + { + } + [Fact] public void ForAssembly_ReturnsSameInstanceForSameAssembly() { From 6cb7b318ef43ef682cacda6d48a8ba740064d4bb Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Wed, 26 Feb 2020 10:27:22 -0800 Subject: [PATCH 12/13] Normalize all file headers to the expected Apache 2.0 license --- src/Testing/src/CollectDumpAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Testing/src/CollectDumpAttribute.cs b/src/Testing/src/CollectDumpAttribute.cs index 012a5c8fa1..c9e8ee3fa6 100644 --- a/src/Testing/src/CollectDumpAttribute.cs +++ b/src/Testing/src/CollectDumpAttribute.cs @@ -1,4 +1,4 @@ -// Copyright(c) .NET Foundation.All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; From e3b0fdf81cd001a1940b05d1c8c644050d7629f0 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Wed, 26 Feb 2020 10:31:24 -0800 Subject: [PATCH 13/13] Switch file headers to the MIT license --- src/Testing/src/AssemblyTestLog.cs | 5 +++-- src/Testing/src/CollectDumpAttribute.cs | 5 +++-- src/Testing/src/DumpCollector/DumpCollector.Windows.cs | 5 +++-- src/Testing/src/DumpCollector/DumpCollector.cs | 5 +++-- src/Testing/src/LoggedTest/ILoggedTest.cs | 5 +++-- src/Testing/src/LoggedTest/LoggedTest.cs | 5 +++-- src/Testing/src/LoggedTest/LoggedTestBase.cs | 5 +++-- src/Testing/src/TestFrameworkFileLoggerAttribute.cs | 5 +++-- src/Testing/test/AssemblyTestLogTests.cs | 5 +++-- src/Testing/test/LoggedTestXunitTests.cs | 5 +++-- src/Testing/test/TestTestOutputHelper.cs | 5 +++-- 11 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/Testing/src/AssemblyTestLog.cs b/src/Testing/src/AssemblyTestLog.cs index 611b853fac..bd69cd20a3 100644 --- a/src/Testing/src/AssemblyTestLog.cs +++ b/src/Testing/src/AssemblyTestLog.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Testing/src/CollectDumpAttribute.cs b/src/Testing/src/CollectDumpAttribute.cs index c9e8ee3fa6..5f4a1eee59 100644 --- a/src/Testing/src/CollectDumpAttribute.cs +++ b/src/Testing/src/CollectDumpAttribute.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/Testing/src/DumpCollector/DumpCollector.Windows.cs b/src/Testing/src/DumpCollector/DumpCollector.Windows.cs index 8d4168c20c..20395208d7 100644 --- a/src/Testing/src/DumpCollector/DumpCollector.Windows.cs +++ b/src/Testing/src/DumpCollector/DumpCollector.Windows.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/Testing/src/DumpCollector/DumpCollector.cs b/src/Testing/src/DumpCollector/DumpCollector.cs index 67043ed827..d67e109b38 100644 --- a/src/Testing/src/DumpCollector/DumpCollector.cs +++ b/src/Testing/src/DumpCollector/DumpCollector.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System.Diagnostics; using System.Runtime.InteropServices; diff --git a/src/Testing/src/LoggedTest/ILoggedTest.cs b/src/Testing/src/LoggedTest/ILoggedTest.cs index a906ae84a2..750f45cd91 100644 --- a/src/Testing/src/LoggedTest/ILoggedTest.cs +++ b/src/Testing/src/LoggedTest/ILoggedTest.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.Reflection; diff --git a/src/Testing/src/LoggedTest/LoggedTest.cs b/src/Testing/src/LoggedTest/LoggedTest.cs index d108ffb7e8..169a94f59d 100644 --- a/src/Testing/src/LoggedTest/LoggedTest.cs +++ b/src/Testing/src/LoggedTest/LoggedTest.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System.Reflection; using Microsoft.AspNetCore.Testing; diff --git a/src/Testing/src/LoggedTest/LoggedTestBase.cs b/src/Testing/src/LoggedTest/LoggedTestBase.cs index 324b855319..16dde9676c 100644 --- a/src/Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Testing/src/LoggedTest/LoggedTestBase.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.Linq; diff --git a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs index 1059fa76f2..61fa9993e8 100644 --- a/src/Testing/src/TestFrameworkFileLoggerAttribute.cs +++ b/src/Testing/src/TestFrameworkFileLoggerAttribute.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using Microsoft.AspNetCore.Testing; diff --git a/src/Testing/test/AssemblyTestLogTests.cs b/src/Testing/test/AssemblyTestLogTests.cs index dbefa4ccd2..27a7cf83cf 100644 --- a/src/Testing/test/AssemblyTestLogTests.cs +++ b/src/Testing/test/AssemblyTestLogTests.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.IO; diff --git a/src/Testing/test/LoggedTestXunitTests.cs b/src/Testing/test/LoggedTestXunitTests.cs index d8454023a2..61d7802508 100644 --- a/src/Testing/test/LoggedTestXunitTests.cs +++ b/src/Testing/test/LoggedTestXunitTests.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System.Linq; using System.Reflection; diff --git a/src/Testing/test/TestTestOutputHelper.cs b/src/Testing/test/TestTestOutputHelper.cs index 7043fe4ed2..5a5f6aa85f 100644 --- a/src/Testing/test/TestTestOutputHelper.cs +++ b/src/Testing/test/TestTestOutputHelper.cs @@ -1,5 +1,6 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. using System; using System.Text;