From 3b5b40884f36d29a844c2e30d2bc215719af23b8 Mon Sep 17 00:00:00 2001 From: "Chris Ross (ASP.NET)" Date: Tue, 8 May 2018 11:04:13 -0700 Subject: [PATCH] Refactor integration testing for test matrix generation. --- build/dependencies.props | 56 +-- .../Common/ANCMVersion.cs | 3 +- .../Common/ApplicationType.cs | 7 + .../Common/DeploymentParameters.cs | 39 +- .../Common/HostingModel.cs | 4 + .../Common/RuntimeFlavor.cs | 5 +- .../Common/ServerType.cs | 3 +- .../Common/Tfm.cs | 20 ++ .../Deployers/ApplicationDeployer.cs | 62 +++- .../Deployers/ApplicationDeployerFactory.cs | 4 +- .../Deployers/IApplicationDeployer.cs | 20 -- .../Deployers/IISExpressDeployer.cs | 20 +- .../RemoteWindowsDeployer.cs | 4 +- .../Deployers/SelfHostDeployer.cs | 14 +- .../TestMatrix.cs | 336 ++++++++++++++++++ .../TestVariant.cs | 49 +++ .../xunit/IISExpressAncmSchema.cs | 55 +++ ...SExpressSchemaMissingInProcessAttribute.cs | 16 + .../ShutdownTests.cs | 2 +- .../WebHostBuilderTests.cs | 24 +- 20 files changed, 622 insertions(+), 121 deletions(-) create mode 100644 src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/Tfm.cs delete mode 100644 src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IApplicationDeployer.cs create mode 100644 src/Microsoft.AspNetCore.Server.IntegrationTesting/TestMatrix.cs create mode 100644 src/Microsoft.AspNetCore.Server.IntegrationTesting/TestVariant.cs create mode 100644 src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/IISExpressAncmSchema.cs create mode 100644 src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/SkipIfIISExpressSchemaMissingInProcessAttribute.cs diff --git a/build/dependencies.props b/build/dependencies.props index 7dde71cb55..c06cdb96ea 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -4,34 +4,34 @@ 2.2.0-preview1-17051 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 - 2.2.0-preview1-34184 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 + 2.2.0-preview1-34216 2.0.0 2.1.0-rc1 2.2.0-preview1-26509-06 diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ANCMVersion.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ANCMVersion.cs index 5345c2dd19..38cf93ffcd 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ANCMVersion.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ANCMVersion.cs @@ -3,8 +3,9 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting { - public enum ANCMVersion + public enum AncmVersion { + None, AspNetCoreModule, AspNetCoreModuleV2 } diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ApplicationType.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ApplicationType.cs index 02d8715984..3a8afeb94d 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ApplicationType.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ApplicationType.cs @@ -5,7 +5,14 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting { public enum ApplicationType { + /// + /// Does not target a specific platform. Requires the matching runtime to be installed. + /// Portable, + + /// + /// All dlls are published with the app for x-copy deploy. Net461 requires this because ASP.NET Core is not in the GAC. + /// Standalone } } diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/DeploymentParameters.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/DeploymentParameters.cs index 3cd6e3067b..538cd5ffe3 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/DeploymentParameters.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/DeploymentParameters.cs @@ -13,6 +13,35 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting /// public class DeploymentParameters { + public DeploymentParameters() + { + EnvironmentVariables["ASPNETCORE_DETAILEDERRORS"] = "true"; + + var configAttribute = Assembly.GetCallingAssembly().GetCustomAttribute(); + if (configAttribute != null && !string.IsNullOrEmpty(configAttribute.Configuration)) + { + Configuration = configAttribute.Configuration; + } + } + + public DeploymentParameters(TestVariant variant) + { + EnvironmentVariables["ASPNETCORE_DETAILEDERRORS"] = "true"; + + var configAttribute = Assembly.GetCallingAssembly().GetCustomAttribute(); + if (configAttribute != null && !string.IsNullOrEmpty(configAttribute.Configuration)) + { + Configuration = configAttribute.Configuration; + } + + ServerType = variant.Server; + TargetFramework = variant.Tfm; + ApplicationType = variant.ApplicationType; + RuntimeArchitecture = variant.Architecture; + HostingModel = variant.HostingModel; + AncmVersion = variant.AncmVersion; + } + /// /// Creates an instance of . /// @@ -54,11 +83,11 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting } } - public ServerType ServerType { get; } + public ServerType ServerType { get; set; } - public RuntimeFlavor RuntimeFlavor { get; } + public RuntimeFlavor RuntimeFlavor { get; set; } - public RuntimeArchitecture RuntimeArchitecture { get; } = RuntimeArchitecture.x64; + public RuntimeArchitecture RuntimeArchitecture { get; set; } = RuntimeArchitecture.x64; /// /// Suggested base url for the deployed application. The final deployed url could be @@ -80,7 +109,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting public string SiteName { get; set; } - public string ApplicationPath { get; } + public string ApplicationPath { get; set; } /// /// Gets or sets the name of the application. This is used to execute the application when deployed. @@ -124,7 +153,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting /// When using the IISExpressDeployer, determines whether to use the older or newer version /// of ANCM. /// - public ANCMVersion ANCMVersion { get; set; } = ANCMVersion.AspNetCoreModule; + public AncmVersion AncmVersion { get; set; } = AncmVersion.AspNetCoreModule; /// /// Environment variables to be set before starting the host. diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/HostingModel.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/HostingModel.cs index 1eb1aea8c0..5eea2b8ce3 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/HostingModel.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/HostingModel.cs @@ -3,8 +3,12 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting { + /// + /// For ANCM + /// public enum HostingModel { + None, OutOfProcess, InProcess } diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/RuntimeFlavor.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/RuntimeFlavor.cs index 510c713f59..3d65f0eaca 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/RuntimeFlavor.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/RuntimeFlavor.cs @@ -5,7 +5,8 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting { public enum RuntimeFlavor { - Clr, - CoreClr + None, + CoreClr, + Clr } } diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ServerType.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ServerType.cs index 060c8ed0ca..ac84d0225f 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ServerType.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ServerType.cs @@ -5,9 +5,10 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting { public enum ServerType { + None, IISExpress, IIS, - WebListener, + HttpSys, Kestrel, Nginx } diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/Tfm.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/Tfm.cs new file mode 100644 index 0000000000..56715d3c08 --- /dev/null +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/Tfm.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNetCore.Server.IntegrationTesting +{ + public static class Tfm + { + public const string Net461 = "net461"; + public const string NetCoreApp20 = "netcoreapp2.0"; + public const string NetCoreApp21 = "netcoreapp2.1"; + public const string NetCoreApp22 = "netcoreapp2.2"; + + public static bool Matches(string tfm1, string tfm2) + { + return string.Equals(tfm1, tfm2, StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployer.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployer.cs index 3c81f32902..f475c65d81 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployer.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployer.cs @@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting /// /// Abstract base class of all deployers with implementation of some of the common helpers. /// - public abstract class ApplicationDeployer : IApplicationDeployer + public abstract class ApplicationDeployer : IDisposable { public static readonly string DotnetCommandName = "dotnet"; @@ -31,11 +31,56 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting DeploymentParameters = deploymentParameters; LoggerFactory = loggerFactory; Logger = LoggerFactory.CreateLogger(GetType().FullName); + + ValidateParameters(); + } + + private void ValidateParameters() + { + if (DeploymentParameters.ServerType == ServerType.None) + { + throw new ArgumentException($"Invalid ServerType '{DeploymentParameters.ServerType}'."); + } + + if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.None && !string.IsNullOrEmpty(DeploymentParameters.TargetFramework)) + { + DeploymentParameters.RuntimeFlavor = GetRuntimeFlavor(DeploymentParameters.TargetFramework); + } + + if (DeploymentParameters.RuntimeArchitecture == RuntimeArchitecture.x86 && DeploymentParameters.RuntimeFlavor == RuntimeFlavor.CoreClr) + { + throw new NotSupportedException("32 bit deployment is not yet supported for CoreCLR. Don't remove the tests, just disable them for now."); + } + + if (string.IsNullOrEmpty(DeploymentParameters.ApplicationPath)) + { + throw new ArgumentException("ApplicationPath cannot be null."); + } + + if (!Directory.Exists(DeploymentParameters.ApplicationPath)) + { + throw new DirectoryNotFoundException(string.Format("Application path {0} does not exist.", DeploymentParameters.ApplicationPath)); + } + + if (string.IsNullOrEmpty(DeploymentParameters.ApplicationName)) + { + DeploymentParameters.ApplicationName = new DirectoryInfo(DeploymentParameters.ApplicationPath).Name; + } + } + + private RuntimeFlavor GetRuntimeFlavor(string tfm) + { + if (Tfm.Matches(Tfm.Net461, tfm)) + { + return RuntimeFlavor.Clr; + } + return RuntimeFlavor.CoreClr; } protected DeploymentParameters DeploymentParameters { get; } protected ILoggerFactory LoggerFactory { get; } + protected ILogger Logger { get; } public abstract Task DeployAsync(); @@ -217,7 +262,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting private string GetRuntimeIdentifier() { - var architecture = GetArchitecture(); + var architecture = DeploymentParameters.RuntimeArchitecture; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { return "win7-" + architecture; @@ -235,18 +280,5 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting throw new InvalidOperationException("Unrecognized operation system platform"); } } - - private string GetArchitecture() - { - switch (RuntimeInformation.OSArchitecture) - { - case Architecture.X86: - return "x86"; - case Architecture.X64: - return "x64"; - default: - throw new NotSupportedException($"Unsupported architecture: {RuntimeInformation.OSArchitecture}"); - } - } } } diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployerFactory.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployerFactory.cs index eb66761807..76f3588dd9 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployerFactory.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployerFactory.cs @@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting /// /// /// - public static IApplicationDeployer Create(DeploymentParameters deploymentParameters, ILoggerFactory loggerFactory) + public static ApplicationDeployer Create(DeploymentParameters deploymentParameters, ILoggerFactory loggerFactory) { if (deploymentParameters == null) { @@ -35,7 +35,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting return new IISExpressDeployer(deploymentParameters, loggerFactory); case ServerType.IIS: throw new NotSupportedException("The IIS deployer is no longer supported"); - case ServerType.WebListener: + case ServerType.HttpSys: case ServerType.Kestrel: return new SelfHostDeployer(deploymentParameters, loggerFactory); case ServerType.Nginx: diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IApplicationDeployer.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IApplicationDeployer.cs deleted file mode 100644 index 400ae978ed..0000000000 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IApplicationDeployer.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Threading.Tasks; - -namespace Microsoft.AspNetCore.Server.IntegrationTesting -{ - /// - /// Common operations on an application deployer. - /// - public interface IApplicationDeployer : IDisposable - { - /// - /// Deploys the application to the target with specified . - /// - /// - Task DeployAsync(); - } -} \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IISExpressDeployer.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IISExpressDeployer.cs index 029ddfa22e..8a7439809e 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IISExpressDeployer.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IISExpressDeployer.cs @@ -35,16 +35,6 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting { } - public bool IsWin8OrLater - { - get - { - var win8Version = new Version(6, 2); - - return (Environment.OSVersion.Version >= win8Version); - } - } - public bool Is64BitHost { get @@ -61,13 +51,9 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting StartTimer(); // For now we always auto-publish. Otherwise we'll have to write our own local web.config for the HttpPlatformHandler - DeploymentParameters.PublishApplicationBeforeDeployment = true; - if (DeploymentParameters.PublishApplicationBeforeDeployment) - { - DotnetPublish(); - } + DotnetPublish(); - var contentRoot = DeploymentParameters.PublishApplicationBeforeDeployment ? DeploymentParameters.PublishedApplicationRootPath : DeploymentParameters.ApplicationPath; + var contentRoot = DeploymentParameters.PublishedApplicationRootPath; var testUri = TestUriHelper.BuildTestUri(ServerType.IISExpress, DeploymentParameters.ApplicationBaseUriHint); @@ -141,7 +127,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting ModifyAspNetCoreSectionInWebConfig(key: "hostingModel", value: "inprocess"); } - ModifyHandlerSectionInWebConfig(key: "modules", value: DeploymentParameters.ANCMVersion.ToString()); + ModifyHandlerSectionInWebConfig(key: "modules", value: DeploymentParameters.AncmVersion.ToString()); var parameters = string.IsNullOrWhiteSpace(DeploymentParameters.ServerConfigLocation) ? string.Format("/port:{0} /path:\"{1}\" /trace:error", uri.Port, contentRoot) : diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/RemoteWindowsDeployer/RemoteWindowsDeployer.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/RemoteWindowsDeployer/RemoteWindowsDeployer.cs index a102cd02da..dc764bf98a 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/RemoteWindowsDeployer/RemoteWindowsDeployer.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/RemoteWindowsDeployer/RemoteWindowsDeployer.cs @@ -33,10 +33,10 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting if (_deploymentParameters.ServerType != ServerType.IIS && _deploymentParameters.ServerType != ServerType.Kestrel - && _deploymentParameters.ServerType != ServerType.WebListener) + && _deploymentParameters.ServerType != ServerType.HttpSys) { throw new InvalidOperationException($"Server type {_deploymentParameters.ServerType} is not supported for remote deployment." + - $" Supported server types are {nameof(ServerType.Kestrel)}, {nameof(ServerType.IIS)} and {nameof(ServerType.WebListener)}"); + $" Supported server types are {nameof(ServerType.Kestrel)}, {nameof(ServerType.IIS)} and {nameof(ServerType.HttpSys)}"); } if (string.IsNullOrWhiteSpace(_deploymentParameters.ServerName)) diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/SelfHostDeployer.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/SelfHostDeployer.cs index dcb34788f6..e9ee52ac3d 100644 --- a/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/SelfHostDeployer.cs +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/SelfHostDeployer.cs @@ -76,12 +76,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting DeploymentParameters.ApplicationType == ApplicationType.Portable ? ".dll" : ""; var executable = Path.Combine(DeploymentParameters.PublishedApplicationRootPath, DeploymentParameters.ApplicationName + executableExtension); - if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.Clr && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - executableName = "mono"; - executableArgs = executable; - } - else if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.CoreClr && DeploymentParameters.ApplicationType == ApplicationType.Portable) + if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.CoreClr && DeploymentParameters.ApplicationType == ApplicationType.Portable) { executableName = "dotnet"; executableArgs = executable; @@ -94,14 +89,13 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting else { workingDirectory = DeploymentParameters.ApplicationPath; - var targetFramework = DeploymentParameters.TargetFramework ?? (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.Clr ? "net461" : "netcoreapp2.2"); - + var targetFramework = DeploymentParameters.TargetFramework ?? (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.Clr ? Tfm.Net461 : Tfm.NetCoreApp22); executableName = DotnetCommandName; executableArgs = $"run --no-build -c {DeploymentParameters.Configuration} --framework {targetFramework} {DotnetArgumentSeparator}"; } - executableArgs += $" --server.urls {hintUrl} " - + $" --server {(DeploymentParameters.ServerType == ServerType.WebListener ? "Microsoft.AspNetCore.Server.HttpSys" : "Microsoft.AspNetCore.Server.Kestrel")}"; + executableArgs += $" --urls {hintUrl} " + + $" --server {(DeploymentParameters.ServerType == ServerType.HttpSys ? "Microsoft.AspNetCore.Server.HttpSys" : "Microsoft.AspNetCore.Server.Kestrel")}"; Logger.LogInformation($"Executing {executableName} {executableArgs}"); diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/TestMatrix.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/TestMatrix.cs new file mode 100644 index 0000000000..857bb143d9 --- /dev/null +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/TestMatrix.cs @@ -0,0 +1,336 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Microsoft.AspNetCore.Server.IntegrationTesting +{ + public class TestMatrix : IEnumerable + { + public IList Servers { get; set; } = new List(); + public IList Tfms { get; set; } = new List(); + public IList ApplicationTypes { get; set; } = new List(); + public IList Architectures { get; set; } = new List(); + + // ANCM specific... + public IList HostingModels { get; set; } = new List(); + public IList AncmVersions { get; set; } = new List(); + + private IList, string>> Skips { get; } = new List, string>>(); + + public static TestMatrix ForServers(params ServerType[] types) + { + return new TestMatrix() + { + Servers = types + }; + } + + public TestMatrix WithTfms(params string[] tfms) + { + Tfms = tfms; + return this; + } + + public TestMatrix WithApplicationTypes(params ApplicationType[] types) + { + ApplicationTypes = types; + return this; + } + + public TestMatrix WithAllApplicationTypes() + { + ApplicationTypes.Add(ApplicationType.Portable); + ApplicationTypes.Add(ApplicationType.Standalone); + return this; + } + public TestMatrix WithArchitectures(params RuntimeArchitecture[] archs) + { + Architectures = archs; + return this; + } + + public TestMatrix WithAllArchitectures() + { + Architectures.Add(RuntimeArchitecture.x64); + Architectures.Add(RuntimeArchitecture.x86); + return this; + } + + public TestMatrix WithHostingModels(params HostingModel[] models) + { + HostingModels = models; + return this; + } + + public TestMatrix WithAllHostingModels() + { + HostingModels.Add(HostingModel.OutOfProcess); + HostingModels.Add(HostingModel.InProcess); + return this; + } + + public TestMatrix WithAncmVersions(params AncmVersion[] versions) + { + AncmVersions = versions; + return this; + } + + public TestMatrix WithAllAncmVersions() + { + AncmVersions.Add(AncmVersion.AspNetCoreModule); + AncmVersions.Add(AncmVersion.AspNetCoreModuleV2); + return this; + } + + /// + /// V2 + InProc + /// + /// + public TestMatrix WithAncmV2InProcess() => WithAncmVersions(AncmVersion.AspNetCoreModuleV2).WithHostingModels(HostingModel.InProcess); + + public TestMatrix Skip(string message, Func check) + { + Skips.Add(new Tuple, string>(check, message)); + return this; + } + + private IEnumerable Build() + { + if (!Servers.Any()) + { + throw new ArgumentException("No servers were specified."); + } + + // TFMs. + if (!Tfms.Any()) + { + throw new ArgumentException("No TFMs were specified."); + } + + ResolveDefaultArchitecture(); + + if (!ApplicationTypes.Any()) + { + ApplicationTypes.Add(ApplicationType.Portable); + } + + if (!AncmVersions.Any()) + { + AncmVersions.Add(AncmVersion.AspNetCoreModule); + } + + if (!HostingModels.Any()) + { + HostingModels.Add(HostingModel.OutOfProcess); + } + + var variants = new List(); + VaryByServer(variants); + + CheckForSkips(variants); + + return variants; + } + + private void ResolveDefaultArchitecture() + { + if (!Architectures.Any()) + { + switch (RuntimeInformation.OSArchitecture) + { + case Architecture.X86: + Architectures.Add(RuntimeArchitecture.x86); + break; + case Architecture.X64: + Architectures.Add(RuntimeArchitecture.x64); + break; + default: + throw new ArgumentException(RuntimeInformation.OSArchitecture.ToString()); + } + } + } + + private void VaryByServer(List variants) + { + foreach (var server in Servers) + { + var skip = SkipIfServerIsNotSupportedOnThisOS(server); + + VaryByTfm(variants, server, skip); + } + } + + private static string SkipIfServerIsNotSupportedOnThisOS(ServerType server) + { + var skip = false; + switch (server) + { + case ServerType.IIS: + case ServerType.IISExpress: + case ServerType.HttpSys: + skip = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + break; + case ServerType.Kestrel: + break; + case ServerType.Nginx: + // Technically it's possible but we don't test it. + skip = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + break; + default: + throw new ArgumentException(server.ToString()); + } + + return skip ? "This server is not supported on this operating system." : null; + } + + private void VaryByTfm(List variants, ServerType server, string skip) + { + foreach (var tfm in Tfms) + { + if (!CheckTfmIsSupportedForServer(tfm, server)) + { + // Don't generate net461 variations for nginx server. + continue; + } + + var skipTfm = skip ?? SkipIfTfmIsNotSupportedOnThisOS(tfm); + + VaryByApplicationType(variants, server, tfm, skipTfm); + } + } + + private bool CheckTfmIsSupportedForServer(string tfm, ServerType server) + { + // Not a combination we test + return !(Tfm.Matches(Tfm.Net461, tfm) && ServerType.Nginx == server); + } + + private static string SkipIfTfmIsNotSupportedOnThisOS(string tfm) + { + if (Tfm.Matches(Tfm.Net461, tfm) && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "This TFM is not supported on this operating system."; + } + + return null; + } + + private void VaryByApplicationType(List variants, ServerType server, string tfm, string skip) + { + foreach (var t in ApplicationTypes) + { + var type = t; + if (Tfm.Matches(Tfm.Net461, tfm) && type == ApplicationType.Portable) + { + if (ApplicationTypes.Count == 1) + { + // Override the default + type = ApplicationType.Standalone; + } + else + { + continue; + } + } + + VaryByArchitecture(variants, server, tfm, skip, type); + } + } + + private void VaryByArchitecture(List variants, ServerType server, string tfm, string skip, ApplicationType type) + { + foreach (var arch in Architectures) + { + if (server == ServerType.IISExpress) + { + VaryByAncmVersion(variants, server, tfm, type, arch, skip); + } + else + { + variants.Add(new TestVariant() + { + Server = server, + Tfm = tfm, + ApplicationType = type, + Architecture = arch, + Skip = skip, + }); + } + } + } + + private void VaryByAncmVersion(IList variants, ServerType server, string tfm, ApplicationType type, RuntimeArchitecture arch, string skip) + { + foreach (var version in AncmVersions) + { + VaryByAncmHostingModel(variants, server, tfm, type, arch, skip, version); + } + } + + private void VaryByAncmHostingModel(IList variants, ServerType server, string tfm, ApplicationType type, RuntimeArchitecture arch, string skip, AncmVersion version) + { + foreach (var hostingModel in HostingModels) + { + var skipAncm = skip; + if (hostingModel == HostingModel.InProcess) + { + // Not supported + if (Tfm.Matches(Tfm.Net461, tfm) || version == AncmVersion.AspNetCoreModule) + { + continue; + } + + if (!IISExpressAncmSchema.SupportsInProcessHosting) + { + skipAncm = skipAncm ?? IISExpressAncmSchema.SkipReason; + } + } + + variants.Add(new TestVariant() + { + Server = server, + Tfm = tfm, + ApplicationType = type, + Architecture = arch, + AncmVersion = version, + HostingModel = hostingModel, + Skip = skipAncm, + }); + } + } + + private void CheckForSkips(List variants) + { + foreach (var variant in variants) + { + foreach (var skipPair in Skips) + { + if (skipPair.Item1(variant)) + { + variant.Skip = skipPair.Item2; + break; + } + } + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)this).GetEnumerator(); + } + + // This is what Xunit MemberData expects + public IEnumerator GetEnumerator() + { + foreach (var v in Build()) + { + yield return new[] { v }; + } + } + } +} diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/TestVariant.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/TestVariant.cs new file mode 100644 index 0000000000..e9b9cda607 --- /dev/null +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/TestVariant.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit.Abstractions; + +namespace Microsoft.AspNetCore.Server.IntegrationTesting +{ + public class TestVariant : IXunitSerializable + { + public ServerType Server { get; set; } + public string Tfm { get; set; } + public ApplicationType ApplicationType { get; set; } + public RuntimeArchitecture Architecture { get; set; } + + public string Skip { get; set; } + + // ANCM specifics... + public HostingModel HostingModel { get; set; } + public AncmVersion AncmVersion { get; set; } + + public override string ToString() + { + // For debug and test explorer view + return $"Server: {Server}, TFM: {Tfm}, Type: {ApplicationType}, Host: {HostingModel}, ANCM: {AncmVersion}, Arch: {Architecture}"; + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(Skip), Skip, typeof(string)); + info.AddValue(nameof(Server), Server, typeof(ServerType)); + info.AddValue(nameof(Tfm), Tfm, typeof(string)); + info.AddValue(nameof(ApplicationType), ApplicationType, typeof(ApplicationType)); + info.AddValue(nameof(Architecture), Architecture, typeof(RuntimeArchitecture)); + info.AddValue(nameof(HostingModel), HostingModel, typeof(HostingModel)); + info.AddValue(nameof(AncmVersion), AncmVersion, typeof(AncmVersion)); + } + + public void Deserialize(IXunitSerializationInfo info) + { + Skip = info.GetValue(nameof(Skip)); + Server = info.GetValue(nameof(Server)); + Tfm = info.GetValue(nameof(Tfm)); + ApplicationType = info.GetValue(nameof(ApplicationType)); + Architecture = info.GetValue(nameof(Architecture)); + HostingModel = info.GetValue(nameof(HostingModel)); + AncmVersion = info.GetValue(nameof(AncmVersion)); + } + } +} diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/IISExpressAncmSchema.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/IISExpressAncmSchema.cs new file mode 100644 index 0000000000..53ea67ddd2 --- /dev/null +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/IISExpressAncmSchema.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Xml.Linq; + +namespace Microsoft.AspNetCore.Server.IntegrationTesting +{ + public class IISExpressAncmSchema + { + public static bool SupportsInProcessHosting { get; } + public static string SkipReason { get; } + + static IISExpressAncmSchema() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + SkipReason = "IIS Express tests can only be run on Windows"; + return; + } + + var ancmConfigPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), + "IIS Express", "config", "schema", "aspnetcore_schema.xml"); + + if (!File.Exists(ancmConfigPath)) + { + SkipReason = "IIS Express is not installed."; + return; + } + + XDocument ancmConfig; + + try + { + ancmConfig = XDocument.Load(ancmConfigPath); + } + catch + { + SkipReason = "Could not read ANCM schema configuration"; + return; + } + + SupportsInProcessHosting = ancmConfig + .Root + .Descendants("attribute") + .Any(n => "hostingModel".Equals(n.Attribute("name")?.Value, StringComparison.Ordinal)); + + SkipReason = SupportsInProcessHosting ? null : "IIS Express must be upgraded to support in-process hosting."; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/SkipIfIISExpressSchemaMissingInProcessAttribute.cs b/src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/SkipIfIISExpressSchemaMissingInProcessAttribute.cs new file mode 100644 index 0000000000..ecadf4522a --- /dev/null +++ b/src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/SkipIfIISExpressSchemaMissingInProcessAttribute.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.Testing.xunit; + +namespace Microsoft.AspNetCore.Server.IntegrationTesting +{ + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Assembly | AttributeTargets.Class)] + public sealed partial class SkipIfIISExpressSchemaMissingInProcessAttribute : Attribute, ITestCondition + { + public bool IsMet => IISExpressAncmSchema.SupportsInProcessHosting; + + public string SkipReason => IISExpressAncmSchema.SkipReason; + } +} diff --git a/test/Microsoft.AspNetCore.Hosting.FunctionalTests/ShutdownTests.cs b/test/Microsoft.AspNetCore.Hosting.FunctionalTests/ShutdownTests.cs index b54b923a89..4249627f56 100644 --- a/test/Microsoft.AspNetCore.Hosting.FunctionalTests/ShutdownTests.cs +++ b/test/Microsoft.AspNetCore.Hosting.FunctionalTests/ShutdownTests.cs @@ -57,7 +57,7 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests RuntimeArchitecture.x64) { EnvironmentName = "Shutdown", - TargetFramework = "netcoreapp2.2", + TargetFramework = Tfm.NetCoreApp22, ApplicationType = ApplicationType.Portable, PublishApplicationBeforeDeployment = true, StatusMessagesEnabled = false diff --git a/test/Microsoft.AspNetCore.Hosting.FunctionalTests/WebHostBuilderTests.cs b/test/Microsoft.AspNetCore.Hosting.FunctionalTests/WebHostBuilderTests.cs index ec5dd12651..f594af25b0 100644 --- a/test/Microsoft.AspNetCore.Hosting.FunctionalTests/WebHostBuilderTests.cs +++ b/test/Microsoft.AspNetCore.Hosting.FunctionalTests/WebHostBuilderTests.cs @@ -17,17 +17,12 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests { public WebHostBuilderTests(ITestOutputHelper output) : base(output) { } - [Fact] - public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly_CoreClr() - => await InjectedStartup_DefaultApplicationNameIsEntryAssembly(RuntimeFlavor.CoreClr); + public static TestMatrix TestVariants => TestMatrix.ForServers(ServerType.Kestrel) + .WithTfms(Tfm.Net461, Tfm.NetCoreApp22); - [ConditionalFact] - [OSSkipCondition(OperatingSystems.MacOSX)] - [OSSkipCondition(OperatingSystems.Linux)] - public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly_Clr() - => await InjectedStartup_DefaultApplicationNameIsEntryAssembly(RuntimeFlavor.Clr); - - private async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly(RuntimeFlavor runtimeFlavor) + [ConditionalTheory] + [MemberData(nameof(TestVariants))] + public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly(TestVariant variant) { using (StartLog(out var loggerFactory)) { @@ -35,14 +30,9 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests var applicationPath = Path.Combine(TestPathUtilities.GetSolutionRootDirectory("Hosting"), "test", "TestAssets", "IStartupInjectionAssemblyName"); - var deploymentParameters = new DeploymentParameters( - applicationPath, - ServerType.Kestrel, - runtimeFlavor, - RuntimeArchitecture.x64) + var deploymentParameters = new DeploymentParameters(variant) { - TargetFramework = runtimeFlavor == RuntimeFlavor.Clr ? "net461" : "netcoreapp2.2", - ApplicationType = ApplicationType.Portable, + ApplicationPath = applicationPath, StatusMessagesEnabled = false };