Refactor integration testing for test matrix generation.

build/dependencies.props

@@ -4,34 +4,34 @@
   </PropertyGroup>
   <PropertyGroup Label="Package Versions">
     <InternalAspNetCoreSdkPackageVersion>2.2.0-preview1-17051</InternalAspNetCoreSdkPackageVersion>
-    <MicrosoftAspNetCoreHttpAbstractionsPackageVersion>2.2.0-preview1-34184</MicrosoftAspNetCoreHttpAbstractionsPackageVersion>
+    <MicrosoftAspNetCoreHttpAbstractionsPackageVersion>2.2.0-preview1-34216</MicrosoftAspNetCoreHttpAbstractionsPackageVersion>
-    <MicrosoftAspNetCoreHttpExtensionsPackageVersion>2.2.0-preview1-34184</MicrosoftAspNetCoreHttpExtensionsPackageVersion>
+    <MicrosoftAspNetCoreHttpExtensionsPackageVersion>2.2.0-preview1-34216</MicrosoftAspNetCoreHttpExtensionsPackageVersion>
-    <MicrosoftAspNetCoreHttpFeaturesPackageVersion>2.2.0-preview1-34184</MicrosoftAspNetCoreHttpFeaturesPackageVersion>
+    <MicrosoftAspNetCoreHttpFeaturesPackageVersion>2.2.0-preview1-34216</MicrosoftAspNetCoreHttpFeaturesPackageVersion>
-    <MicrosoftAspNetCoreHttpPackageVersion>2.2.0-preview1-34184</MicrosoftAspNetCoreHttpPackageVersion>
+    <MicrosoftAspNetCoreHttpPackageVersion>2.2.0-preview1-34216</MicrosoftAspNetCoreHttpPackageVersion>
-    <MicrosoftAspNetCoreOwinPackageVersion>2.2.0-preview1-34184</MicrosoftAspNetCoreOwinPackageVersion>
+    <MicrosoftAspNetCoreOwinPackageVersion>2.2.0-preview1-34216</MicrosoftAspNetCoreOwinPackageVersion>
-    <MicrosoftAspNetCoreTestingPackageVersion>2.2.0-preview1-34184</MicrosoftAspNetCoreTestingPackageVersion>
+    <MicrosoftAspNetCoreTestingPackageVersion>2.2.0-preview1-34216</MicrosoftAspNetCoreTestingPackageVersion>
-    <MicrosoftExtensionsConfigurationAbstractionsPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationAbstractionsPackageVersion>
+    <MicrosoftExtensionsConfigurationAbstractionsPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationAbstractionsPackageVersion>
-    <MicrosoftExtensionsConfigurationBinderPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationBinderPackageVersion>
+    <MicrosoftExtensionsConfigurationBinderPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationBinderPackageVersion>
-    <MicrosoftExtensionsConfigurationCommandLinePackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationCommandLinePackageVersion>
+    <MicrosoftExtensionsConfigurationCommandLinePackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationCommandLinePackageVersion>
-    <MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>
+    <MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>
-    <MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>
+    <MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>
-    <MicrosoftExtensionsConfigurationJsonPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationJsonPackageVersion>
+    <MicrosoftExtensionsConfigurationJsonPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationJsonPackageVersion>
-    <MicrosoftExtensionsConfigurationPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsConfigurationPackageVersion>
+    <MicrosoftExtensionsConfigurationPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsConfigurationPackageVersion>
-    <MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>
+    <MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>
-    <MicrosoftExtensionsDependencyInjectionPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsDependencyInjectionPackageVersion>
+    <MicrosoftExtensionsDependencyInjectionPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsDependencyInjectionPackageVersion>
-    <MicrosoftExtensionsDiagnosticAdapterPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsDiagnosticAdapterPackageVersion>
+    <MicrosoftExtensionsDiagnosticAdapterPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsDiagnosticAdapterPackageVersion>
-    <MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>
+    <MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>
-    <MicrosoftExtensionsFileProvidersEmbeddedPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsFileProvidersEmbeddedPackageVersion>
+    <MicrosoftExtensionsFileProvidersEmbeddedPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsFileProvidersEmbeddedPackageVersion>
-    <MicrosoftExtensionsFileProvidersPhysicalPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsFileProvidersPhysicalPackageVersion>
+    <MicrosoftExtensionsFileProvidersPhysicalPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsFileProvidersPhysicalPackageVersion>
-    <MicrosoftExtensionsLoggingAbstractionsPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsLoggingAbstractionsPackageVersion>
+    <MicrosoftExtensionsLoggingAbstractionsPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsLoggingAbstractionsPackageVersion>
-    <MicrosoftExtensionsLoggingConsolePackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsLoggingConsolePackageVersion>
+    <MicrosoftExtensionsLoggingConsolePackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsLoggingConsolePackageVersion>
-    <MicrosoftExtensionsLoggingPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsLoggingPackageVersion>
+    <MicrosoftExtensionsLoggingPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsLoggingPackageVersion>
-    <MicrosoftExtensionsLoggingTestingPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsLoggingTestingPackageVersion>
+    <MicrosoftExtensionsLoggingTestingPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsLoggingTestingPackageVersion>
-    <MicrosoftExtensionsOptionsPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsOptionsPackageVersion>
+    <MicrosoftExtensionsOptionsPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsOptionsPackageVersion>
-    <MicrosoftExtensionsProcessSourcesPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsProcessSourcesPackageVersion>
+    <MicrosoftExtensionsProcessSourcesPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsProcessSourcesPackageVersion>
-    <MicrosoftExtensionsRazorViewsSourcesPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsRazorViewsSourcesPackageVersion>
+    <MicrosoftExtensionsRazorViewsSourcesPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsRazorViewsSourcesPackageVersion>
-    <MicrosoftExtensionsStackTraceSourcesPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsStackTraceSourcesPackageVersion>
+    <MicrosoftExtensionsStackTraceSourcesPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsStackTraceSourcesPackageVersion>
-    <MicrosoftExtensionsTypeNameHelperSourcesPackageVersion>2.2.0-preview1-34184</MicrosoftExtensionsTypeNameHelperSourcesPackageVersion>
+    <MicrosoftExtensionsTypeNameHelperSourcesPackageVersion>2.2.0-preview1-34216</MicrosoftExtensionsTypeNameHelperSourcesPackageVersion>
     <MicrosoftNETCoreApp20PackageVersion>2.0.0</MicrosoftNETCoreApp20PackageVersion>
     <MicrosoftNETCoreApp21PackageVersion>2.1.0-rc1</MicrosoftNETCoreApp21PackageVersion>
     <MicrosoftNETCoreApp22PackageVersion>2.2.0-preview1-26509-06</MicrosoftNETCoreApp22PackageVersion>

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ANCMVersion.cs

@@ -3,8 +3,9 @@
 
 namespace Microsoft.AspNetCore.Server.IntegrationTesting
 {
-    public enum ANCMVersion
+    public enum AncmVersion
     {
+        None,
         AspNetCoreModule,
         AspNetCoreModuleV2
     }

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ApplicationType.cs

@@ -5,7 +5,14 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
 {
     public enum ApplicationType
     {
+        /// <summary>
+        /// Does not target a specific platform. Requires the matching runtime to be installed.
+        /// </summary>
         Portable,
+
+        /// <summary>
+        /// All dlls are published with the app for x-copy deploy. Net461 requires this because ASP.NET Core is not in the GAC.
+        /// </summary>
         Standalone
     }
 }

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/DeploymentParameters.cs

@@ -13,6 +13,35 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
     /// </summary>
     public class DeploymentParameters
     {
+        public DeploymentParameters()
+        {
+            EnvironmentVariables["ASPNETCORE_DETAILEDERRORS"] = "true";
+
+            var configAttribute = Assembly.GetCallingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
+            if (configAttribute != null && !string.IsNullOrEmpty(configAttribute.Configuration))
+            {
+                Configuration = configAttribute.Configuration;
+            }
+        }
+
+        public DeploymentParameters(TestVariant variant)
+        {
+            EnvironmentVariables["ASPNETCORE_DETAILEDERRORS"] = "true";
+
+            var configAttribute = Assembly.GetCallingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
+            if (configAttribute != null && !string.IsNullOrEmpty(configAttribute.Configuration))
+            {
+                Configuration = configAttribute.Configuration;
+            }
+
+            ServerType = variant.Server;
+            TargetFramework = variant.Tfm;
+            ApplicationType = variant.ApplicationType;
+            RuntimeArchitecture = variant.Architecture;
+            HostingModel = variant.HostingModel;
+            AncmVersion = variant.AncmVersion;
+        }
+
         /// <summary>
         /// Creates an instance of <see cref="DeploymentParameters"/>.
         /// </summary>
@@ -54,11 +83,11 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
             }
         }
 
-        public ServerType ServerType { get; }
+        public ServerType ServerType { get; set;  }
 
-        public RuntimeFlavor RuntimeFlavor { get; }
+        public RuntimeFlavor RuntimeFlavor { get; set;  }
 
-        public RuntimeArchitecture RuntimeArchitecture { get; } = RuntimeArchitecture.x64;
+        public RuntimeArchitecture RuntimeArchitecture { get; set; } = RuntimeArchitecture.x64;
 
         /// <summary>
         /// Suggested base url for the deployed application. The final deployed url could be
@@ -80,7 +109,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
 
         public string SiteName { get; set; }
 
-        public string ApplicationPath { get; }
+        public string ApplicationPath { get; set; }
 
         /// <summary>
         /// Gets or sets the name of the application. This is used to execute the application when deployed.
@@ -124,7 +153,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
         /// When using the IISExpressDeployer, determines whether to use the older or newer version
         /// of ANCM.
         /// </summary>
-        public ANCMVersion ANCMVersion { get; set; } = ANCMVersion.AspNetCoreModule;
+        public AncmVersion AncmVersion { get; set; } = AncmVersion.AspNetCoreModule;
 
         /// <summary>
         /// Environment variables to be set before starting the host.

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/HostingModel.cs

@@ -3,8 +3,12 @@
 
 namespace Microsoft.AspNetCore.Server.IntegrationTesting
 {
+    /// <summary>
+    /// For ANCM
+    /// </summary>
     public enum HostingModel
     {
+        None,
         OutOfProcess,
         InProcess
     }

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/RuntimeFlavor.cs

@@ -5,7 +5,8 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
 {
     public enum RuntimeFlavor
     {
-        Clr,
+        None,
-        CoreClr
+        CoreClr,
+        Clr
     }
 }

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/ServerType.cs

@@ -5,9 +5,10 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
 {
     public enum ServerType
     {
+        None,
         IISExpress,
         IIS,
-        WebListener,
+        HttpSys,
         Kestrel,
         Nginx
     }

src/Microsoft.AspNetCore.Server.IntegrationTesting/Common/Tfm.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+    public static class Tfm
+    {
+        public const string Net461 = "net461";
+        public const string NetCoreApp20 = "netcoreapp2.0";
+        public const string NetCoreApp21 = "netcoreapp2.1";
+        public const string NetCoreApp22 = "netcoreapp2.2";
+
+        public static bool Matches(string tfm1, string tfm2)
+        {
+            return string.Equals(tfm1, tfm2, StringComparison.OrdinalIgnoreCase);
+        }
+    }
+}

src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployer.cs

@@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
     /// <summary>
     /// Abstract base class of all deployers with implementation of some of the common helpers.
     /// </summary>
-    public abstract class ApplicationDeployer : IApplicationDeployer
+    public abstract class ApplicationDeployer : IDisposable
     {
         public static readonly string DotnetCommandName = "dotnet";
 
@@ -31,11 +31,56 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
             DeploymentParameters = deploymentParameters;
             LoggerFactory = loggerFactory;
             Logger = LoggerFactory.CreateLogger(GetType().FullName);
+
+            ValidateParameters();
+        }
+
+        private void ValidateParameters()
+        {
+            if (DeploymentParameters.ServerType == ServerType.None)
+            {
+                throw new ArgumentException($"Invalid ServerType '{DeploymentParameters.ServerType}'.");
+            }
+
+            if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.None && !string.IsNullOrEmpty(DeploymentParameters.TargetFramework))
+            {
+                DeploymentParameters.RuntimeFlavor = GetRuntimeFlavor(DeploymentParameters.TargetFramework);
+            }
+
+            if (DeploymentParameters.RuntimeArchitecture == RuntimeArchitecture.x86 && DeploymentParameters.RuntimeFlavor == RuntimeFlavor.CoreClr)
+            {
+                throw new NotSupportedException("32 bit deployment is not yet supported for CoreCLR. Don't remove the tests, just disable them for now.");
+            }
+
+            if (string.IsNullOrEmpty(DeploymentParameters.ApplicationPath))
+            {
+                throw new ArgumentException("ApplicationPath cannot be null.");
+            }
+
+            if (!Directory.Exists(DeploymentParameters.ApplicationPath))
+            {
+                throw new DirectoryNotFoundException(string.Format("Application path {0} does not exist.", DeploymentParameters.ApplicationPath));
+            }
+
+            if (string.IsNullOrEmpty(DeploymentParameters.ApplicationName))
+            {
+                DeploymentParameters.ApplicationName = new DirectoryInfo(DeploymentParameters.ApplicationPath).Name;
+            }
+        }
+
+        private RuntimeFlavor GetRuntimeFlavor(string tfm)
+        {
+            if (Tfm.Matches(Tfm.Net461, tfm))
+            {
+                return RuntimeFlavor.Clr;
+            }
+            return RuntimeFlavor.CoreClr;
         }
 
         protected DeploymentParameters DeploymentParameters { get; }
 
         protected ILoggerFactory LoggerFactory { get; }
+
         protected ILogger Logger { get; }
 
         public abstract Task<DeploymentResult> DeployAsync();
@@ -217,7 +262,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
 
         private string GetRuntimeIdentifier()
         {
-            var architecture = GetArchitecture();
+            var architecture = DeploymentParameters.RuntimeArchitecture;
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
                 return "win7-" + architecture;
@@ -235,18 +280,5 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
                 throw new InvalidOperationException("Unrecognized operation system platform");
             }
         }
-
-        private string GetArchitecture()
-        {
-            switch (RuntimeInformation.OSArchitecture)
-            {
-                case Architecture.X86:
-                    return "x86";
-                case Architecture.X64:
-                    return "x64";
-                default:
-                    throw new NotSupportedException($"Unsupported architecture: {RuntimeInformation.OSArchitecture}");
-            }
-        }
     }
 }

src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/ApplicationDeployerFactory.cs

@@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
         /// <param name="deploymentParameters"></param>
         /// <param name="loggerFactory"></param>
         /// <returns></returns>
-        public static IApplicationDeployer Create(DeploymentParameters deploymentParameters, ILoggerFactory loggerFactory)
+        public static ApplicationDeployer Create(DeploymentParameters deploymentParameters, ILoggerFactory loggerFactory)
         {
             if (deploymentParameters == null)
             {
@@ -35,7 +35,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
                     return new IISExpressDeployer(deploymentParameters, loggerFactory);
                 case ServerType.IIS:
                     throw new NotSupportedException("The IIS deployer is no longer supported");
-                case ServerType.WebListener:
+                case ServerType.HttpSys:
                 case ServerType.Kestrel:
                     return new SelfHostDeployer(deploymentParameters, loggerFactory);
                 case ServerType.Nginx:

src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IApplicationDeployer.cs

@@ -1,20 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Threading.Tasks;
-
-namespace Microsoft.AspNetCore.Server.IntegrationTesting
-{
-    /// <summary>
-    /// Common operations on an application deployer.
-    /// </summary>
-    public interface IApplicationDeployer : IDisposable
-    {
-        /// <summary>
-        /// Deploys the application to the target with specified <see cref="DeploymentParameters"/>.
-        /// </summary>
-        /// <returns></returns>
-        Task<DeploymentResult> DeployAsync();
-    }
-}

src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/IISExpressDeployer.cs

@@ -35,16 +35,6 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
         {
         }
 
-        public bool IsWin8OrLater
-        {
-            get
-            {
-                var win8Version = new Version(6, 2);
-
-                return (Environment.OSVersion.Version >= win8Version);
-            }
-        }
-
         public bool Is64BitHost
         {
             get
@@ -61,13 +51,9 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
                 StartTimer();
 
                 // For now we always auto-publish. Otherwise we'll have to write our own local web.config for the HttpPlatformHandler
-                DeploymentParameters.PublishApplicationBeforeDeployment = true;
-                if (DeploymentParameters.PublishApplicationBeforeDeployment)
-                {
                 DotnetPublish();
-                }
 
-                var contentRoot = DeploymentParameters.PublishApplicationBeforeDeployment ? DeploymentParameters.PublishedApplicationRootPath : DeploymentParameters.ApplicationPath;
+                var contentRoot = DeploymentParameters.PublishedApplicationRootPath;
 
                 var testUri = TestUriHelper.BuildTestUri(ServerType.IISExpress, DeploymentParameters.ApplicationBaseUriHint);
 
@@ -141,7 +127,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
                         ModifyAspNetCoreSectionInWebConfig(key: "hostingModel", value: "inprocess");
                     }
                     
-                    ModifyHandlerSectionInWebConfig(key: "modules", value: DeploymentParameters.ANCMVersion.ToString());
+                    ModifyHandlerSectionInWebConfig(key: "modules", value: DeploymentParameters.AncmVersion.ToString());
 
                     var parameters = string.IsNullOrWhiteSpace(DeploymentParameters.ServerConfigLocation) ?
                                     string.Format("/port:{0} /path:\"{1}\" /trace:error", uri.Port, contentRoot) :

src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/RemoteWindowsDeployer/RemoteWindowsDeployer.cs

@@ -33,10 +33,10 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
 
             if (_deploymentParameters.ServerType != ServerType.IIS
                 && _deploymentParameters.ServerType != ServerType.Kestrel
-                && _deploymentParameters.ServerType != ServerType.WebListener)
+                && _deploymentParameters.ServerType != ServerType.HttpSys)
             {
                 throw new InvalidOperationException($"Server type {_deploymentParameters.ServerType} is not supported for remote deployment." +
-                    $" Supported server types are {nameof(ServerType.Kestrel)}, {nameof(ServerType.IIS)} and {nameof(ServerType.WebListener)}");
+                    $" Supported server types are {nameof(ServerType.Kestrel)}, {nameof(ServerType.IIS)} and {nameof(ServerType.HttpSys)}");
             }
 
             if (string.IsNullOrWhiteSpace(_deploymentParameters.ServerName))

src/Microsoft.AspNetCore.Server.IntegrationTesting/Deployers/SelfHostDeployer.cs

@@ -76,12 +76,7 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
                         DeploymentParameters.ApplicationType == ApplicationType.Portable ? ".dll" : "";
                     var executable = Path.Combine(DeploymentParameters.PublishedApplicationRootPath, DeploymentParameters.ApplicationName + executableExtension);
 
-                    if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.Clr && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                    if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.CoreClr && DeploymentParameters.ApplicationType == ApplicationType.Portable)
-                    {
-                        executableName = "mono";
-                        executableArgs = executable;
-                    }
-                    else if (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.CoreClr && DeploymentParameters.ApplicationType == ApplicationType.Portable)
                     {
                         executableName = "dotnet";
                         executableArgs = executable;
@@ -94,14 +89,13 @@ namespace Microsoft.AspNetCore.Server.IntegrationTesting
                 else
                 {
                     workingDirectory = DeploymentParameters.ApplicationPath;
-                    var targetFramework = DeploymentParameters.TargetFramework ?? (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.Clr ? "net461" : "netcoreapp2.2");
+                    var targetFramework = DeploymentParameters.TargetFramework ?? (DeploymentParameters.RuntimeFlavor == RuntimeFlavor.Clr ? Tfm.Net461 : Tfm.NetCoreApp22);
-
                     executableName = DotnetCommandName;
                     executableArgs = $"run --no-build -c {DeploymentParameters.Configuration} --framework {targetFramework} {DotnetArgumentSeparator}";
                 }
 
-                executableArgs += $" --server.urls {hintUrl} "
+                executableArgs += $" --urls {hintUrl} "
-                + $" --server {(DeploymentParameters.ServerType == ServerType.WebListener ? "Microsoft.AspNetCore.Server.HttpSys" : "Microsoft.AspNetCore.Server.Kestrel")}";
+                + $" --server {(DeploymentParameters.ServerType == ServerType.HttpSys ? "Microsoft.AspNetCore.Server.HttpSys" : "Microsoft.AspNetCore.Server.Kestrel")}";
 
                 Logger.LogInformation($"Executing {executableName} {executableArgs}");
 

src/Microsoft.AspNetCore.Server.IntegrationTesting/TestMatrix.cs

@@ -0,0 +1,336 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+    public class TestMatrix : IEnumerable<object[]>
+    {
+        public IList<ServerType> Servers { get; set; } = new List<ServerType>();
+        public IList<string> Tfms { get; set; } = new List<string>();
+        public IList<ApplicationType> ApplicationTypes { get; set; } = new List<ApplicationType>();
+        public IList<RuntimeArchitecture> Architectures { get; set; } = new List<RuntimeArchitecture>();
+
+        // ANCM specific...
+        public IList<HostingModel> HostingModels { get; set; } = new List<HostingModel>();
+        public IList<AncmVersion> AncmVersions { get; set; } = new List<AncmVersion>();
+
+        private IList<Tuple<Func<TestVariant, bool>, string>> Skips { get; } = new List<Tuple<Func<TestVariant, bool>, string>>();
+
+        public static TestMatrix ForServers(params ServerType[] types)
+        {
+            return new TestMatrix()
+            {
+                Servers = types
+            };
+        }
+
+        public TestMatrix WithTfms(params string[] tfms)
+        {
+            Tfms = tfms;
+            return this;
+        }
+
+        public TestMatrix WithApplicationTypes(params ApplicationType[] types)
+        {
+            ApplicationTypes = types;
+            return this;
+        }
+
+        public TestMatrix WithAllApplicationTypes()
+        {
+            ApplicationTypes.Add(ApplicationType.Portable);
+            ApplicationTypes.Add(ApplicationType.Standalone);
+            return this;
+        }
+        public TestMatrix WithArchitectures(params RuntimeArchitecture[] archs)
+        {
+            Architectures = archs;
+            return this;
+        }
+
+        public TestMatrix WithAllArchitectures()
+        {
+            Architectures.Add(RuntimeArchitecture.x64);
+            Architectures.Add(RuntimeArchitecture.x86);
+            return this;
+        }
+
+        public TestMatrix WithHostingModels(params HostingModel[] models)
+        {
+            HostingModels = models;
+            return this;
+        }
+
+        public TestMatrix WithAllHostingModels()
+        {
+            HostingModels.Add(HostingModel.OutOfProcess);
+            HostingModels.Add(HostingModel.InProcess);
+            return this;
+        }
+
+        public TestMatrix WithAncmVersions(params AncmVersion[] versions)
+        {
+            AncmVersions = versions;
+            return this;
+        }
+
+        public TestMatrix WithAllAncmVersions()
+        {
+            AncmVersions.Add(AncmVersion.AspNetCoreModule);
+            AncmVersions.Add(AncmVersion.AspNetCoreModuleV2);
+            return this;
+        }
+
+        /// <summary>
+        /// V2 + InProc
+        /// </summary>
+        /// <returns></returns>
+        public TestMatrix WithAncmV2InProcess() => WithAncmVersions(AncmVersion.AspNetCoreModuleV2).WithHostingModels(HostingModel.InProcess);
+
+        public TestMatrix Skip(string message, Func<TestVariant, bool> check)
+        {
+            Skips.Add(new Tuple<Func<TestVariant, bool>, string>(check, message));
+            return this;
+        }
+
+        private IEnumerable<TestVariant> Build()
+        {
+            if (!Servers.Any())
+            {
+                throw new ArgumentException("No servers were specified.");
+            }
+
+            // TFMs.
+            if (!Tfms.Any())
+            {
+                throw new ArgumentException("No TFMs were specified.");
+            }
+
+            ResolveDefaultArchitecture();
+
+            if (!ApplicationTypes.Any())
+            {
+                ApplicationTypes.Add(ApplicationType.Portable);
+            }
+
+            if (!AncmVersions.Any())
+            {
+                AncmVersions.Add(AncmVersion.AspNetCoreModule);
+            }
+
+            if (!HostingModels.Any())
+            {
+                HostingModels.Add(HostingModel.OutOfProcess);
+            }
+
+            var variants = new List<TestVariant>();
+            VaryByServer(variants);
+
+            CheckForSkips(variants);
+
+            return variants;
+        }
+
+        private void ResolveDefaultArchitecture()
+        {
+            if (!Architectures.Any())
+            {
+                switch (RuntimeInformation.OSArchitecture)
+                {
+                    case Architecture.X86:
+                        Architectures.Add(RuntimeArchitecture.x86);
+                        break;
+                    case Architecture.X64:
+                        Architectures.Add(RuntimeArchitecture.x64);
+                        break;
+                    default:
+                        throw new ArgumentException(RuntimeInformation.OSArchitecture.ToString());
+                }
+            }
+        }
+
+        private void VaryByServer(List<TestVariant> variants)
+        {
+            foreach (var server in Servers)
+            {
+                var skip = SkipIfServerIsNotSupportedOnThisOS(server);
+
+                VaryByTfm(variants, server, skip);
+            }
+        }
+
+        private static string SkipIfServerIsNotSupportedOnThisOS(ServerType server)
+        {
+            var skip = false;
+            switch (server)
+            {
+                case ServerType.IIS:
+                case ServerType.IISExpress:
+                case ServerType.HttpSys:
+                    skip = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+                    break;
+                case ServerType.Kestrel:
+                    break;
+                case ServerType.Nginx:
+                    // Technically it's possible but we don't test it.
+                    skip = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+                    break;
+                default:
+                    throw new ArgumentException(server.ToString());
+            }
+
+            return skip ? "This server is not supported on this operating system." : null;
+        }
+
+        private void VaryByTfm(List<TestVariant> variants, ServerType server, string skip)
+        {
+            foreach (var tfm in Tfms)
+            {
+                if (!CheckTfmIsSupportedForServer(tfm, server))
+                {
+                    // Don't generate net461 variations for nginx server.
+                    continue;
+                }
+
+                var skipTfm = skip ?? SkipIfTfmIsNotSupportedOnThisOS(tfm);
+
+                VaryByApplicationType(variants, server, tfm, skipTfm);
+            }
+        }
+
+        private bool CheckTfmIsSupportedForServer(string tfm, ServerType server)
+        {
+            // Not a combination we test
+            return !(Tfm.Matches(Tfm.Net461, tfm) && ServerType.Nginx == server);
+        }
+
+        private static string SkipIfTfmIsNotSupportedOnThisOS(string tfm)
+        {
+            if (Tfm.Matches(Tfm.Net461, tfm) && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                return "This TFM is not supported on this operating system.";
+            }
+
+            return null;
+        }
+
+        private void VaryByApplicationType(List<TestVariant> variants, ServerType server, string tfm, string skip)
+        {
+            foreach (var t in ApplicationTypes)
+            {
+                var type = t;
+                if (Tfm.Matches(Tfm.Net461, tfm) && type == ApplicationType.Portable)
+                {
+                    if (ApplicationTypes.Count == 1)
+                    {
+                        // Override the default
+                        type = ApplicationType.Standalone;
+                    }
+                    else
+                    {
+                        continue;
+                    }
+                }
+
+                VaryByArchitecture(variants, server, tfm, skip, type);
+            }
+        }
+
+        private void VaryByArchitecture(List<TestVariant> variants, ServerType server, string tfm, string skip, ApplicationType type)
+        {
+            foreach (var arch in Architectures)
+            {
+                if (server == ServerType.IISExpress)
+                {
+                    VaryByAncmVersion(variants, server, tfm, type, arch, skip);
+                }
+                else
+                {
+                    variants.Add(new TestVariant()
+                    {
+                        Server = server,
+                        Tfm = tfm,
+                        ApplicationType = type,
+                        Architecture = arch,
+                        Skip = skip,
+                    });
+                }
+            }
+        }
+
+        private void VaryByAncmVersion(IList<TestVariant> variants, ServerType server, string tfm, ApplicationType type, RuntimeArchitecture arch, string skip)
+        {
+            foreach (var version in AncmVersions)
+            {
+                VaryByAncmHostingModel(variants, server, tfm, type, arch, skip, version);
+            }
+        }
+
+        private void VaryByAncmHostingModel(IList<TestVariant> variants, ServerType server, string tfm, ApplicationType type, RuntimeArchitecture arch, string skip, AncmVersion version)
+        {
+            foreach (var hostingModel in HostingModels)
+            {
+                var skipAncm = skip;
+                if (hostingModel == HostingModel.InProcess)
+                {
+                    // Not supported
+                    if (Tfm.Matches(Tfm.Net461, tfm) || version == AncmVersion.AspNetCoreModule)
+                    {
+                        continue;
+                    }
+
+                    if (!IISExpressAncmSchema.SupportsInProcessHosting)
+                    {
+                        skipAncm = skipAncm ?? IISExpressAncmSchema.SkipReason;
+                    }
+                }
+
+                variants.Add(new TestVariant()
+                {
+                    Server = server,
+                    Tfm = tfm,
+                    ApplicationType = type,
+                    Architecture = arch,
+                    AncmVersion = version,
+                    HostingModel = hostingModel,
+                    Skip = skipAncm,
+                });
+            }
+        }
+
+        private void CheckForSkips(List<TestVariant> variants)
+        {
+            foreach (var variant in variants)
+            {
+                foreach (var skipPair in Skips)
+                {
+                    if (skipPair.Item1(variant))
+                    {
+                        variant.Skip = skipPair.Item2;
+                        break;
+                    }
+                }
+            }
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return ((IEnumerable<object[]>)this).GetEnumerator();
+        }
+
+        // This is what Xunit MemberData expects
+        public IEnumerator<object[]> GetEnumerator()
+        {
+            foreach (var v in Build())
+            {
+                yield return new[] { v };
+            }
+        }
+    }
+}

src/Microsoft.AspNetCore.Server.IntegrationTesting/TestVariant.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Xunit.Abstractions;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+    public class TestVariant : IXunitSerializable
+    {
+        public ServerType Server { get; set; }
+        public string Tfm { get; set; }
+        public ApplicationType ApplicationType { get; set; }
+        public RuntimeArchitecture Architecture { get; set; }
+
+        public string Skip { get; set; }
+
+        // ANCM specifics...
+        public HostingModel HostingModel { get; set; }
+        public AncmVersion AncmVersion { get; set; }
+
+        public override string ToString()
+        {
+            // For debug and test explorer view
+            return $"Server: {Server}, TFM: {Tfm}, Type: {ApplicationType}, Host: {HostingModel}, ANCM: {AncmVersion}, Arch: {Architecture}";
+        }
+
+        public void Serialize(IXunitSerializationInfo info)
+        {
+            info.AddValue(nameof(Skip), Skip, typeof(string));
+            info.AddValue(nameof(Server), Server, typeof(ServerType));
+            info.AddValue(nameof(Tfm), Tfm, typeof(string));
+            info.AddValue(nameof(ApplicationType), ApplicationType, typeof(ApplicationType));
+            info.AddValue(nameof(Architecture), Architecture, typeof(RuntimeArchitecture));
+            info.AddValue(nameof(HostingModel), HostingModel, typeof(HostingModel));
+            info.AddValue(nameof(AncmVersion), AncmVersion, typeof(AncmVersion));
+        }
+
+        public void Deserialize(IXunitSerializationInfo info)
+        {
+            Skip = info.GetValue<string>(nameof(Skip));
+            Server = info.GetValue<ServerType>(nameof(Server));
+            Tfm = info.GetValue<string>(nameof(Tfm));
+            ApplicationType = info.GetValue<ApplicationType>(nameof(ApplicationType));
+            Architecture = info.GetValue<RuntimeArchitecture>(nameof(Architecture));
+            HostingModel = info.GetValue<HostingModel>(nameof(HostingModel));
+            AncmVersion = info.GetValue<AncmVersion>(nameof(AncmVersion));
+        }
+    }
+}

src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/IISExpressAncmSchema.cs

@@ -0,0 +1,55 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Xml.Linq;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+    public class IISExpressAncmSchema
+    {
+        public static bool SupportsInProcessHosting { get; }
+        public static string SkipReason { get; }
+
+        static IISExpressAncmSchema()
+        {
+            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                SkipReason = "IIS Express tests can only be run on Windows";
+                return;
+            }
+
+            var ancmConfigPath = Path.Combine(
+                Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles),
+                "IIS Express", "config", "schema", "aspnetcore_schema.xml");
+
+            if (!File.Exists(ancmConfigPath))
+            {
+                SkipReason = "IIS Express is not installed.";
+                return;
+            }
+
+            XDocument ancmConfig;
+
+            try
+            {
+                ancmConfig = XDocument.Load(ancmConfigPath);
+            }
+            catch
+            {
+                SkipReason = "Could not read ANCM schema configuration";
+                return;
+            }
+
+            SupportsInProcessHosting = ancmConfig
+                .Root
+                .Descendants("attribute")
+                .Any(n => "hostingModel".Equals(n.Attribute("name")?.Value, StringComparison.Ordinal));
+
+            SkipReason = SupportsInProcessHosting ? null : "IIS Express must be upgraded to support in-process hosting.";
+        }
+    }
+}

src/Microsoft.AspNetCore.Server.IntegrationTesting/xunit/SkipIfIISExpressSchemaMissingInProcessAttribute.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.AspNetCore.Testing.xunit;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Assembly | AttributeTargets.Class)]
+    public sealed partial class SkipIfIISExpressSchemaMissingInProcessAttribute : Attribute, ITestCondition
+    {
+        public bool IsMet => IISExpressAncmSchema.SupportsInProcessHosting;
+
+        public string SkipReason => IISExpressAncmSchema.SkipReason;
+    }
+}

test/Microsoft.AspNetCore.Hosting.FunctionalTests/ShutdownTests.cs

@@ -57,7 +57,7 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests
                     RuntimeArchitecture.x64)
                 {
                     EnvironmentName = "Shutdown",
-                    TargetFramework = "netcoreapp2.2",
+                    TargetFramework = Tfm.NetCoreApp22,
                     ApplicationType = ApplicationType.Portable,
                     PublishApplicationBeforeDeployment = true,
                     StatusMessagesEnabled = false

test/Microsoft.AspNetCore.Hosting.FunctionalTests/WebHostBuilderTests.cs

@@ -17,17 +17,12 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests
     {
         public WebHostBuilderTests(ITestOutputHelper output) : base(output) { }
 
-        [Fact]
+        public static TestMatrix TestVariants => TestMatrix.ForServers(ServerType.Kestrel)
-        public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly_CoreClr()
+                .WithTfms(Tfm.Net461, Tfm.NetCoreApp22);
-            => await InjectedStartup_DefaultApplicationNameIsEntryAssembly(RuntimeFlavor.CoreClr);
 
-        [ConditionalFact]
+        [ConditionalTheory]
-        [OSSkipCondition(OperatingSystems.MacOSX)]
+        [MemberData(nameof(TestVariants))]
-        [OSSkipCondition(OperatingSystems.Linux)]
+        public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly(TestVariant variant)
-        public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly_Clr()
-            => await InjectedStartup_DefaultApplicationNameIsEntryAssembly(RuntimeFlavor.Clr);
-
-        private async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly(RuntimeFlavor runtimeFlavor)
         {
             using (StartLog(out var loggerFactory))
             {
@@ -35,14 +30,9 @@ namespace Microsoft.AspNetCore.Hosting.FunctionalTests
 
                 var applicationPath = Path.Combine(TestPathUtilities.GetSolutionRootDirectory("Hosting"), "test", "TestAssets", "IStartupInjectionAssemblyName");
 
-                var deploymentParameters = new DeploymentParameters(
+                var deploymentParameters = new DeploymentParameters(variant)
-                    applicationPath,
-                    ServerType.Kestrel,
-                    runtimeFlavor,
-                    RuntimeArchitecture.x64)
                 {
-                    TargetFramework = runtimeFlavor == RuntimeFlavor.Clr ? "net461" : "netcoreapp2.2",
+                    ApplicationPath = applicationPath,
-                    ApplicationType = ApplicationType.Portable,
                     StatusMessagesEnabled = false
                 };