Run helix tests on master (#6728)

.azure/pipelines/helix-test.yml

@@ -0,0 +1,28 @@
+# Don't run CI for this config yet. We're not ready to move official builds on to Azure Pipelines
+trigger: none
+
+# Run PR validation on all branches
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- job: Helix
+  timeoutInMinutes: 240
+  pool:
+    name: Hosted VS2017
+    vmImage: vs2017-win2016
+  steps:
+  - checkout: self
+    clean: true
+  - script: .\build.cmd -all -ci /t:Helix /p:IsHelixJob=true /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\SendToHelix.binlog
+    displayName: Run build.cmd helix target
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Logs to VSTS
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log'
+      PublishLocation: Container
+      ArtifactName: $(Agent.Os)_$(Agent.JobName)
+    continueOnError: true
+    condition: always() 

Directory.Build.props

@@ -1,4 +1,4 @@
-<Project>
+<Project>
   <Import Project="version.props" />
 
   <PropertyGroup>

Directory.Build.targets

@@ -60,6 +60,8 @@
     <IsPackable Condition="'$(IsPackable)' == '' ">false</IsPackable>
 
     <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp3.0' ">$(MicrosoftNETCoreAppPackageVersion)</RuntimeFrameworkVersion>
+    
+    <BuildHelixPayload Condition="'$(BuildHelixPayload)' == '' AND '$(IsTestProject)' == 'true'">true</BuildHelixPayload> 
   </PropertyGroup>
 
   <Import Project="eng\targets\ResolveIisReferences.targets" Condition=" '$(MSBuildProjectExtension)' != '.vcxproj' " />
@@ -68,4 +70,96 @@
   <Import Project="eng\targets\FSharp.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.fsproj'" />
   <Import Project="eng\targets\Wix.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.wixproj'" />
   <Import Project="eng\targets\Npm.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
+
+  <!-- Move to CSharp.Common.props -->
+  <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' AND '$(IsHelixJob)' == 'true' ">
+    <PackageReference Include="xunit.runner.console" Version="2.4.1" />
+  </ItemGroup>  
+  
+  <!-- Prepares the test projects for helix by including xunit and publishing -->
+  <Target Name="PrepareHelixPayload" Returns="@(HelixDirectory)">
+    <ItemGroup>
+      <_TargetFrameworks Remove="@(_TargetFrameworks)" />
+      <_TargetFrameworks Include="$(TargetFrameworks);$(TargetFramework)" />
+    </ItemGroup>
+
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="_PrepareHelixPayloadInner"
+             Properties="TargetFramework=%(_TargetFrameworks.Identity);IsWindowsHelixQueue=$(HelixTargetQueue.Contains('Windows'))">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixDirectory" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="_PrepareHelixPayloadInner" 
+    DependsOnTargets="Publish" 
+    Condition="'$(BuildHelixPayload)' == 'true'" 
+    Returns="@(HelixDirectory)">
+
+    <ItemGroup>
+      <HelixDirectory Include="$(MSBuildProjectFullPath)" />
+    </ItemGroup>
+  </Target>
+
+  <!-- Build the actual helix work items to send to helix queues -->
+  <Target Name="CreateHelixPayload" Returns="@(HelixPayload)">
+    <ItemGroup>
+      <_TargetFrameworks Remove="@(_TargetFrameworks)" />
+      <_TargetFrameworks Include="$(TargetFrameworks);$(TargetFramework)" />
+    </ItemGroup>
+
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="_CreateHelixPayloadInner"
+             Properties="TargetFramework=%(_TargetFrameworks.Identity);IsWindowsHelixQueue=$(HelixTargetQueue.Contains('Windows'))">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixPayload" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="CollectXunitConsoleRunner" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
+    <PropertyGroup>
+      <XunitConsoleRunnerDir>$([System.IO.Path]::GetDirectoryName($(XunitConsole472Path)))</XunitConsoleRunnerDir>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <XunitConsoleRunnerFiles Include="$(XunitConsoleRunnerDir)/**/*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(XunitConsoleRunnerFiles)" DestinationFolder="$(PublishDir)" />
+  </Target>
+
+  <Target Name="_CreateHelixPayloadInner" 
+    DependsOnTargets="CollectXunitConsoleRunner" 
+    Condition="($(IsWindowsHelixQueue) OR '$(TargetFrameworkIdentifier)' != '.NETFramework')" 
+    Returns="@(HelixPayload)">
+
+    <ConvertToAbsolutePath Paths="$(PublishDir)">
+      <Output TaskParameter="AbsolutePaths" PropertyName="PublishAbsoluteDir" />
+    </ConvertToAbsolutePath>
+
+    <!-- Windows NetCore -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' != '.NETFramework' AND $(IsWindowsHelixQueue)" >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\vstest\runtests.cmd" />
+    </ItemGroup>
+
+    <!-- Windows NetFramework -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' AND $(IsWindowsHelixQueue)" >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\xunit\runtests.cmd" />
+    </ItemGroup>
+
+    <!-- NonWindows -->
+    <ItemGroup Condition="!$(IsWindowsHelixQueue)"  >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\vstest\runtests.sh" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_CopyItems)" DestinationFolder="$(PublishAbsoluteDir)" />
+
+    <ItemGroup>
+      <HelixPayload Include="$(PublishAbsoluteDir)">
+        <TestAssembly>$(TargetFileName)</TestAssembly>
+        <TestName>$(MSBuildProjectName)-$(TargetFramework)</TestName>
+        <Command Condition="$(IsWindowsHelixQueue)">runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <TestTimeout>00:30:00</TestTimeout>
+      </HelixPayload>
+    </ItemGroup>
+  </Target>
 </Project>

NuGet.config

@@ -7,6 +7,7 @@
       The only allowed feed here is myget.org/aspnet-tools which is required to work around
       https://github.com/Microsoft/msbuild/issues/2914
     -->
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="myget.org aspnetcore-tools" value="https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json" />
   </packageSources>
 </configuration>

build/repo.targets

@@ -111,4 +111,30 @@
              Properties="$(BuildProperties);__DummyTarget=GenerateBuildAssetManifest" />
   </Target>
 
+  <!-- will move into korebuild -->
+  <Target Name="HelixPublish" DependsOnTargets="Restore">
+    <ItemGroup>
+      <CsProjects Include="@(ProjectToBuild)" Condition="'%(Extension)' == '.csproj'" />
+    </ItemGroup>
+    <MSBuild Projects="@(CsProjects)"
+              Targets="PrepareHelixPayload"
+              BuildInParallel="false">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixDirectory" />
+    </MSBuild>
+
+    <ItemGroup>
+      <HelixTestProject Include="@(HelixDirectory)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="Helix" DependsOnTargets="HelixPublish">
+    <PropertyGroup>
+      <HelixTestProjects>@(HelixTestProject)</HelixTestProjects>
+    </PropertyGroup>
+    <MSBuild Projects="$(MSBuildThisFileDirectory)..\test\helix.proj"
+             Targets="Test"
+             Properties="ProjectsToTest=$(HelixTestProjects)"
+             ContinueOnError="ErrorAndStop"
+             BuildInParallel="false" />
+  </Target>
 </Project>

eng/targets/CSharp.Common.targets

@@ -5,7 +5,7 @@
     <TargetFramework>net$(TargetFrameworkVersion.Substring(1).Replace('.',''))</TargetFramework>
     <TargetFrameworkIdentifier>.NETFramework</TargetFrameworkIdentifier>
   </PropertyGroup>
-
+  
   <Import Project="Packaging.targets" />
   <Import Project="ResolveReferences.targets" />
 </Project>

src/Azure/AzureAD/test/FunctionalTests/Microsoft.AspNetCore.Authentication.AzureAD.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/Blazor/Build/test/Microsoft.AspNetCore.Blazor.Build.Tests.csproj

@@ -30,6 +30,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\testassets\StandaloneApp\StandaloneApp.csproj" />
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
 
     <Compile Include="$(ComponentsSharedSourceRoot)test\**\*.cs" LinkBase="Helpers" />
   </ItemGroup>

src/Components/Blazor/Build/test/RuntimeDependenciesResolverTest.cs

@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.AspNetCore.Testing.xunit;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -19,7 +20,8 @@ namespace Microsoft.AspNetCore.Blazor.Build.Test
             _output = output;
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6549
         public void FindsReferenceAssemblyGraph_ForStandaloneApp()
         {
             // Arrange

src/Components/Server/test/Microsoft.AspNetCore.Components.Server.Tests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/7202 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj

@@ -10,6 +10,8 @@
       Temporarily disabled until this runs on macOS
     -->
     <SkipTests Condition="'$(BlazorAllTests)' != 'true'">true</SkipTests>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6857 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/DataProtection/Extensions/test/DataProtectionProviderTests.cs

@@ -113,8 +113,9 @@ namespace Microsoft.AspNetCore.DataProtection
             });
         }
 
-        [ConditionalFact]
+        [ConditionalFact] 
         [X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void System_UsesProvidedDirectoryAndCertificate()
         {
             var filePath = Path.Combine(GetTestFilesPath(), "TestCert.pfx");
@@ -164,6 +165,7 @@ namespace Microsoft.AspNetCore.DataProtection
 
         [ConditionalFact]
         [X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void System_UsesProvidedCertificateNotFromStore()
         {
             using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser))

src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="..\..\shared\test\*.cs" />
     <Content Include="TestFiles\**\*" CopyToOutputDirectory="PreserveNewest" />
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/DefaultBuilder/test/Microsoft.AspNetCore.FunctionalTests/Microsoft.AspNetCore.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Hosting/Hosting/test/Microsoft.AspNetCore.Hosting.Tests.csproj

@@ -5,6 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />    
     <Content Include="testroot\**\*" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
   </ItemGroup>
 

src/Hosting/Hosting/test/WebHostTests.cs

@@ -14,6 +14,7 @@ using Microsoft.AspNetCore.Hosting.Server;
 using Microsoft.AspNetCore.Hosting.Server.Features;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Testing.xunit;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -194,7 +195,8 @@ namespace Microsoft.AspNetCore.Hosting
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
         public async Task WebHostStopAsyncUsesDefaultTimeoutIfGivenTokenDoesNotFire()
         {
             var data = new Dictionary<string, string>
@@ -233,7 +235,8 @@ namespace Microsoft.AspNetCore.Hosting
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
         public async Task WebHostStopAsyncUsesDefaultTimeoutIfNoTokenProvided()
         {
             var data = new Dictionary<string, string>
@@ -308,7 +311,8 @@ namespace Microsoft.AspNetCore.Hosting
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
         public void WebHostApplicationLifetimeEventsOrderedCorrectlyDuringShutdown()
         {
             using (var host = CreateBuilder()

src/Hosting/test/FunctionalTests/Microsoft.AspNetCore.Hosting.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Http/Routing/test/UnitTests/EndpointNameAddressSchemeTest.cs

@@ -173,17 +173,17 @@ namespace Microsoft.AspNetCore.Routing
             var ex = Assert.Throws<InvalidOperationException>(() => addressScheme.FindEndpoints("any name"));
 
             // Assert
-            Assert.Equal(@"The following endpoints with a duplicate endpoint name were found.
+            Assert.Equal(String.Join(Environment.NewLine, @"The following endpoints with a duplicate endpoint name were found.",
-
+"",
-Endpoints with endpoint name 'name1':
+"Endpoints with endpoint name 'name1':",
-a
+"a",
-b
+"b",
-c
+"c",
-
+"",
-Endpoints with endpoint name 'name2':
+"Endpoints with endpoint name 'name2':",
-e
+"e",
-f
+"f",
-", ex.Message);
+""), ex.Message);
         }
 
         private EndpointNameAddressScheme CreateAddressScheme(params Endpoint[] endpoints)

src/Http/Routing/test/UnitTests/Internal/DfaGraphWriterTest.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing.Patterns;
@@ -33,10 +34,9 @@ namespace Microsoft.AspNetCore.Routing.Internal
             graphWriter.Write(endpointsDataSource, writer);
 
             // Assert
-            Assert.Equal(@"digraph DFA {
+            Assert.Equal(String.Join(Environment.NewLine, @"digraph DFA {",
-0 [label=""/""]
+@"0 [label=""/""]",
-}
+"}") + Environment.NewLine, writer.ToString());
-", writer.ToString());
         }
 
         [Fact]
@@ -57,10 +57,9 @@ namespace Microsoft.AspNetCore.Routing.Internal
             graphWriter.Write(endpointsDataSource, writer);
 
             // Assert
-            Assert.Equal(@"digraph DFA {
+            Assert.Equal(String.Join(Environment.NewLine, @"digraph DFA {",
-0 [label=""/""]
+@"0 [label=""/""]",
-}
+@"}") + Environment.NewLine, writer.ToString());
-", writer.ToString());
         }
 
         [Fact]
@@ -82,14 +81,13 @@ namespace Microsoft.AspNetCore.Routing.Internal
 
             // Assert
             var sdf = writer.ToString();
-            Assert.Equal(@"digraph DFA {
+            Assert.Equal(String.Join(Environment.NewLine, @"digraph DFA {",
-0 [label=""/ HTTP: GET""]
+@"0 [label=""/ HTTP: GET""]",
-1 [label=""/ HTTP: *""]
+@"1 [label=""/ HTTP: *""]",
-2 -> 0 [label=""HTTP: GET""]
+@"2 -> 0 [label=""HTTP: GET""]",
-2 -> 1 [label=""HTTP: *""]
+@"2 -> 1 [label=""HTTP: *""]",
-2 [label=""/""]
+@"2 [label=""/""]",
-}
+@"}") + Environment.NewLine, sdf);
-", sdf);
         }
     }
 }

src/Http/Routing/test/UnitTests/Matching/DefaultEndpointSelectorTest.cs

@@ -167,10 +167,8 @@ namespace Microsoft.AspNetCore.Routing.Matching
 
             // Assert
             Assert.Equal(
-@"The request matched multiple endpoints. Matches: 
+@"The request matched multiple endpoints. Matches: " + Environment.NewLine + Environment.NewLine +
-
+"test: /test2" + Environment.NewLine + "test: /test3", ex.Message);
-test: /test2
-test: /test3", ex.Message);
             Assert.Null(context.Endpoint);
         }
 
@@ -204,4 +202,4 @@ test: /test3", ex.Message);
             return new DefaultEndpointSelector();
         }
     }
-}
+}

src/Identity/ApiAuthorization.IdentityServer/test/Configuration/ConfigureSigningCredentialsTests.cs

@@ -22,7 +22,8 @@ namespace Microsoft.AspNetCore.ApiAuthorization.IdentityServer
             UnsafeEphemeralKeySet : (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? X509KeyStorageFlags.PersistKeySet :
             X509KeyStorageFlags.DefaultKeySet);
 
-        [ConditionalFact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         [FrameworkSkipCondition(RuntimeFrameworks.CLR)]
         public void Configure_AddsDevelopmentKeyFromConfiguration()
         {
@@ -62,7 +63,8 @@ namespace Microsoft.AspNetCore.ApiAuthorization.IdentityServer
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void Configure_LoadsPfxCertificateCredentialFromConfiguration()
         {
             // Arrange
@@ -91,7 +93,8 @@ namespace Microsoft.AspNetCore.ApiAuthorization.IdentityServer
             Assert.Equal("AC8FDF4BD4C10841BD24DC88D983225D10B43BB2", key.Certificate.Thumbprint);
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void Configure_LoadsCertificateStoreCertificateCredentialFromConfiguration()
         {
             try

src/Identity/ApiAuthorization.IdentityServer/test/Configuration/SigningKeysLoaderTests.cs

@@ -58,7 +58,8 @@ namespace Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Configuration
             Assert.Equal("Couldn't find a valid certificate with subject 'Invalid' on the 'CurrentUser\\My'", exception.Message);
         }
 
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public static void LoadFromStoreCert_SkipsCertificatesNotYetValid()
         {
             try
@@ -80,7 +81,8 @@ namespace Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Configuration
             }
         }
 
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public static void LoadFromStoreCert_PrefersCertificatesCloserToExpirationDate()
         {
             try
@@ -102,7 +104,8 @@ namespace Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Configuration
             }
         }
 
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public static void LoadFromStoreCert_SkipsExpiredCertificates()
         {
             try

src/Identity/ApiAuthorization.IdentityServer/test/Microsoft.AspNetCore.ApiAuthorization.IdentityServer.Tests.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <Reference Include="Microsoft.AspNetCore.ApiAuthorization.IdentityServer" />
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/Identity/test/Identity.FunctionalTests/Microsoft.AspNetCore.Identity.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs

@@ -11,6 +11,7 @@ using System.Net.Http;
 using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Testing.xunit;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -37,7 +38,8 @@ namespace Microsoft.AspNetCore.Identity.Test
             }
         }
 
-        [Theory]
+        [ConditionalTheory]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6549
         [MemberData(nameof(ScriptWithIntegrityData))]
         public async Task IdentityUI_ScriptTags_SubresourceIntegrityCheck(ScriptTag scriptTag)
         {
@@ -65,7 +67,8 @@ namespace Microsoft.AspNetCore.Identity.Test
             }
         }
 
-        [Theory]
+        [ConditionalTheory]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6549
         [MemberData(nameof(ScriptWithFallbackSrcData))]
         public async Task IdentityUI_ScriptTags_FallbackSourceContent_Matches_CDNContent(ScriptTag scriptTag)
         {

src/Identity/test/Identity.Test/Microsoft.AspNetCore.Identity.Test.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <Compile Include="$(IdentityTestSharedSourceRoot)**\*.cs" />
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/Middleware/CORS/test/FunctionalTests/FunctionalTests.csproj

@@ -4,6 +4,8 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <TestGroupName>Cors.FunctionalTests</TestGroupName>
     <DefaultItemExcludes>$(DefaultItemExcludes);node_modules\**\*</DefaultItemExcludes>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/MusicStore/test/MusicStore.E2ETests/MusicStore.E2ETests.csproj

@@ -9,6 +9,8 @@
     <WarningsNotAsErrors>$(WarningsNotAsErrors);xUnit1004</WarningsNotAsErrors>
     <!-- Temporarily suppress warnings about Microsoft.AspNetCore.Server.IntegrationTesting -->
     <NoWarn>$(NoWarn);NU1605</NoWarn>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Mvc/test/Microsoft.AspNetCore.Mvc.FunctionalTests/Microsoft.AspNetCore.Mvc.FunctionalTests.csproj

@@ -8,6 +8,8 @@
     <DefineConstants>$(DefineConstants);__RemoveThisBitTo__GENERATE_BASELINES</DefineConstants>
     <!-- The test asset projects this depends on are not strong-named. -->
     <SignAssembly>false</SignAssembly>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Mvc/test/Mvc.Analyzers.Test/Mvc.Analyzers.Test.csproj

@@ -4,6 +4,8 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <PreserveCompilationContext>true</PreserveCompilationContext>
     <RootNamespace>Microsoft.AspNetCore.Mvc.Analyzers</RootNamespace>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Mvc/test/Mvc.Api.Analyzers.Test/Mvc.Api.Analyzers.Test.csproj

@@ -3,6 +3,8 @@
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <RootNamespace>Microsoft.AspNetCore.Mvc.Api.Analyzers</RootNamespace>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/ProjectTemplates/test/ProjectTemplates.Tests.csproj

@@ -8,6 +8,8 @@
     <!-- Workaround until https://github.com/aspnet/AspNetCore/issues/4321 is resolved. -->
     <RunTemplateTests Condition="'$(OS)' != 'Windows_NT'">true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6857 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Security/test/AuthSamples.FunctionalTests/AuthSamples.FunctionalTests.csproj

@@ -4,6 +4,8 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <IsSampleProject>false</IsSampleProject>
     <PlatformTarget>AnyCPU</PlatformTarget>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <Import Project="$(MvcTestingTargets)" Condition="'$(MvcTestingTargets)' != ''" />

src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj

@@ -5,6 +5,8 @@
     <DisableFastUpToDateCheck>True</DisableFastUpToDateCheck>
     <TestGroupName>IISExpress.FunctionalTests</TestGroupName>
     <SkipTests Condition=" '$(SkipIISExpressTests)' == 'true' ">true</SkipTests>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Servers/Kestrel/Kestrel/test/GeneratedCodeTests.cs

@@ -5,13 +5,15 @@ using System;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using Microsoft.AspNetCore.Testing.xunit;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
 {
     public class GeneratedCodeTests
     {
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void GeneratedCodeIsUpToDate()
         {
             var repositoryRoot = typeof(GeneratedCodeTests).Assembly.GetCustomAttributes<AssemblyMetadataAttribute>().First(f => string.Equals(f.Key, "RepositoryRoot", StringComparison.OrdinalIgnoreCase)).Value;

src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="$(SharedSourceRoot)NullScope.cs" />
     <Compile Include="$(KestrelSharedSourceRoot)test\*.cs" LinkBase="shared" />
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
     <Content Include="$(KestrelSharedSourceRoot)test\TestCertificates\*.pfx" LinkBase="shared\TestCertificates" CopyToOutputDirectory="PreserveNewest" />
 
     <AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">

src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs

@@ -34,6 +34,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
 
         [ConditionalFact]
         [HostNameIsReachable]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7267
         public async Task RegisterAddresses_HostName_Success()
         {
             var hostName = Dns.GetHostName();
@@ -332,6 +333,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
 
         [ConditionalFact]
         [HostNameIsReachable]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7267
         public async Task ListenAnyIP_HostName_Success()
         {
             var hostName = Dns.GetHostName();

src/Servers/Kestrel/test/InMemory.FunctionalTests/InMemory.FunctionalTests.csproj

@@ -4,6 +4,8 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <ServerGarbageCollection>true</ServerGarbageCollection>
     <TestGroupName>InMemory.FunctionalTests</TestGroupName>
+    <!-- // https://github.com/aspnet/AspNetCore/issues/7000 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs

@@ -25,6 +25,7 @@ namespace Interop.FunctionalTests
     {
         [ConditionalTheory]
         [MemberData(nameof(H2SpecTestCases))]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7299
         public async Task RunIndividualTestCase(H2SpecTestCase testCase)
         {
             var hostBuilder = new WebHostBuilder()

src/Servers/Kestrel/test/Interop.FunctionalTests/Interop.FunctionalTests.csproj

@@ -13,6 +13,7 @@
     <Compile Include="$(KestrelSharedSourceRoot)test\TransportTestHelpers\IWebHostPortExtensions.cs" LinkBase="shared" />
     <Compile Include="$(KestrelSharedSourceRoot)test\TestResources.cs" LinkBase="shared" />
     <Content Include="$(KestrelSharedSourceRoot)test\TestCertificates\*.pfx" LinkBase="shared\TestCertificates" CopyToOutputDirectory="PreserveNewest" />
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/Servers/Kestrel/test/Libuv.BindTests/Libuv.BindTests.csproj

@@ -7,13 +7,14 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
     <Compile Include="..\BindTests\**\*.cs" />
     <Compile Include="..\Libuv.FunctionalTests\TransportSelector.cs" />
     <Compile Include="$(SharedSourceRoot)NullScope.cs" />
     <Compile Include="$(KestrelSharedSourceRoot)test\*.cs" LinkBase="shared" />
     <Compile Include="$(KestrelSharedSourceRoot)test\TransportTestHelpers\*.cs" LinkBase="shared\TransportTestHelpers" />
     <Content Include="$(KestrelSharedSourceRoot)test\TestCertificates\*.pfx" LinkBase="shared\TestCertificates" CopyToOutputDirectory="PreserveNewest" />
-  </ItemGroup>
+   </ItemGroup>
 
   <ItemGroup>
     <Reference Include="Microsoft.AspNetCore.Server.Kestrel" />

src/Servers/Kestrel/test/Sockets.BindTests/Sockets.BindTests.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
     <Compile Include="..\BindTests\**\*.cs" />
     <Compile Include="..\Sockets.FunctionalTests\TransportSelector.cs" />
     <Compile Include="$(SharedSourceRoot)NullScope.cs" />

src/Servers/test/FunctionalTests/ServerComparison.FunctionalTests.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Tools/FirstRunCertGenerator/test/CertificateManagerTests.cs

@@ -8,6 +8,7 @@ using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
+using Microsoft.AspNetCore.Testing.xunit;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -107,7 +108,8 @@ namespace Microsoft.AspNetCore.Certificates.Generation.Tests
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6721
         public void EnsureCreateHttpsCertificate2_CreatesACertificate_WhenThereAreNoHttpsCertificates()
         {
             try

src/Tools/FirstRunCertGenerator/test/Microsoft.AspNetCore.DeveloperCertificates.XPlat.Tests.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
   	<Reference Include="Microsoft.AspNetCore.DeveloperCertificates.XPlat" />
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
 </Project>

src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj

@@ -5,6 +5,8 @@
     <AssemblyName>Microsoft.DotNet.Watcher.Tools.Tests</AssemblyName>
     <DefaultItemExcludes>$(DefaultItemExcludes);TestProjects\**\*</DefaultItemExcludes>
     <TestGroupName>DotNetWatcherToolsTests</TestGroupName>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

test/helix.proj

@@ -0,0 +1,51 @@
+<Project DefaultTargets="Test">
+
+  <!-- Version included until we get global.json generation to support this SDK. -->
+  <Sdk Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19073.6" />
+
+  <Target Name="Gather" BeforeTargets="Test">
+    <ItemGroup>
+      <ProjectsToTest Include="$(ProjectsToTest)" />
+    </ItemGroup>
+    <MSBuild Projects="@(ProjectsToTest)"
+              Targets="CreateHelixPayload"
+              BuildInParallel="true">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixPayload" />
+    </MSBuild>
+    <ItemGroup>
+      <HelixWorkItem Include="%(HelixPayload.TestName)">
+        <Command>%(HelixPayload.Command)</Command>
+        <PayloadDirectory>%(HelixPayload.Identity)</PayloadDirectory>
+        <Timeout>%(HelixPayload.TestTimeout)</Timeout>
+      </HelixWorkItem>
+    </ItemGroup>
+  </Target>
+
+  <PropertyGroup>
+    <!-- helix SDK issue https://github.com/dotnet/arcade/issues/1605 -->
+    <SkipInvalidConfigurations>true</SkipInvalidConfigurations>
+    <HelixSource>pr/aspnet/aspnetcore</HelixSource>
+    <HelixType>ci</HelixType>
+    <HelixBuild>private-$(USERNAME)</HelixBuild>
+    <HelixBuild Condition=" '$(USERNAME)' == '' ">private-$(USER)</HelixBuild>
+    <HelixBuild Condition=" '$(CI)' == 'true' ">$(BUILD_BUILDNUMBER)</HelixBuild>
+    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+    <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
+    <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
+    <IsExternal>true</IsExternal>
+    <Creator>aspnetcore</Creator>
+
+    <HelixTargetQueues>
+      Windows.10.Amd64.ClientRS4.VS2017.Open;
+      OSX.1012.Amd64.Open;
+      Ubuntu.1810.Amd64.Open;
+      Ubuntu.1604.Amd64.Open;
+      Centos.7.Amd64.Open;
+      Debian.8.Amd64.Open;
+      Debian.9.Amd64.Open;
+      Fedora.27.Amd64.Open;
+      Fedora.28.Amd64.Open;
+      Redhat.7.Amd64.Open;
+    </HelixTargetQueues>
+  </PropertyGroup>
+</Project> 

test/vstest/runtests.cmd

@@ -0,0 +1,14 @@
+set target=%1
+set sdkVersion=%2
+set runtimeVersion=%3
+powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Version %sdkVersion% -InstallDir %HELIX_CORRELATION_PAYLOAD%\sdk"
+powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Runtime dotnet -Version %runtimeVersion% -InstallDir %HELIX_CORRELATION_PAYLOAD%\sdk"
+set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+set DOTNET_ROOT="$HELIX_CORRELATION_PAYLOAD/sdk"
+set PATH="$DOTNET_ROOT:$PATH"
+set DOTNET_MULTILEVEL_LOOKUP=0
+set DOTNET_CLI_HOME="$HELIX_CORRELATION_PAYLOAD/home"
+set helix=true
+%HELIX_CORRELATION_PAYLOAD%\sdk\dotnet vstest %target% --logger:trx
+
+

test/vstest/runtests.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+curl -o dotnet-install.sh -sSL https://dot.net/v1/dotnet-install.sh
+if [ $? -ne 0 ]; then
+    download_retries=3
+    while [ $download_retries -gt 0 ]; do
+        curl -sSL https://dot.net/v1/dotnet-install.sh
+        if [ $? -ne 0 ]; then
+            let download_retries=download_retries-1
+            echo -e "${YELLOW}Failed to download dotnet-install.sh. Retries left: $download_retries.${RESET}"
+        else
+            download_retries=0
+        fi
+    done
+fi
+
+# Call "sync" between "chmod" and execution to prevent "text file busy" error in Docker (aufs)
+chmod +x "dotnet-install.sh"; sync
+
+./dotnet-install.sh --version $2 --install-dir $HELIX_CORRELATION_PAYLOAD/sdk
+if [ $? -ne 0 ]; then
+    sdk_retries=3
+    while [ $sdk_retries -gt 0 ]; do
+        ./dotnet-install.sh --version $2 --install-dir $HELIX_CORRELATION_PAYLOAD/sdk
+        if [ $? -ne 0 ]; then
+            let sdk_retries=sdk_retries-1
+            echo -e "${YELLOW}Failed to install .NET Core SDK $version. Retries left: $sdk_retries.${RESET}"
+        else
+            sdk_retries=0
+        fi
+    done
+fi
+
+./dotnet-install.sh --runtime dotnet --version $3 --install-dir $HELIX_CORRELATION_PAYLOAD/sdk
+if [ $? -ne 0 ]; then
+    runtime_retries=3
+    while [ $runtime_retries -gt 0 ]; do
+        ./dotnet-install.sh --runtime dotnet --version $3 --install-dir $HELIX_CORRELATION_PAYLOAD/sdk
+        if [ $? -ne 0 ]; then
+            let runtime_retries=runtime_retries-1
+            echo -e "${YELLOW}Failed to install .NET Core runtime $version. Retries left: $runtime_retries.${RESET}"
+        else
+            runtime_retries=0
+        fi
+    done
+fi
+
+export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+
+# Ensures every invocation of dotnet apps uses the same dotnet.exe
+export DOTNET_ROOT="$HELIX_CORRELATION_PAYLOAD/sdk"
+
+# Ensure dotnet comes first on PATH
+export PATH="$DOTNET_ROOT:$PATH"
+
+# Prevent fallback to global .NET locations. This ensures our tests use the shared frameworks we specify and don't rollforward to something else that might be installed on the machine
+export DOTNET_MULTILEVEL_LOOKUP=0
+
+# Avoid contaminating userprofiles
+export DOTNET_CLI_HOME="$HELIX_CORRELATION_PAYLOAD/home"
+
+export helix="true"
+
+$HELIX_CORRELATION_PAYLOAD/sdk/dotnet vstest $1 --logger:trx

test/xunit/SkipOnHelixAttribute.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace Microsoft.AspNetCore.Testing.xunit
+{
+    /// <summary>
+    /// Skip test if a given environment variable is not enabled. To enable the test, set environment variable
+    /// to "true" for the test process.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class SkipOnHelixAttribute : Attribute, ITestCondition
+    {
+        public bool IsMet
+        {
+            get
+            {
+                return !string.Equals(Environment.GetEnvironmentVariable("helix"), "true", StringComparison.OrdinalIgnoreCase);
+            }
+        }
+
+        public string SkipReason
+        {
+            get
+            {
+                return $"This test is skipped on helix";
+            }
+        }
+    }
+}

test/xunit/runtests.cmd

@@ -0,0 +1,3 @@
+set target=%1
+set helix=true
+xunit.console.exe %target% -xml testResults.xml