[Helix] Shared framework support + Templates tests (#19177)

.azure/pipelines/ci.yml

@@ -154,8 +154,7 @@ stages:
         displayName: Build x64
 
       # Build the x86 shared framework
-      # TODO: make it possible to build for one Windows architecture at a time
+      # This is going to actually build x86 native assets.
-      # This is going to actually build x86 native assets. See https://github.com/dotnet/aspnetcore/issues/7196
       - script: ./build.cmd
                 -ci
                 -arch x86
@@ -631,7 +630,7 @@ stages:
         publishOnError: true
         includeForks: true
 
-# Helix x64
+  # Helix x64
   - template: jobs/default-build.yml
     parameters:
       condition: eq(variables['Build.Reason'], 'PullRequest')
@@ -640,9 +639,12 @@ stages:
       agentOs: Windows
       timeoutInMinutes: 180
       steps:
+      # Build the shared framework
+      - script: ./build.cmd -ci -all -pack -arch x64 -buildNative /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog
+        displayName: Build shared fx
       - script: .\restore.cmd -ci
         displayName: Restore
-      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.cmd helix target
         env:
           HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
@@ -661,9 +663,13 @@ stages:
       agentOs: Windows
       timeoutInMinutes: 180
       steps:
+      # Build the shared framework
+      - script: ./build.cmd -ci -all -pack -arch x64 -buildNative /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.daily.build.x64.binlog
+        displayName: Build shared fx
+      # Build the x86 shared framework
       - script: .\restore.cmd -ci
         displayName: Restore
-      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.cmd helix target
         env:
           HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
@@ -683,6 +689,7 @@ stages:
       agentOs: Linux
       timeoutInMinutes: 180
       steps:
+      # Build the shared framework
       - script: ./restore.sh -ci
         displayName: Restore
       - script: ./build.sh -ci --arch arm64 -test --no-build-nodejs -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl

.azure/pipelines/quarantined-tests.yml

@@ -30,9 +30,12 @@ jobs:
     agentOs: Windows
     timeoutInMinutes: 240
     steps:
+    # Build the shared framework
+    - script: ./build.cmd -ci -all -pack -arch x64 -buildNative /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog
+      displayName: Build shared fx    
     - script: .\restore.cmd -ci
       displayName: Restore
-    - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+    - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
       displayName: Run build.cmd helix target
       env:
         HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues

eng/helix/content/InstallAppRuntime.ps1

@@ -0,0 +1,54 @@
+ <# 
+ .SYNOPSIS 
+     Installs an AspNetCore shared framework on a machine
+ .DESCRIPTION 
+     This script installs an AspNetCore shared framework on a machine
+ .PARAMETER AppRuntimePath
+     The path to the app runtime package to install.
+ .PARAMETER InstallDir
+     The directory to install the shared framework to.     
+ .PARAMETER Framework
+     The framework directory to copy the shared framework from.
+ .PARAMETER RuntimeIdentifier
+     The runtime identifier for the shared framework.
+ #> 
+param(
+    [Parameter(Mandatory = $true)]
+    $AppRuntimePath,
+    
+    [Parameter(Mandatory = $true)]
+    $InstallDir,
+
+    [Parameter(Mandatory = $true)]
+    $Framework,
+    
+    [Parameter(Mandatory = $true)]
+    $RuntimeIdentifier)
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+
+Set-StrictMode -Version 1
+
+Write-Host "Extracting to $InstallDir"
+
+$zipPackage = [io.path]::ChangeExtension($AppRuntimePath, ".zip")
+Write-Host "Renaming to $zipPackage"
+Rename-Item -Path $AppRuntimePath -NewName $zipPackage
+if (Get-Command -Name 'Microsoft.PowerShell.Archive\Expand-Archive' -ErrorAction Ignore) {
+    # Use built-in commands where possible as they are cross-plat compatible
+    Microsoft.PowerShell.Archive\Expand-Archive -Path $zipPackage -DestinationPath ".\tmpRuntime" -Force
+}
+else {
+    Remove-Item ".\tmpRuntime" -Recurse -ErrorAction Ignore
+    # Fallback to old approach for old installations of PowerShell
+    Add-Type -AssemblyName System.IO.Compression.FileSystem
+    [System.IO.Compression.ZipFile]::ExtractToDirectory($zipPackage, ".\tmpRuntime")
+}
+
+Get-ChildItem -Path ".\tmpRuntime" -Recurse
+
+Write-Host "Copying managed files to $InstallDir"
+Copy-Item -Path ".\tmpRuntime\runtimes\$RuntimeIdentifier\lib\$Framework\*" $InstallDir
+Write-Host "Copying native files to $InstallDir"
+Copy-Item -Path ".\tmpRuntime\runtimes\$RuntimeIdentifier\native\*" $InstallDir

eng/helix/content/InstallNode.ps1

@@ -29,9 +29,9 @@ if (Get-Command "node.exe" -ErrorAction SilentlyContinue)
     exit
 }
 
-if (Test-Path "$output_dir\node.exe")
+if (Test-Path "$InstallDir\node.exe")
 {
-    Write-Host "Node.exe found at $output_dir"
+    Write-Host "Node.exe found at $InstallDir"
     exit
 }
 

eng/helix/content/installappruntime.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Cause the script to fail if any subcommand fails
+set -e
+
+appRuntimePath=$1
+output_dir=$2
+framework=$3
+rid=$4
+tmpDir=./tmpRuntime
+
+echo "Installing shared framework from $appRuntimePath"
+cp $appRuntimePath sharedFx.zip
+
+mkdir -p $tmpDir
+unzip sharedFx.zip -d $tmpDir
+mkdir -p $output_dir
+echo "Copying to $output_dir"
+cp $tmpDir/runtimes/$rid/lib/$framework/* $output_dir
+cp $tmpDir/runtimes/$rid/native/* $output_dir

eng/helix/content/runtests.cmd

@@ -21,7 +21,22 @@ set PATH=%DOTNET_ROOT%;%PATH%;%HELIX_CORRELATION_PAYLOAD%\node\bin
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Version %$sdkVersion% -InstallDir %DOTNET_ROOT%"
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Runtime dotnet -Version %$runtimeVersion% -InstallDir %DOTNET_ROOT%"
 
+if EXIST ".\Microsoft.AspNetCore.App" (
+    echo "Found Microsoft.AspNetCore.App, copying to %DOTNET_ROOT%\shared\Microsoft.AspNetCore.App\%runtimeVersion%"
+    xcopy /i /y ".\Microsoft.AspNetCore.App" %DOTNET_ROOT%\shared\Microsoft.AspNetCore.App\%runtimeVersion%\
+)
+
+echo "Current Directory: %HELIX_WORKITEM_ROOT%"
 set HELIX=%$helixQueue%
+set HELIX_DIR=%HELIX_WORKITEM_ROOT%
+set NUGET_FALLBACK_PACKAGES=%HELIX_DIR%
+set NUGET_RESTORE=%HELIX_DIR%\nugetRestore
+echo "Setting HELIX_DIR: %HELIX_DIR%"
+echo Creating nuget restore directory: %NUGET_RESTORE%
+mkdir %NUGET_RESTORE%
+mkdir logs
+
+dir
 
 %DOTNET_ROOT%\dotnet vstest %$target% -lt >discovered.txt
 find /c "Exception thrown" discovered.txt

eng/helix/content/runtests.sh

@@ -31,6 +31,14 @@ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
 # Used by SkipOnHelix attribute
 export helix="$helix_queue_name"
+export HELIX_DIR="$DIR"
+export NUGET_FALLBACK_PACKAGES="$DIR"
+export NUGET_RESTORE="$DIR/nugetRestore"
+echo "Creating nugetRestore directory: $NUGET_RESTORE"
+mkdir $NUGET_RESTORE
+mkdir logs
+
+ls -la
 
 RESET="\033[0m"
 RED="\033[0;31m"
@@ -82,6 +90,13 @@ if [ $? -ne 0 ]; then
     done
 fi
 
+# Copy over any local shared fx if found
+if [ -d "Microsoft.AspNetCore.App" ]
+then
+    echo "Found Microsoft.AspNetCore.App directory, copying to $DOTNET_ROOT/shared/Microsoft.AspNetCore.App/$dotnet_runtime_version."
+    cp -r Microsoft.AspNetCore.App $DOTNET_ROOT/shared/Microsoft.AspNetCore.App/$dotnet_runtime_version
+fi
+
 if [ -e /proc/self/coredump_filter ]; then
   # Include memory in private and shared file-backed mappings in the dump.
   # This ensures that we can see disassembly from our shared libraries when

eng/targets/Helix.props

@@ -21,6 +21,8 @@
     <HelixUseArchive>false</HelixUseArchive>
     <LoggingTestingDisableFileLogging Condition="'$(IsHelixJob)' == 'true'">false</LoggingTestingDisableFileLogging>
     <NodeVersion>10.15.3</NodeVersion>
+    <AppRuntimeVersion>5.0.0-ci</AppRuntimeVersion>
+    <TestDependsOnAspNetRuntime>false</TestDependsOnAspNetRuntime>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">

eng/targets/Helix.targets

@@ -16,6 +16,20 @@
     <HelixPreCommand Include="call RunPowershell.cmd InstallNode.ps1 $(NodeVersion) %25HELIX_CORRELATION_PAYLOAD%25\node\bin || exit /b 1" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(IsHelixJob)' == 'true' AND '$(IsWindowsHelixQueue)' == 'true' AND '$(TestDependsOnAspNetRuntime)' == 'true'">
+    <HelixContent Include="$(RepoRoot)artifacts\packages\Release\Shipping\Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg" />
+    <HelixPreCommand Include="call RunPowershell.cmd InstallAppRuntime.ps1 Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg Microsoft.AspNetCore.App netcoreapp5.0 win-x64 || exit /b 1" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IsHelixJob)' == 'true' AND '$(IsWindowsHelixQueue)' == 'false' AND '$(TestDependsOnAspNetRuntime)' == 'true'">
+    <HelixContent Include="$(RepoRoot)artifacts\packages\Release\Shipping\Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg" />
+    <HelixPreCommand Include="./installappruntime.sh Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg Microsoft.AspNetCore.App netcoreapp5.0 win-x64" />
+  </ItemGroup>
+  
+  <ItemGroup Condition="'$(IsHelixJob)' == 'true' AND '$(TestDependsOnAspNetRuntime)' == 'true'">
+    <HelixContent Include="$(RepoRoot)artifacts\packages\Release\Shipping\*-ci.nupkg" />
+  </ItemGroup>  
+  
   <!-- Item group has to be defined here becasue Helix.props is evaluated before xunit.runner.console.props  -->
   <ItemGroup Condition="$(BuildHelixPayload)">
     <Content Include="@(HelixContent)" />

src/ProjectTemplates/test/BlazorServerTemplateTest.cs

@@ -24,7 +24,8 @@ namespace Templates.Test
 
         public Project Project { get; private set; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public async Task BlazorServerTemplateWorks_NoAuth()
         {
             Project = await ProjectFactory.GetOrCreateProject("blazorservernoauth", Output);
@@ -79,9 +80,10 @@ namespace Templates.Test
             }
         }
 
-        [Theory]
+        [ConditionalTheory]
         [InlineData(true)]
         [InlineData(false)]
+        [SkipOnHelix("ef restore no worky")]
         public async Task BlazorServerTemplateWorks_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("blazorserverindividual" + (useLocalDB ? "uld" : ""), Output);

src/ProjectTemplates/test/ByteOrderMarkTest.cs

@@ -5,6 +5,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
+using Microsoft.AspNetCore.Testing;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -19,7 +20,8 @@ namespace Templates.Test
             _output = output;
         }
 
-        [Theory]
+        [ConditionalTheory]
+        [SkipOnHelix("missing files")]
         [InlineData("Web.ProjectTemplates")]
         [InlineData("Web.Spa.ProjectTemplates")]
         public void JSAndJSONInAllTemplates_ShouldNotContainBOM(string projectName)
@@ -60,7 +62,8 @@ namespace Templates.Test
             Assert.False(filesWithBOMCharactersPresent);
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("missing files")]
         public void RazorFilesInWebProjects_ShouldContainBOM()
         {
             var projectName = "Web.ProjectTemplates";

src/ProjectTemplates/test/EmptyWebTemplateTest.cs

@@ -3,6 +3,7 @@
 
 using System.Threading.Tasks;
 using Templates.Test.Helpers;
+using Microsoft.AspNetCore.Testing;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -22,7 +23,8 @@ namespace Templates.Test
 
         public ITestOutputHelper Output { get; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("Cert failures", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task EmptyWebTemplateCSharp()
         {
             await EmtpyTemplateCore(languageOverride: null);

src/ProjectTemplates/test/GrpcTemplateTest.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Testing;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
@@ -23,7 +24,8 @@ namespace Templates.Test
         public ProjectFactoryFixture ProjectFactory { get; }
         public ITestOutputHelper Output { get; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("Not supported queues", Queues = "Windows.7.Amd64;Windows.7.Amd64.Open;OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task GrpcTemplate()
         {
             Project = await ProjectFactory.GetOrCreateProject("grpc", Output);

src/ProjectTemplates/test/Helpers/ProcessEx.cs

@@ -5,6 +5,7 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
@@ -99,6 +100,11 @@ namespace Templates.Test.Helpers
             }
 
             startInfo.EnvironmentVariables["NUGET_PACKAGES"] = NUGET_PACKAGES;
+            
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix")))
+            {
+                startInfo.EnvironmentVariables["NUGET_FALLBACK_PACKAGES"] = Environment.GetEnvironmentVariable("NUGET_FALLBACK_PACKAGES");
+            }
 
             output.WriteLine($"==> {startInfo.FileName} {startInfo.Arguments} [{startInfo.WorkingDirectory}]");
             var proc = Process.Start(startInfo);
@@ -189,11 +195,12 @@ namespace Templates.Test.Helpers
             }
         }
 
-        private static string GetNugetPackagesRestorePath() =>
+        private static string GetNugetPackagesRestorePath() => (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NUGET_RESTORE"))) 
-            typeof(ProcessEx).Assembly
+            ? typeof(ProcessEx).Assembly
                 .GetCustomAttributes<AssemblyMetadataAttribute>()
                 .First(attribute => attribute.Key == "TestPackageRestorePath")
-                .Value;
+                .Value
+            : Environment.GetEnvironmentVariable("NUGET_RESTORE");
 
         public void Dispose()
         {

src/ProjectTemplates/test/Helpers/Project.cs

@@ -24,7 +24,16 @@ namespace Templates.Test.Helpers
         public static bool IsCIEnvironment => typeof(Project).Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
             .Any(a => a.Key == "ContinuousIntegrationBuild");
 
-        public static string ArtifactsLogDir => GetAssemblyMetadata("ArtifactsLogDir");
+        public static string ArtifactsLogDir => (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX_DIR"))) 
+            ? GetAssemblyMetadata("ArtifactsLogDir")
+            : Path.Combine(Environment.GetEnvironmentVariable("HELIX_DIR"), "logs");
+        
+        // FIGURE OUT EF PATH
+        public static string DotNetEfFullPath => (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix"))) 
+            ? typeof(ProjectFactoryFixture).Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
+                .First(attribute => attribute.Key == "DotNetEfFullPath")
+                .Value
+            : Path.Combine("NuGetPackageRoot", "dotnet-ef/$(DotnetEfPackageVersion)/tools/netcoreapp3.1/any/dotnet-ef.dll");
 
         public SemaphoreSlim DotNetNewLock { get; set; }
         public SemaphoreSlim NodeLock { get; set; }
@@ -297,13 +306,7 @@ namespace Templates.Test.Helpers
 
         internal async Task<ProcessEx> RunDotNetEfCreateMigrationAsync(string migrationName)
         {
-            var assembly = typeof(ProjectFactoryFixture).Assembly;
+            var args = $"\"{DotNetEfFullPath}\" --verbose --no-build migrations add {migrationName}";
-
-            var dotNetEfFullPath = assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
-                .First(attribute => attribute.Key == "DotNetEfFullPath")
-                .Value;
-
-            var args = $"\"{dotNetEfFullPath}\" --verbose --no-build migrations add {migrationName}";
 
             // Only run one instance of 'dotnet new' at once, as a workaround for
             // https://github.com/aspnet/templating/issues/63
@@ -322,13 +325,7 @@ namespace Templates.Test.Helpers
 
         internal async Task<ProcessEx> RunDotNetEfUpdateDatabaseAsync()
         {
-            var assembly = typeof(ProjectFactoryFixture).Assembly;
+            var args = $"\"{DotNetEfFullPath}\" --verbose --no-build database update";
-
-            var dotNetEfFullPath = assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
-                .First(attribute => attribute.Key == "DotNetEfFullPath")
-                .Value;
-
-            var args = $"\"{dotNetEfFullPath}\" --verbose --no-build database update";
 
             // Only run one instance of 'dotnet new' at once, as a workaround for
             // https://github.com/aspnet/templating/issues/63

src/ProjectTemplates/test/Helpers/ProjectFactoryFixture.cs

@@ -61,9 +61,11 @@ namespace Templates.Test.Helpers
         }
 
         private static string GetTemplateFolderBasePath(Assembly assembly) =>
-            assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
+            (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX_DIR"))) 
+            ? assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
                 .Single(a => a.Key == "TestTemplateCreationFolder")
-                .Value;
+                .Value
+            : Path.Combine(Environment.GetEnvironmentVariable("HELIX_DIR"), "Templates", "BaseFolder");
 
         public void Dispose()
         {

src/ProjectTemplates/test/Helpers/TemplatePackageInstaller.cs

@@ -42,10 +42,12 @@ namespace Templates.Test.Helpers
             "Microsoft.AspNetCore.Blazor.Templates",
         };
 
-        public static string CustomHivePath { get; } = typeof(TemplatePackageInstaller)
+        public static string CustomHivePath { get; } = (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix"))) 
-            .Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
+                    ? typeof(TemplatePackageInstaller)
-            .Single(s => s.Key == "CustomTemplateHivePath").Value;
+                        .Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
-
+                        .Single(s => s.Key == "CustomTemplateHivePath").Value
+                    : Path.Combine("Hives", ".templateEngine");
+                                        
         public static async Task EnsureTemplatingEngineInitializedAsync(ITestOutputHelper output)
         {
             await InstallerLock.WaitAsync();
@@ -81,11 +83,19 @@ namespace Templates.Test.Helpers
 
         private static async Task InstallTemplatePackages(ITestOutputHelper output)
         {
-            var builtPackages = Directory.EnumerateFiles(
+            string packagesDir;
-                    typeof(TemplatePackageInstaller).Assembly
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix")))
+            {
+                packagesDir = ".";
+            }
+            else
+            {
+                packagesDir = typeof(TemplatePackageInstaller).Assembly
                     .GetCustomAttributes<AssemblyMetadataAttribute>()
-                    .Single(a => a.Key == "ArtifactsShippingPackagesDir").Value,
+                    .Single(a => a.Key == "ArtifactsShippingPackagesDir").Value;
-                    "*.nupkg")
+            }
+
+            var builtPackages = Directory.EnumerateFiles(packagesDir, "*Templates*.nupkg")
                 .Where(p => _templatePackages.Any(t => Path.GetFileName(p).StartsWith(t, StringComparison.OrdinalIgnoreCase)))
                 .ToArray();
 

src/ProjectTemplates/test/IdentityUIPackageTest.cs

@@ -5,6 +5,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Net;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Testing;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
@@ -117,8 +118,9 @@ namespace Templates.Test
             "Identity/lib/jquery-validation-unobtrusive/LICENSE.txt",
         };
 
-        [Theory]
+        [ConditionalTheory]
         [MemberData(nameof(MSBuildIdentityUIPackageOptions))]
+        [SkipOnHelix("ef restore no worky")]
         public async Task IdentityUIPackage_WorksWithDifferentOptions(IDictionary<string, string> packageOptions, string versionValidator, string[] expectedFiles)
         {
             Project = await ProjectFactory.GetOrCreateProject("identityuipackage" + string.Concat(packageOptions.Values), Output);

src/ProjectTemplates/test/MvcTemplateTest.cs

@@ -28,7 +28,8 @@ namespace Templates.Test
         [Fact]
         public async Task MvcTemplate_NoAuthFSharp() => await MvcTemplateCore(languageOverride: "F#");
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("cert failure", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task MvcTemplate_NoAuthCSharp() => await MvcTemplateCore(languageOverride: null);
 
         private async Task MvcTemplateCore(string languageOverride)
@@ -103,9 +104,10 @@ namespace Templates.Test
             }
         }
 
-        [Theory]
+        [ConditionalTheory]
         [InlineData(true)]
         [InlineData(false)]
+        [SkipOnHelix("ef restore no worky")]
         public async Task MvcTemplate_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("mvcindividual" + (useLocalDB ? "uld" : ""), Output);
@@ -220,7 +222,8 @@ namespace Templates.Test
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("razor compilation restore no worky")]
         public async Task MvcTemplate_RazorRuntimeCompilation_BuildsAndPublishes()
         {
             Project = await ProjectFactory.GetOrCreateProject("mvc_rc", Output);

src/ProjectTemplates/test/ProjectTemplates.Tests.csproj

@@ -10,10 +10,9 @@
 
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''" >true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
-
+    <SkipHelixArm>true</SkipHelixArm>
-    <!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
+    
-    <!-- https://github.com/dotnet/aspnetcore/issues/6857 -->
+    <!-- Some tests still do not work on Helix, so continue running these on azdo for now-->
-    <BuildHelixPayload>false</BuildHelixPayload>
     <SkipTests Condition="'$(RunTemplateTests)' == 'true'">false</SkipTests>
     <BaseOutputPath />
     <OutputPath />
@@ -24,12 +23,14 @@
     <TestPackageRestorePath>$([MSBuild]::EnsureTrailingSlash('$(RepoRoot)'))obj\template-restore\</TestPackageRestorePath>
     <TestTemplateTestsProps>TemplateTests.props</TestTemplateTestsProps>
     <GenerateLoggingTestingAssemblyAttributes>false</GenerateLoggingTestingAssemblyAttributes>
+    <TestDependsOnAspNetRuntime>true</TestDependsOnAspNetRuntime>
   </PropertyGroup>
 
   <ItemGroup>
     <EmbeddedResource Include="template-baselines.json" />
     <Compile Include="$(SharedSourceRoot)Process\*.cs" LinkBase="shared\Process" />
     <Compile Include="$(SharedSourceRoot)CertificateGeneration\**\*.cs" LinkBase="shared\CertificateGeneration" />
+    <Compile Include="$(SharedSourceRoot)test\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/ProjectTemplates/test/RazorClassLibraryTemplateTest.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
+using Microsoft.AspNetCore.Testing;
 
 namespace Templates.Test
 {
@@ -40,7 +41,8 @@ namespace Templates.Test
             Assert.True(0 == buildResult.ExitCode, ErrorMessages.GetFailedProcessMessage("build", Project, buildResult));
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("restore no worky")]
         public async Task RazorClassLibraryTemplateAsync()
         {
             Project = await ProjectFactory.GetOrCreateProject("razorclasslib", Output);

src/ProjectTemplates/test/RazorPagesTemplateTest.cs

@@ -25,7 +25,8 @@ namespace Templates.Test
 
         public ITestOutputHelper Output { get; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("Cert failures", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task RazorPagesTemplate_NoAuth()
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpagesnoauth", Output);
@@ -93,9 +94,10 @@ namespace Templates.Test
             }
         }
 
-        [Theory]
+        [ConditionalTheory]
         [InlineData(false)]
         [InlineData(true)]
+        [SkipOnHelix("ef restore no worky")]
         public async Task RazorPagesTemplate_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpagesindividual" + (useLocalDB ? "uld" : ""), Output);
@@ -210,7 +212,8 @@ namespace Templates.Test
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("runtime compliation restore no worky")]
         public async Task RazorPagesTemplate_RazorRuntimeCompilation_BuildsAndPublishes()
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpages_rc", Output);

src/ProjectTemplates/test/SpaTemplateTest/AngularTemplateTest.cs

@@ -15,15 +15,18 @@ namespace Templates.Test.SpaTemplateTest
         public AngularTemplateTest(ProjectFactoryFixture projectFactory, BrowserFixture browserFixture, ITestOutputHelper output)
             : base(projectFactory, browserFixture, output) { }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task AngularTemplate_Works()
             => SpaTemplateImplAsync("angularnoauth", "angular", useLocalDb: false, usesAuth: false);
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task AngularTemplate_IndividualAuth_Works()
             => SpaTemplateImplAsync("angularindividual", "angular", useLocalDb: false, usesAuth: true);
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task AngularTemplate_IndividualAuth_Works_LocalDb()
             => SpaTemplateImplAsync("angularindividualuld", "angular", useLocalDb: true, usesAuth: true);
     }

src/ProjectTemplates/test/SpaTemplateTest/ReactReduxTemplateTest.cs

@@ -3,6 +3,7 @@
 
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.E2ETesting;
+using Microsoft.AspNetCore.Testing;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
@@ -16,7 +17,8 @@ namespace Templates.Test.SpaTemplateTest
         {
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task ReactReduxTemplate_Works_NetCore()
             => SpaTemplateImplAsync("reactredux", "reactredux", useLocalDb: false, usesAuth: false);
     }

src/ProjectTemplates/test/SpaTemplateTest/ReactTemplateTest.cs

@@ -17,15 +17,18 @@ namespace Templates.Test.SpaTemplateTest
         {
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task ReactTemplate_Works_NetCore()
             => SpaTemplateImplAsync("reactnoauth", "react", useLocalDb: false, usesAuth: false);
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task ReactTemplate_IndividualAuth_NetCore()
             => SpaTemplateImplAsync("reactindividual", "react", useLocalDb: false, usesAuth: true);
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public Task ReactTemplate_IndividualAuth_NetCore_LocalDb()
             => SpaTemplateImplAsync("reactindividualuld", "react", useLocalDb: true, usesAuth: true);
     }

src/ProjectTemplates/test/WebApiTemplateTest.cs

@@ -3,6 +3,7 @@
 
 using System.Threading.Tasks;
 using Templates.Test.Helpers;
+using Microsoft.AspNetCore.Testing;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -25,7 +26,8 @@ namespace Templates.Test
         [Fact]
         public async Task WebApiTemplateFSharp() => await WebApiTemplateCore(languageOverride: "F#");
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("Cert failures", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task WebApiTemplateCSharp() => await WebApiTemplateCore(languageOverride: null);
 
         private async Task WebApiTemplateCore(string languageOverride)

src/ProjectTemplates/test/WorkerTemplateTest.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
+using Microsoft.AspNetCore.Testing;
 
 namespace Templates.Test
 {
@@ -20,7 +21,8 @@ namespace Templates.Test
         public ProjectFactoryFixture ProjectFactory { get; }
         public ITestOutputHelper Output { get; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("restore no worky")]        
         public async Task WorkerTemplateAsync()
         {
             Project = await ProjectFactory.GetOrCreateProject("worker", Output);

src/Shared/E2ETesting/SeleniumStandaloneServer.cs

@@ -113,6 +113,13 @@ namespace Microsoft.AspNetCore.E2ETesting
             // It's important that we get the folder value before we start the process to prevent
             // untracked processes when the tracking folder is not correctly configure.
             var trackingFolder = GetProcessTrackingFolder();
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix")))
+            {
+                // Just create a random tracking folder on helix
+                trackingFolder = Path.Combine(Directory.GetCurrentDirectory(), Path.GetRandomFileName());
+                Directory.CreateDirectory(trackingFolder);
+            }
+            
             if (!Directory.Exists(trackingFolder))
             {
                 throw new InvalidOperationException($"Invalid tracking folder. Set the 'SeleniumProcessTrackingFolder' MSBuild property to a valid folder.");