[Helix] Reenable some template tests again (#19520)

eng/helix/content/runtests.cmd

@@ -1,6 +1,6 @@
 @echo off
-REM Disable "!Foo!" expansions because they break the filter syntax
+REM Need delayed expansion !PATH! so parens in the path don't mess up the parens for the if statements that use parens for blocks
-setlocal disableextensions
+setlocal enabledelayedexpansion
 
 REM Use '$' as a variable name prefix to avoid MSBuild variable collisions with these variables
 set $target=%1
@@ -9,6 +9,7 @@ set $runtimeVersion=%3
 set $helixQueue=%4
 set $arch=%5
 set $quarantined=%6
+set $efVersion=%7
 
 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
 set DOTNET_ROOT=%DOTNET_HOME%\%$arch%
@@ -16,14 +17,24 @@ set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 set DOTNET_MULTILEVEL_LOOKUP=0
 set DOTNET_CLI_HOME=%HELIX_CORRELATION_PAYLOAD%\home
 
-set PATH=%DOTNET_ROOT%;%PATH%;%HELIX_CORRELATION_PAYLOAD%\node\bin
+set PATH=%DOTNET_ROOT%;!PATH!;%HELIX_CORRELATION_PAYLOAD%\node\bin
-
+echo Set path to: %PATH%
+echo "Installing SDK"
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Version %$sdkVersion% -InstallDir %DOTNET_ROOT%"
+echo "Installing Runtime"
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Runtime dotnet -Version %$runtimeVersion% -InstallDir %DOTNET_ROOT%"
-
+echo "Checking for Microsoft.AspNetCore.App"
 if EXIST ".\Microsoft.AspNetCore.App" (
     echo "Found Microsoft.AspNetCore.App, copying to %DOTNET_ROOT%\shared\Microsoft.AspNetCore.App\%runtimeVersion%"
     xcopy /i /y ".\Microsoft.AspNetCore.App" %DOTNET_ROOT%\shared\Microsoft.AspNetCore.App\%runtimeVersion%\
+    
+    echo "Adding current directory to nuget sources: %HELIX_WORKITEM_ROOT%"
+    dotnet nuget add source %HELIX_WORKITEM_ROOT%
+    dotnet nuget add source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json
+    dotnet nuget list source
+    dotnet tool install dotnet-ef --global --version %$efVersion%
+    
+    set PATH=!PATH!;%DOTNET_CLI_HOME%\.dotnet\tools
 )
 
 echo "Current Directory: %HELIX_WORKITEM_ROOT%"
@@ -31,6 +42,8 @@ set HELIX=%$helixQueue%
 set HELIX_DIR=%HELIX_WORKITEM_ROOT%
 set NUGET_FALLBACK_PACKAGES=%HELIX_DIR%
 set NUGET_RESTORE=%HELIX_DIR%\nugetRestore
+set DotNetEfFullPath=%HELIX_DIR%\nugetRestore\dotnet-ef\%$efVersion%\tools\netcoreapp3.1\any\dotnet-ef.exe
+echo "Set DotNetEfFullPath: %DotNetEfFullPath%"
 echo "Setting HELIX_DIR: %HELIX_DIR%"
 echo Creating nuget restore directory: %NUGET_RESTORE%
 mkdir %NUGET_RESTORE%
@@ -53,6 +66,8 @@ if %$quarantined%==True (
     set %$quarantined=true
 )
 
+REM Disable "!Foo!" expansions because they break the filter syntax
+setlocal disabledelayedexpansion
 set NONQUARANTINE_FILTER="Quarantined!=true"
 set QUARANTINE_FILTER="Quarantined=true"
 if %$quarantined%==true (

eng/helix/content/runtests.sh

@@ -6,6 +6,7 @@ dotnet_runtime_version="$3"
 helix_queue_name="$4"
 target_arch="$5"
 quarantined="$6"
+efVersion="$7"
 
 RESET="\033[0m"
 RED="\033[0;31m"
@@ -33,6 +34,8 @@ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 export helix="$helix_queue_name"
 export HELIX_DIR="$DIR"
 export NUGET_FALLBACK_PACKAGES="$DIR"
+export DotNetEfFullPath=$DIR\nugetRestore\dotnet-ef\$efVersion\tools\netcoreapp3.1\any\dotnet-ef.dll
+echo "Set DotNetEfFullPath: $DotNetEfFullPath"
 export NUGET_RESTORE="$DIR/nugetRestore"
 echo "Creating nugetRestore directory: $NUGET_RESTORE"
 mkdir $NUGET_RESTORE
@@ -95,6 +98,17 @@ if [ -d "Microsoft.AspNetCore.App" ]
 then
     echo "Found Microsoft.AspNetCore.App directory, copying to $DOTNET_ROOT/shared/Microsoft.AspNetCore.App/$dotnet_runtime_version."
     cp -r Microsoft.AspNetCore.App $DOTNET_ROOT/shared/Microsoft.AspNetCore.App/$dotnet_runtime_version
+    
+    echo "Adding current directory to nuget sources: $DIR"
+    dotnet nuget add source $DIR
+    dotnet nuget add source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json
+    dotnet nuget list source
+    
+    dotnet tool install dotnet-ef --global --version $efVersion
+    
+    # Ensure tools are on on PATH
+    export PATH="$PATH:$DOTNET_CLI_HOME/.dotnet/tools"
+    
 fi
 
 if [ -e /proc/self/coredump_filter ]; then

eng/targets/Helix.targets

@@ -120,8 +120,8 @@ Usage: dotnet msbuild /t:Helix src/MyTestProject.csproj
         <TestAssembly>$(TargetFileName)</TestAssembly>
         <PreCommands>@(HelixPreCommand)</PreCommands>
         <PostCommands>@(HelixPostCommand)</PostCommands>
-        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests)</Command>
+        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests) $(DotnetEfPackageVersion)</Command>
-        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests) $(DotnetEfPackageVersion)</Command>
         <Command Condition="$(HelixCommand) != ''">$(HelixCommand)</Command>
         <Timeout>$(HelixTimeout)</Timeout>
       </HelixWorkItem>

src/ProjectTemplates/test/Helpers/Project.cs

@@ -7,6 +7,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.CommandLineUtils;
@@ -28,12 +29,11 @@ namespace Templates.Test.Helpers
             ? GetAssemblyMetadata("ArtifactsLogDir")
             : Path.Combine(Environment.GetEnvironmentVariable("HELIX_DIR"), "logs");
         
-        // FIGURE OUT EF PATH
+        public static string DotNetEfFullPath => (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DotNetEfFullPath"))) 
-        public static string DotNetEfFullPath => (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix"))) 
             ? typeof(ProjectFactoryFixture).Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
                 .First(attribute => attribute.Key == "DotNetEfFullPath")
                 .Value
-            : Path.Combine("NuGetPackageRoot", "dotnet-ef/$(DotnetEfPackageVersion)/tools/netcoreapp3.1/any/dotnet-ef.dll");
+            : Environment.GetEnvironmentVariable("DotNetEfFullPath");
 
         public SemaphoreSlim DotNetNewLock { get; set; }
         public SemaphoreSlim NodeLock { get; set; }
@@ -306,14 +306,24 @@ namespace Templates.Test.Helpers
 
         internal async Task<ProcessEx> RunDotNetEfCreateMigrationAsync(string migrationName)
         {
-            var args = $"\"{DotNetEfFullPath}\" --verbose --no-build migrations add {migrationName}";
+            var args = $"--verbose --no-build migrations add {migrationName}";
-
+            
             // Only run one instance of 'dotnet new' at once, as a workaround for
             // https://github.com/aspnet/templating/issues/63
             await DotNetNewLock.WaitAsync();
             try
             {
-                var result = ProcessEx.Run(Output, TemplateOutputDir, DotNetMuxer.MuxerPathOrDefault(), args);
+                var command = DotNetMuxer.MuxerPathOrDefault();
+                if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DotNetEfFullPath")))
+                {
+                    args = $"\"{DotNetEfFullPath}\" " + args;
+                }
+                else
+                {
+                    command = "dotnet-ef";
+                }
+                
+                var result = ProcessEx.Run(Output, TemplateOutputDir, command, args);
                 await result.Exited;
                 return result;
             }
@@ -325,14 +335,24 @@ namespace Templates.Test.Helpers
 
         internal async Task<ProcessEx> RunDotNetEfUpdateDatabaseAsync()
         {
-            var args = $"\"{DotNetEfFullPath}\" --verbose --no-build database update";
+            var args = "--verbose --no-build database update";
 
             // Only run one instance of 'dotnet new' at once, as a workaround for
             // https://github.com/aspnet/templating/issues/63
             await DotNetNewLock.WaitAsync();
             try
             {
-                var result = ProcessEx.Run(Output, TemplateOutputDir, DotNetMuxer.MuxerPathOrDefault(), args);
+                var command = DotNetMuxer.MuxerPathOrDefault();
+                if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DotNetEfFullPath")))
+                {
+                    args = $"\"{DotNetEfFullPath}\" " + args;
+                }
+                else
+                {
+                    command = "dotnet-ef";
+                }
+                
+                var result = ProcessEx.Run(Output, TemplateOutputDir, command, args);
                 await result.Exited;
                 return result;
             }

src/ProjectTemplates/test/IdentityUIPackageTest.cs

@@ -120,7 +120,7 @@ namespace Templates.Test
 
         [ConditionalTheory]
         [MemberData(nameof(MSBuildIdentityUIPackageOptions))]
-        [SkipOnHelix("ef restore no worky")]
+        [SkipOnHelix("cert failure", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task IdentityUIPackage_WorksWithDifferentOptions(IDictionary<string, string> packageOptions, string versionValidator, string[] expectedFiles)
         {
             Project = await ProjectFactory.GetOrCreateProject("identityuipackage" + string.Concat(packageOptions.Values), Output);

src/ProjectTemplates/test/MvcTemplateTest.cs

@@ -3,12 +3,12 @@
 
 using System.Collections.Generic;
 using System.IO;
-using System.Threading.Tasks;
-using Templates.Test.Helpers;
 using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Testing;
+using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
-using Microsoft.AspNetCore.Testing;
 
 namespace Templates.Test
 {
@@ -107,7 +107,7 @@ namespace Templates.Test
         [ConditionalTheory]
         [InlineData(true)]
         [InlineData(false)]
-        [SkipOnHelix("ef restore no worky")]
+        [SkipOnHelix("cert failure", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task MvcTemplate_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("mvcindividual" + (useLocalDB ? "uld" : ""), Output);
@@ -222,8 +222,7 @@ namespace Templates.Test
             }
         }
 
-        [ConditionalFact]
+        [Fact]
-        [SkipOnHelix("razor compilation restore no worky")]
         public async Task MvcTemplate_RazorRuntimeCompilation_BuildsAndPublishes()
         {
             Project = await ProjectFactory.GetOrCreateProject("mvc_rc", Output);

src/ProjectTemplates/test/RazorClassLibraryTemplateTest.cs

@@ -5,7 +5,6 @@ using System.Threading.Tasks;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
-using Microsoft.AspNetCore.Testing;
 
 namespace Templates.Test
 {
@@ -41,8 +40,7 @@ namespace Templates.Test
             Assert.True(0 == buildResult.ExitCode, ErrorMessages.GetFailedProcessMessage("build", Project, buildResult));
         }
 
-        [ConditionalFact]
+        [Fact]
-        [SkipOnHelix("restore no worky")]
         public async Task RazorClassLibraryTemplateAsync()
         {
             Project = await ProjectFactory.GetOrCreateProject("razorclasslib", Output);

src/ProjectTemplates/test/RazorPagesTemplateTest.cs

@@ -97,7 +97,7 @@ namespace Templates.Test
         [ConditionalTheory]
         [InlineData(false)]
         [InlineData(true)]
-        [SkipOnHelix("ef restore no worky")]
+        [SkipOnHelix("cert failure", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]        
         public async Task RazorPagesTemplate_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpagesindividual" + (useLocalDB ? "uld" : ""), Output);
@@ -212,8 +212,7 @@ namespace Templates.Test
             }
         }
 
-        [ConditionalFact]
+        [Fact]
-        [SkipOnHelix("runtime compliation restore no worky")]
         public async Task RazorPagesTemplate_RazorRuntimeCompilation_BuildsAndPublishes()
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpages_rc", Output);

src/ProjectTemplates/test/WorkerTemplateTest.cs

@@ -21,8 +21,7 @@ namespace Templates.Test
         public ProjectFactoryFixture ProjectFactory { get; }
         public ITestOutputHelper Output { get; }
 
-        [ConditionalFact]
+        [Fact]
-        [SkipOnHelix("restore no worky")]        
         public async Task WorkerTemplateAsync()
         {
             Project = await ProjectFactory.GetOrCreateProject("worker", Output);