Template test infrastructure fixups

* Add timeouts to process launches and lock acquisitions
* Dispose launched processes
* Remove unused code

src/ProjectTemplates/Shared/AspNetProcess.cs

@@ -183,28 +183,6 @@ namespace Templates.Test.Helpers
             }
         }
 
-        private async Task<T> RequestWithRetries<T>(Func<HttpClient, Task<T>> requester, HttpClient client, int retries = 3, TimeSpan initialDelay = default)
-        {
-            var currentDelay = initialDelay == default ? TimeSpan.FromSeconds(30) : initialDelay;
-            for (int i = 0; i <= retries; i++)
-            {
-                try
-                {
-                    return await requester(client);
-                }
-                catch (Exception)
-                {
-                    if (i == retries)
-                    {
-                        throw;
-                    }
-                    await Task.Delay(currentDelay);
-                    currentDelay *= 2;
-                }
-            }
-            throw new InvalidOperationException("Max retries reached.");
-        }
-
         private Uri ResolveListeningUrl(ITestOutputHelper output)
         {
             // Wait until the app is accepting HTTP requests

src/ProjectTemplates/Shared/ErrorMessages.cs

@@ -7,16 +7,16 @@ namespace Templates.Test.Helpers
 {
     internal static class ErrorMessages
     {
-        public static string GetFailedProcessMessage(string step, Project project, ProcessEx processResult)
+        public static string GetFailedProcessMessage(string step, Project project, ProcessResult processResult)
         {
-            return $@"Project {project.ProjectArguments} failed to {step}.
+            return $@"Project {project.ProjectArguments} failed to {step}. Exit code {processResult.ExitCode}.
-{processResult.GetFormattedOutput()}";
+{processResult.Process}\nStdErr: {processResult.Error}\nStdOut: {processResult.Output}";
         }
 
-        public static string GetFailedProcessMessageOrEmpty(string step, Project project, ProcessEx processResult)
+        public static string GetFailedProcessMessageOrEmpty(string step, Project project, ProcessEx process)
         {
-            return processResult.HasExited ? $@"Project {project.ProjectArguments} failed to {step}.
+            return process.HasExited ? $@"Project {project.ProjectArguments} failed to {step}.
-{processResult.GetFormattedOutput()}" : "";
+{process.GetFormattedOutput()}" : "";
         }
     }
 }

src/ProjectTemplates/Shared/ProcessLock.cs

@@ -0,0 +1,36 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace Templates.Test.Helpers
+{
+    public class ProcessLock
+    {
+        public static readonly ProcessLock DotNetNewLock = new ProcessLock("dotnet-new");
+        public static readonly ProcessLock NodeLock = new ProcessLock("node");
+
+        public ProcessLock(string name)
+        {
+            Name = name;
+            Semaphore = new SemaphoreSlim(1);
+        }
+
+        public string Name { get; }
+        private SemaphoreSlim Semaphore { get; }
+
+        public async Task WaitAsync(TimeSpan? timeout = null)
+        {
+            timeout ??= TimeSpan.FromMinutes(2);
+            Assert.True(await Semaphore.WaitAsync(timeout.Value), $"Unable to acquire process lock for process {Name}");
+        }
+
+        public void Release()
+        {
+            Semaphore.Release();
+        }
+    }
+}

src/ProjectTemplates/Shared/ProcessResult.cs

@@ -0,0 +1,26 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.AspNetCore.Internal;
+
+namespace Templates.Test.Helpers
+{
+    internal class ProcessResult
+    {
+        public ProcessResult(ProcessEx process)
+        {
+            Process = process.Process.StartInfo.FileName + " " + process.Process.StartInfo.Arguments;
+            ExitCode = process.ExitCode;
+            Output = process.Output;
+            Error = process.Error;
+        }
+
+        public string Process { get; }
+
+        public int ExitCode { get; }
+
+        public string Error { get; }
+
+        public string Output { get; }
+    }
+}

src/ProjectTemplates/Shared/Project.cs

@@ -15,11 +15,12 @@ using Microsoft.Extensions.Logging;
 using Xunit;
 using Xunit.Abstractions;
 using Xunit.Sdk;
+using static Templates.Test.Helpers.ProcessLock;
 
 namespace Templates.Test.Helpers
 {
     [DebuggerDisplay("{ToString(),nq}")]
-    public class Project
+    public class Project : IDisposable
     {
         private const string _urls = "http://127.0.0.1:0;https://127.0.0.1:0";
 
@@ -33,8 +34,6 @@ namespace Templates.Test.Helpers
                 .Value
             : Environment.GetEnvironmentVariable("DotNetEfFullPath");
 
-        public SemaphoreSlim DotNetNewLock { get; set; }
-        public SemaphoreSlim NodeLock { get; set; }
         public string ProjectName { get; set; }
         public string ProjectArguments { get; set; }
         public string ProjectGuid { get; set; }
@@ -47,7 +46,7 @@ namespace Templates.Test.Helpers
         public ITestOutputHelper Output { get; set; }
         public IMessageSink DiagnosticsMessageSink { get; set; }
 
-        internal async Task<ProcessEx> RunDotNetNewAsync(
+        internal async Task<ProcessResult> RunDotNetNewAsync(
             string templateName,
             string auth = null,
             string language = null,
@@ -100,9 +99,9 @@ namespace Templates.Test.Helpers
             await DotNetNewLock.WaitAsync();
             try
             {
-                var execution = ProcessEx.Run(Output, AppContext.BaseDirectory, DotNetMuxer.MuxerPathOrDefault(), argString, environmentVariables);
+                using var execution = ProcessEx.Run(Output, AppContext.BaseDirectory, DotNetMuxer.MuxerPathOrDefault(), argString, environmentVariables);
                 await execution.Exited;
-                return execution;
+                return new ProcessResult(execution);
             }
             finally
             {
@@ -110,7 +109,7 @@ namespace Templates.Test.Helpers
             }
         }
 
-        internal async Task<ProcessEx> RunDotNetPublishAsync(bool takeNodeLock = false, IDictionary<string, string> packageOptions = null, string additionalArgs = null)
+        internal async Task<ProcessResult> RunDotNetPublishAsync(bool takeNodeLock = false, IDictionary<string, string> packageOptions = null, string additionalArgs = null)
         {
             Output.WriteLine("Publishing ASP.NET application...");
 
@@ -121,10 +120,10 @@ namespace Templates.Test.Helpers
             await effectiveLock.WaitAsync();
             try
             {
-                var result = ProcessEx.Run(Output, TemplateOutputDir, DotNetMuxer.MuxerPathOrDefault(), $"publish -c Release /bl {additionalArgs}", packageOptions);
+                using var result = ProcessEx.Run(Output, TemplateOutputDir, DotNetMuxer.MuxerPathOrDefault(), $"publish -c Release /bl {additionalArgs}", packageOptions);
                 await result.Exited;
                 CaptureBinLogOnFailure(result);
-                return result;
+                return new ProcessResult(result);
             }
             finally
             {
@@ -132,7 +131,7 @@ namespace Templates.Test.Helpers
             }
         }
 
-        internal async Task<ProcessEx> RunDotNetBuildAsync(bool takeNodeLock = false, IDictionary<string, string> packageOptions = null, string additionalArgs = null)
+        internal async Task<ProcessResult> RunDotNetBuildAsync(bool takeNodeLock = false, IDictionary<string, string> packageOptions = null, string additionalArgs = null)
         {
             Output.WriteLine("Building ASP.NET application...");
 
@@ -143,10 +142,10 @@ namespace Templates.Test.Helpers
             await effectiveLock.WaitAsync();
             try
             {
-                var result = ProcessEx.Run(Output, TemplateOutputDir, DotNetMuxer.MuxerPathOrDefault(), $"build -c Debug /bl {additionalArgs}", packageOptions);
+                using var result = ProcessEx.Run(Output, TemplateOutputDir, DotNetMuxer.MuxerPathOrDefault(), $"build -c Debug /bl {additionalArgs}", packageOptions);
                 await result.Exited;
                 CaptureBinLogOnFailure(result);
-                return result;
+                return new ProcessResult(result);
             }
             finally
             {
@@ -185,7 +184,7 @@ namespace Templates.Test.Helpers
             return new AspNetProcess(Output, TemplatePublishDir, projectDll, environment, hasListeningUri: hasListeningUri);
         }
 
-        internal async Task<ProcessEx> RunDotNetEfCreateMigrationAsync(string migrationName)
+        internal async Task<ProcessResult> RunDotNetEfCreateMigrationAsync(string migrationName)
         {
             var args = $"--verbose --no-build migrations add {migrationName}";
 
@@ -204,9 +203,9 @@ namespace Templates.Test.Helpers
                     command = "dotnet-ef";
                 }
 
-                var result = ProcessEx.Run(Output, TemplateOutputDir, command, args);
+                using var result = ProcessEx.Run(Output, TemplateOutputDir, command, args);
                 await result.Exited;
-                return result;
+                return new ProcessResult(result);
             }
             finally
             {
@@ -320,14 +319,14 @@ namespace Templates.Test.Helpers
             private bool _nodeLockTaken;
             private bool _dotNetLockTaken;
 
-            public OrderedLock(SemaphoreSlim nodeLock, SemaphoreSlim dotnetLock)
+            public OrderedLock(ProcessLock nodeLock, ProcessLock dotnetLock)
             {
                 NodeLock = nodeLock;
                 DotnetLock = dotnetLock;
             }
 
-            public SemaphoreSlim NodeLock { get; }
+            public ProcessLock NodeLock { get; }
-            public SemaphoreSlim DotnetLock { get; }
+            public ProcessLock DotnetLock { get; }
 
             public async Task WaitAsync()
             {

src/ProjectTemplates/Shared/ProjectFactoryFixture.cs

@@ -7,7 +7,6 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Reflection;
-using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.E2ETesting;
 using Xunit.Abstractions;
@@ -16,9 +15,6 @@ namespace Templates.Test.Helpers
 {
     public class ProjectFactoryFixture : IDisposable
     {
-        private readonly static SemaphoreSlim DotNetNewLock = new SemaphoreSlim(1);
-        private readonly static SemaphoreSlim NodeLock = new SemaphoreSlim(1);
-
         private readonly ConcurrentDictionary<string, Project> _projects = new ConcurrentDictionary<string, Project>();
 
         public IMessageSink DiagnosticsMessageSink { get; }
@@ -44,8 +40,6 @@ namespace Templates.Test.Helpers
                 {
                     var project = new Project
                     {
-                        DotNetNewLock = DotNetNewLock,
-                        NodeLock = NodeLock,
                         Output = outputHelper,
                         DiagnosticsMessageSink = DiagnosticsMessageSink,
                         ProjectGuid = Path.GetRandomFileName().Replace(".", string.Empty)
@@ -53,7 +47,7 @@ namespace Templates.Test.Helpers
                     project.ProjectName = $"AspNet.{key}.{project.ProjectGuid}";
 
                     var assemblyPath = GetType().Assembly;
-                    string basePath = GetTemplateFolderBasePath(assemblyPath);
+                    var basePath = GetTemplateFolderBasePath(assemblyPath);
                     project.TemplateOutputDir = Path.Combine(basePath, project.ProjectName);
                     return project;
                 },

src/ProjectTemplates/Shared/TemplatePackageInstaller.cs

@@ -9,6 +9,7 @@ using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Internal;
+using Microsoft.AspNetCore.Testing;
 using Microsoft.Extensions.CommandLineUtils;
 using Xunit;
 using Xunit.Abstractions;
@@ -17,7 +18,7 @@ namespace Templates.Test.Helpers
 {
     internal static class TemplatePackageInstaller
     {
-        private static SemaphoreSlim InstallerLock = new SemaphoreSlim(1);
+        private static readonly SemaphoreSlim InstallerLock = new SemaphoreSlim(1);
         private static bool _haveReinstalledTemplatePackages;
 
         private static readonly string[] _templatePackages = new[]
@@ -43,15 +44,15 @@ namespace Templates.Test.Helpers
             "Microsoft.AspNetCore.Blazor.Templates",
         };
 
-        public static string CustomHivePath { get; } = (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix"))) 
+        public static string CustomHivePath { get; } = (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix")))
-                    ? typeof(TemplatePackageInstaller)
+                     ? typeof(TemplatePackageInstaller)
-                        .Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
+                         .Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
-                        .Single(s => s.Key == "CustomTemplateHivePath").Value
+                         .Single(s => s.Key == "CustomTemplateHivePath").Value
-                    : Path.Combine("Hives", ".templateEngine");
+                     : Path.Combine("Hives", ".templateEngine");
-                                        
+
         public static async Task EnsureTemplatingEngineInitializedAsync(ITestOutputHelper output)
         {
-            await InstallerLock.WaitAsync();
+            Assert.True(await InstallerLock.WaitAsync(TimeSpan.FromMinutes(1)), "Unable to grab installer lock");
             try
             {
                 if (!_haveReinstalledTemplatePackages)
@@ -73,10 +74,11 @@ namespace Templates.Test.Helpers
         public static async Task<ProcessEx> RunDotNetNew(ITestOutputHelper output, string arguments)
         {
             var proc = ProcessEx.Run(
-            output,
+                output,
-            AppContext.BaseDirectory,
+                AppContext.BaseDirectory,
-            DotNetMuxer.MuxerPathOrDefault(),
+                DotNetMuxer.MuxerPathOrDefault(),
-            $"new {arguments} --debug:custom-hive \"{CustomHivePath}\"");
+                $"new {arguments} --debug:custom-hive \"{CustomHivePath}\"");
+
             await proc.Exited;
 
             return proc;

src/ProjectTemplates/test/EmptyWebTemplateTest.cs

@@ -23,7 +23,7 @@ namespace Templates.Test
 
         public ITestOutputHelper Output { get; }
 
-        [ConditionalFact(Skip = "This test ran for over an hour")]
+        [ConditionalFact]
         [SkipOnHelix("Cert failures", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task EmptyWebTemplateCSharp()
         {

src/ProjectTemplates/test/yarn.lock

@@ -427,10 +427,10 @@ safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
 
-selenium-standalone@^6.15.4:
+selenium-standalone@^6.17.0:
-  version "6.15.6"
+  version "6.17.0"
-  resolved "https://registry.yarnpkg.com/selenium-standalone/-/selenium-standalone-6.15.6.tgz#55cc610e405dd74e312c77f3ac7336795ca2cd0c"
+  resolved "https://registry.yarnpkg.com/selenium-standalone/-/selenium-standalone-6.17.0.tgz#0f24b691836205ee9bc3d7a6f207ebcb28170cd9"
-  integrity sha512-iQJbjJyGY4+n9voj+QIxkjEKmk9csPL91pxjqYyX5/1jjJDw7ce0ZuTnLeNaFmFdjY/nAwcoMRvqpDPmmmyr5A==
+  integrity sha512-5PSnDHwMiq+OCiAGlhwQ8BM9xuwFfvBOZ7Tfbw+ifkTnOy0PWbZmI1B9gPGuyGHpbQ/3J3CzIK7BYwrQ7EjtWQ==
   dependencies:
     async "^2.6.2"
     commander "^2.19.0"

src/Shared/Process/ProcessEx.cs

@@ -5,10 +5,8 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
 using System.Linq;
 using System.Reflection;
-using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -19,6 +17,7 @@ namespace Microsoft.AspNetCore.Internal
 {
     internal class ProcessEx : IDisposable
     {
+        private static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes(15);
         private static readonly string NUGET_PACKAGES = GetNugetPackagesRestorePath();
 
         private readonly ITestOutputHelper _output;
@@ -28,11 +27,12 @@ namespace Microsoft.AspNetCore.Internal
         private readonly object _pipeCaptureLock = new object();
         private readonly object _testOutputLock = new object();
         private BlockingCollection<string> _stdoutLines;
-        private TaskCompletionSource<int> _exited;
+        private readonly TaskCompletionSource<int> _exited;
-        private CancellationTokenSource _stdoutLinesCancellationSource = new CancellationTokenSource(TimeSpan.FromMinutes(5));
+        private readonly CancellationTokenSource _stdoutLinesCancellationSource = new CancellationTokenSource(TimeSpan.FromMinutes(5));
+        private readonly CancellationTokenSource _processTimeoutCts;
         private bool _disposed = false;
 
-        public ProcessEx(ITestOutputHelper output, Process proc)
+        private ProcessEx(ITestOutputHelper output, Process proc, TimeSpan timeout)
         {
             _output = output;
             _stdoutCapture = new StringBuilder();
@@ -48,8 +48,16 @@ namespace Microsoft.AspNetCore.Internal
             proc.BeginErrorReadLine();
 
             _exited = new TaskCompletionSource<int>(TaskCreationOptions.RunContinuationsAsynchronously);
+
+            _processTimeoutCts = new CancellationTokenSource(timeout);
+            _processTimeoutCts.Token.Register(() =>
+            {
+                _exited.TrySetException(new TimeoutException($"Process proc {proc.ProcessName} {proc.StartInfo.Arguments} timed out after {DefaultProcessTimeout}."));
+            });
         }
 
+        public Process Process => _process;
+
         public Task Exited => _exited.Task;
 
         public bool HasExited => _process.HasExited;
@@ -82,7 +90,7 @@ namespace Microsoft.AspNetCore.Internal
 
         public object Id => _process.Id;
 
-        public static ProcessEx Run(ITestOutputHelper output, string workingDirectory, string command, string args = null, IDictionary<string, string> envVars = null)
+        public static ProcessEx Run(ITestOutputHelper output, string workingDirectory, string command, string args = null, IDictionary<string, string> envVars = null, TimeSpan? timeout = default)
         {
             var startInfo = new ProcessStartInfo(command, args)
             {
@@ -111,18 +119,7 @@ namespace Microsoft.AspNetCore.Internal
             output.WriteLine($"==> {startInfo.FileName} {startInfo.Arguments} [{startInfo.WorkingDirectory}]");
             var proc = Process.Start(startInfo);
 
-            return new ProcessEx(output, proc);
+            return new ProcessEx(output, proc, timeout ?? DefaultProcessTimeout);
-        }
-
-        public static ProcessEx RunViaShell(ITestOutputHelper output, string workingDirectory, string commandAndArgs)
-        {
-            var (shellExe, argsPrefix) = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-                ? ("cmd", "/c")
-                : ("bash", "-c");
-
-            var result = Run(output, workingDirectory, shellExe, $"{argsPrefix} \"{commandAndArgs}\"");
-            result.WaitForExit(assertSuccess: false);
-            return result;
         }
 
         private void OnErrorData(object sender, DataReceivedEventArgs e)
@@ -218,6 +215,8 @@ namespace Microsoft.AspNetCore.Internal
 
         public void Dispose()
         {
+            _processTimeoutCts.Dispose();
+
             lock (_testOutputLock)
             {
                 _disposed = true;