From 20f6d6582896344cad45adb82cccc445d1e8e762 Mon Sep 17 00:00:00 2001 From: John Luo Date: Wed, 25 Mar 2020 16:00:27 -0700 Subject: [PATCH] Fix flakiness/timeout in gRPC template tests (#19982) Do not search for port number for cases where we are testing for failure. --- src/ProjectTemplates/test/GrpcTemplateTest.cs | 28 ++++++++++++++----- .../test/Helpers/AspNetProcess.cs | 12 +++++++- src/ProjectTemplates/test/Helpers/Project.cs | 5 ++-- .../test/ProjectTemplates.Tests.csproj | 10 +++++++ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/ProjectTemplates/test/GrpcTemplateTest.cs b/src/ProjectTemplates/test/GrpcTemplateTest.cs index 5104dc42b7..371c0022d3 100644 --- a/src/ProjectTemplates/test/GrpcTemplateTest.cs +++ b/src/ProjectTemplates/test/GrpcTemplateTest.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Reflection; using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.AspNetCore.Testing; @@ -24,11 +25,16 @@ namespace Templates.Test public ProjectFactoryFixture ProjectFactory { get; } public ITestOutputHelper Output { get; } - [ConditionalFact(Skip = "This test run for over an hour")] + [ConditionalFact] [SkipOnHelix("Not supported queues", Queues = "Windows.7.Amd64;Windows.7.Amd64.Open;OSX.1014.Amd64;OSX.1014.Amd64.Open")] [QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/19716")] public async Task GrpcTemplate() { + // Setup AssemblyTestLog + var assemblyLog = AssemblyTestLog.Create(Assembly.GetExecutingAssembly(), baseDirectory: Project.ArtifactsLogDir); + using var testLog = assemblyLog.StartTestLog(Output, nameof(GrpcTemplateTest), out var loggerFactory); + var logger = loggerFactory.CreateLogger("TestLogger"); + Project = await ProjectFactory.GetOrCreateProject("grpc", Output); var createResult = await Project.RunDotNetNewAsync("grpc"); @@ -40,18 +46,24 @@ namespace Templates.Test var buildResult = await Project.RunDotNetBuildAsync(); Assert.True(0 == buildResult.ExitCode, ErrorMessages.GetFailedProcessMessage("build", Project, buildResult)); - using (var serverProcess = Project.StartBuiltProjectAsync()) + var isOsx = RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + var isWindowsOld = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version < new Version(6, 2); + var unsupported = isOsx || isWindowsOld; + + using (var serverProcess = Project.StartBuiltProjectAsync(hasListeningUri: !unsupported, logger: logger)) { // These templates are HTTPS + HTTP/2 only which is not supported on Mac due to missing ALPN support. // https://github.com/dotnet/aspnetcore/issues/11061 - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + if (isOsx) { + serverProcess.Process.WaitForExit(assertSuccess: false); Assert.True(serverProcess.Process.HasExited, "built"); Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on macOS due to missing ALPN support.", ErrorMessages.GetFailedProcessMessageOrEmpty("Run built service", Project, serverProcess.Process)); } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version < new Version(6, 2)) + else if (isWindowsOld) { + serverProcess.Process.WaitForExit(assertSuccess: false); Assert.True(serverProcess.Process.HasExited, "built"); Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on Windows 7 due to missing ALPN support.", ErrorMessages.GetFailedProcessMessageOrEmpty("Run built service", Project, serverProcess.Process)); @@ -64,18 +76,20 @@ namespace Templates.Test } } - using (var aspNetProcess = Project.StartPublishedProjectAsync()) + using (var aspNetProcess = Project.StartPublishedProjectAsync(hasListeningUri: !unsupported)) { // These templates are HTTPS + HTTP/2 only which is not supported on Mac due to missing ALPN support. // https://github.com/dotnet/aspnetcore/issues/11061 - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + if (isOsx) { + aspNetProcess.Process.WaitForExit(assertSuccess: false); Assert.True(aspNetProcess.Process.HasExited, "published"); Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on macOS due to missing ALPN support.", ErrorMessages.GetFailedProcessMessageOrEmpty("Run published service", Project, aspNetProcess.Process)); } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version < new Version(6, 2)) + else if (isWindowsOld) { + aspNetProcess.Process.WaitForExit(assertSuccess: false); Assert.True(aspNetProcess.Process.HasExited, "published"); Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on Windows 7 due to missing ALPN support.", ErrorMessages.GetFailedProcessMessageOrEmpty("Run published service", Project, aspNetProcess.Process)); diff --git a/src/ProjectTemplates/test/Helpers/AspNetProcess.cs b/src/ProjectTemplates/test/Helpers/AspNetProcess.cs index 86724b970f..d90eb24713 100644 --- a/src/ProjectTemplates/test/Helpers/AspNetProcess.cs +++ b/src/ProjectTemplates/test/Helpers/AspNetProcess.cs @@ -14,6 +14,7 @@ using Microsoft.AspNetCore.Certificates.Generation; using Microsoft.AspNetCore.Internal; using Microsoft.AspNetCore.Server.IntegrationTesting; using Microsoft.Extensions.CommandLineUtils; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using OpenQA.Selenium; using OpenQA.Selenium.Edge; @@ -38,7 +39,8 @@ namespace Templates.Test.Helpers string dllPath, IDictionary environmentVariables, bool published = true, - bool hasListeningUri = true) + bool hasListeningUri = true, + ILogger logger = null) { _output = output; _httpClient = new HttpClient(new HttpClientHandler() @@ -57,10 +59,18 @@ namespace Templates.Test.Helpers output.WriteLine("Running ASP.NET application..."); var arguments = published ? $"exec {dllPath}" : "run"; + + logger?.LogInformation($"AspNetProcess - process: {DotNetMuxer.MuxerPathOrDefault()} arguments: {arguments}"); + Process = ProcessEx.Run(output, workingDirectory, DotNetMuxer.MuxerPathOrDefault(), arguments, envVars: environmentVariables); + + logger?.LogInformation("AspNetProcess - process started"); + if (hasListeningUri) { + logger?.LogInformation("AspNetProcess - Getting listening uri"); ListeningUri = GetListeningUri(output) ?? throw new InvalidOperationException("Couldn't find the listening URL."); + logger?.LogInformation($"AspNetProcess - Got {ListeningUri.ToString()}"); } } diff --git a/src/ProjectTemplates/test/Helpers/Project.cs b/src/ProjectTemplates/test/Helpers/Project.cs index b16801a5f7..68642c48f4 100644 --- a/src/ProjectTemplates/test/Helpers/Project.cs +++ b/src/ProjectTemplates/test/Helpers/Project.cs @@ -12,6 +12,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Internal; using Microsoft.Extensions.CommandLineUtils; +using Microsoft.Extensions.Logging; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -207,7 +208,7 @@ namespace Templates.Test.Helpers return new AspNetProcess(Output, TemplateClientReleaseDir, projectDll, environment); } - internal AspNetProcess StartBuiltProjectAsync(bool hasListeningUri = true) + internal AspNetProcess StartBuiltProjectAsync(bool hasListeningUri = true, ILogger logger = null) { var environment = new Dictionary { @@ -220,7 +221,7 @@ namespace Templates.Test.Helpers }; var projectDll = Path.Combine(TemplateBuildDir, $"{ProjectName}.dll"); - return new AspNetProcess(Output, TemplateOutputDir, projectDll, environment, hasListeningUri: hasListeningUri); + return new AspNetProcess(Output, TemplateOutputDir, projectDll, environment, hasListeningUri: hasListeningUri, logger: logger); } internal AspNetProcess StartPublishedProjectAsync(bool hasListeningUri = true) diff --git a/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj b/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj index 8f1984b78a..644ab2fa1c 100644 --- a/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj +++ b/src/ProjectTemplates/test/ProjectTemplates.Tests.csproj @@ -52,6 +52,11 @@ + + true + false + + <_Parameter1>DotNetEfFullPath @@ -69,6 +74,11 @@ <_Parameter1>ContinuousIntegrationBuild <_Parameter2>true + + <_Parameter1>$(PreserveExistingLogsInOutput) + <_Parameter2>$(TargetFramework) + <_Parameter3> +