diff --git a/src/Components/test/E2ETest/e2eTestSettings.ci.json b/src/Components/test/E2ETest/e2eTestSettings.ci.json new file mode 100644 index 0000000000..1632d4ade3 --- /dev/null +++ b/src/Components/test/E2ETest/e2eTestSettings.ci.json @@ -0,0 +1,8 @@ +{ + // We give each Selenium test assertion up to two minutes to fail before any other test in the + // build has failed, after that, we fail after 5 seconds. + "DefaultWaitTimeoutInSeconds": 120, + // This value is balanced between completing the build fast enough upon failure and giving + // each E2E test a fair chance to pass even in the event that a separate test has failed already. + "DefaultAfterFailureWaitTimeoutInSeconds": 5 +} diff --git a/src/Components/test/E2ETest/e2eTestSettings.json b/src/Components/test/E2ETest/e2eTestSettings.json new file mode 100644 index 0000000000..809f33f046 --- /dev/null +++ b/src/Components/test/E2ETest/e2eTestSettings.json @@ -0,0 +1,4 @@ +{ + "DefaultWaitTimeoutInSeconds": 20, + "ScreenShotsPath": "../../screenshots" +} diff --git a/src/ProjectTemplates/test/e2eTestSettings.ci.json b/src/ProjectTemplates/test/e2eTestSettings.ci.json new file mode 100644 index 0000000000..325431af35 --- /dev/null +++ b/src/ProjectTemplates/test/e2eTestSettings.ci.json @@ -0,0 +1,8 @@ +{ + // We give each Selenium test assertion up to two minutes to fail before any other test in the + // build has failed + "DefaultWaitTimeoutInSeconds": 120, + // This value is balanced between completing the build fast enough upon failure and giving + // each E2E test a fair chance to pass even in the event that a separate test has failed already. + "DefaultAfterFailureWaitTimeoutInSeconds": 120 +} diff --git a/src/ProjectTemplates/test/e2eTestSettings.json b/src/ProjectTemplates/test/e2eTestSettings.json new file mode 100644 index 0000000000..809f33f046 --- /dev/null +++ b/src/ProjectTemplates/test/e2eTestSettings.json @@ -0,0 +1,4 @@ +{ + "DefaultWaitTimeoutInSeconds": 20, + "ScreenShotsPath": "../../screenshots" +} diff --git a/src/Shared/E2ETesting/BrowserAssertFailedException.cs b/src/Shared/E2ETesting/BrowserAssertFailedException.cs index 579be07d47..508015b361 100644 --- a/src/Shared/E2ETesting/BrowserAssertFailedException.cs +++ b/src/Shared/E2ETesting/BrowserAssertFailedException.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO; using Xunit.Sdk; namespace OpenQA.Selenium @@ -12,16 +13,14 @@ namespace OpenQA.Selenium // case. public class BrowserAssertFailedException : XunitException { - public BrowserAssertFailedException(IReadOnlyList logs, Exception innerException) - : base(BuildMessage(logs), innerException) + public BrowserAssertFailedException(IReadOnlyList logs, Exception innerException, string screenShotPath) + : base(BuildMessage(logs, screenShotPath), innerException) { } - private static string BuildMessage(IReadOnlyList logs) - { - return - "Encountered browser errors while running assertion." + Environment.NewLine + - string.Join(Environment.NewLine, logs); - } + private static string BuildMessage(IReadOnlyList logs, string screenShotPath) => + (File.Exists(screenShotPath) ? $"Screen shot captured at '{screenShotPath}'" + Environment.NewLine : "") + + (logs.Count > 0 ? "Encountered browser errors" : "No browser errors found") + " while running the assertion." + Environment.NewLine + + string.Join(Environment.NewLine, logs); } } diff --git a/src/Shared/E2ETesting/E2ETestOptions.cs b/src/Shared/E2ETesting/E2ETestOptions.cs new file mode 100644 index 0000000000..0b35320176 --- /dev/null +++ b/src/Shared/E2ETesting/E2ETestOptions.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; + +namespace Microsoft.AspNetCore.E2ETesting +{ + public class E2ETestOptions + { + private const string TestingOptionsPrefix = "Microsoft.AspNetCore.E2ETesting"; + public static readonly IConfiguration Configuration; + + public static E2ETestOptions Instance; + + static E2ETestOptions() + { + // Capture all the attributes that start with Microsoft.AspNetCore.E2ETesting and add them as a memory collection + // to the list of settings. We use GetExecutingAssembly, this works because E2ETestOptions is shared source. + var metadataAttributes = Assembly.GetExecutingAssembly() + .GetCustomAttributes() + .Where(ama => ama.Key.StartsWith(TestingOptionsPrefix)) + .ToDictionary(kvp => kvp.Key.Substring(TestingOptionsPrefix.Length + 1), kvp => kvp.Value); + + try + { + // We save the configuration just to make resolved values easier to debug. + var builder = new ConfigurationBuilder() + .AddInMemoryCollection(metadataAttributes); + + if (!metadataAttributes.TryGetValue("CI", out var value) || string.IsNullOrEmpty(value)) + { + builder.AddJsonFile("e2eTestSettings.json", optional: true); + } + else + { + builder.AddJsonFile("e2eTestSettings.ci.json", optional: true); + } + + Configuration = builder + .AddEnvironmentVariables("E2ETESTS") + .Build(); + + var instance = new E2ETestOptions(); + Configuration.Bind(instance); + Instance = instance; + } + catch + { + } + } + + public int DefaultWaitTimeoutInSeconds { get; set; } = 3; + + public string ScreenShotsPath { get; set; } + + public double DefaultAfterFailureWaitTimeoutInSeconds { get; set; } = 3; + } +} diff --git a/src/Shared/E2ETesting/E2ETesting.props b/src/Shared/E2ETesting/E2ETesting.props index 1926e4e57b..e31cbd93ac 100644 --- a/src/Shared/E2ETesting/E2ETesting.props +++ b/src/Shared/E2ETesting/E2ETesting.props @@ -2,6 +2,7 @@ <_DefaultProjectFilter>$(MSBuildProjectDirectory)\..\.. $(DefaultItemExcludes);node_modules\** + $([MSBuild]::NormalizeDirectory('$(ArtifactsTestResultsDir)','$(MSBuildProjectName)')) $([MSBuild]::EnsureTrailingSlash('$(RepoRoot)'))artifacts\tmp\selenium\ true @@ -30,6 +31,22 @@ + + + + + + + + <_Parameter1>Microsoft.AspNetCore.E2ETesting.CI + <_Parameter2>$(ContinuousIntegrationBuild) + + + + <_Parameter1>Microsoft.AspNetCore.E2ETesting.ScreenshotsPath + <_Parameter2>$(SeleniumScreenShotsFolderPath) + + diff --git a/src/Shared/E2ETesting/E2ETesting.targets b/src/Shared/E2ETesting/E2ETesting.targets index 48d8dad7a6..de7110c6d9 100644 --- a/src/Shared/E2ETesting/E2ETesting.targets +++ b/src/Shared/E2ETesting/E2ETesting.targets @@ -2,6 +2,16 @@ + + + + PreserveNewest + + + PreserveNewest + + + @@ -53,20 +63,20 @@ - - <_DefaultProjectRoot>$([System.IO.Path]::GetFullPath($(_DefaultProjectFilter))) - - - <_ContentRootProjectReferencesUnfiltered - Include="@(ReferencePath)" - Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference'" /> - <_ContentRootProjectReferencesFilter - Include="@(_ContentRootProjectReferencesUnfiltered->StartsWith('$(_DefaultProjectRoot)'))" /> - <_ContentRootProjectReferences - Include="@(_ContentRootProjectReferencesFilter)" - Condition="'%(Identity)' == 'True'" /> - - + + <_DefaultProjectRoot>$([System.IO.Path]::GetFullPath($(_DefaultProjectFilter))) + + + <_ContentRootProjectReferencesUnfiltered + Include="@(ReferencePath)" + Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference'" /> + <_ContentRootProjectReferencesFilter + Include="@(_ContentRootProjectReferencesUnfiltered->StartsWith('$(_DefaultProjectRoot)'))" /> + <_ContentRootProjectReferences + Include="@(_ContentRootProjectReferencesFilter)" + Condition="'%(Identity)' == 'True'" /> + + @@ -114,4 +124,8 @@ + + + + diff --git a/src/Shared/E2ETesting/WaitAssert.cs b/src/Shared/E2ETesting/WaitAssert.cs index 1dc707012b..b9bac9f993 100644 --- a/src/Shared/E2ETesting/WaitAssert.cs +++ b/src/Shared/E2ETesting/WaitAssert.cs @@ -4,6 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.IO; using System.Runtime.ExceptionServices; using OpenQA.Selenium; using OpenQA.Selenium.Support.UI; @@ -15,7 +16,9 @@ namespace Microsoft.AspNetCore.E2ETesting public static class WaitAssert { - public static TimeSpan DefaultTimeout = TimeSpan.FromSeconds(3); + private static bool TestRunFailed = false; + public static TimeSpan DefaultTimeout = TimeSpan.FromSeconds(E2ETestOptions.Instance.DefaultWaitTimeoutInSeconds); + public static TimeSpan FailureTimeout = TimeSpan.FromSeconds(E2ETestOptions.Instance.DefaultAfterFailureWaitTimeoutInSeconds); public static void Equal(this IWebDriver driver, T expected, Func actual) => WaitAssertCore(driver, () => Assert.Equal(expected, actual())); @@ -51,7 +54,7 @@ namespace Microsoft.AspNetCore.E2ETesting { if (timeout == default) { - timeout = DefaultTimeout; + timeout = !TestRunFailed ? DefaultTimeout : FailureTimeout; } Exception lastException = null; @@ -64,7 +67,7 @@ namespace Microsoft.AspNetCore.E2ETesting assertion(); return true; } - catch(Exception e) + catch (Exception e) { lastException = e; return false; @@ -73,19 +76,51 @@ namespace Microsoft.AspNetCore.E2ETesting } catch (WebDriverTimeoutException) { + // At this point at least one test failed, so we mark the test as failed. Any assertions after this one + // will fail faster. There's a small race condition here between checking the value for TestRunFailed + // above and setting it here, but nothing bad can come out of it. Worst case scenario, one or more + // tests running concurrently might use the DefaultTimeout in their current assertion, which is fine. + TestRunFailed = true; + + var fileId = $"{Guid.NewGuid():N}.png"; + var screenShotPath = Path.Combine(Path.GetFullPath(E2ETestOptions.Instance.ScreenShotsPath), fileId); var errors = driver.GetBrowserLogs(LogLevel.Severe); - if (errors.Count > 0) + + TakeScreenShot(driver, screenShotPath); + var exceptionInfo = lastException != null ? ExceptionDispatchInfo.Capture(lastException) : + CaptureException(assertion); + + throw new BrowserAssertFailedException(errors, exceptionInfo.SourceException, screenShotPath); + + } + } + private static ExceptionDispatchInfo CaptureException(Action assertion) + { + try + { + assertion(); + throw new InvalidOperationException("The assertion succeded after the timeout."); + } + catch (Exception ex) + { + return ExceptionDispatchInfo.Capture(ex); + } + } + + private static void TakeScreenShot(IWebDriver driver, string screenShotPath) + { + if (driver is ITakesScreenshot takesScreenshot && E2ETestOptions.Instance.ScreenShotsPath != null) + { + try { - throw new BrowserAssertFailedException(errors, lastException); + Directory.CreateDirectory(E2ETestOptions.Instance.ScreenShotsPath); + + var screenShot = takesScreenshot.GetScreenshot(); + screenShot.SaveAsFile(screenShotPath); } - else if (lastException != null) + catch (Exception ex) { - ExceptionDispatchInfo.Capture(lastException).Throw(); - } - else - { - // Instead of reporting it as a timeout, report the Xunit exception - assertion(); + Console.WriteLine($"Failed to take a screenshot {ex.ToString()}"); } } }