huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Templating] Infrastructure improvements (#13672) 

* Automatically capture a screenshot when an assertion fails.
* Enable configurable options for different environments (CI|Dev).
* Include log errors in all exceptions.
* Add default timeout configurations and failure timeout configurations for CI and Dev environments.
This commit is contained in:
Javier Calvarro Nelson 2019-09-06 16:02:49 +02:00 committed by GitHub
parent 5f6c0d3fe8
commit 73f969852b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 183 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Components/test/E2ETest/e2eTestSettings.ci.json Normal file
View File

@ -0,0 +1,8 @@
{
// We give each Selenium test assertion up to two minutes to fail before any other test in the
// build has failed, after that, we fail after 5 seconds.
"DefaultWaitTimeoutInSeconds": 120,
// This value is balanced between completing the build fast enough upon failure and giving
// each E2E test a fair chance to pass even in the event that a separate test has failed already.
"DefaultAfterFailureWaitTimeoutInSeconds": 5
}

4
src/Components/test/E2ETest/e2eTestSettings.json Normal file
View File

@ -0,0 +1,4 @@
{
"DefaultWaitTimeoutInSeconds": 20,
"ScreenShotsPath": "../../screenshots"
}

8
src/ProjectTemplates/test/e2eTestSettings.ci.json Normal file
View File

@ -0,0 +1,8 @@
{
// We give each Selenium test assertion up to two minutes to fail before any other test in the
// build has failed
"DefaultWaitTimeoutInSeconds": 120,
// This value is balanced between completing the build fast enough upon failure and giving
// each E2E test a fair chance to pass even in the event that a separate test has failed already.
"DefaultAfterFailureWaitTimeoutInSeconds": 120
}

4
src/ProjectTemplates/test/e2eTestSettings.json Normal file
View File

@ -0,0 +1,4 @@
{
"DefaultWaitTimeoutInSeconds": 20,
"ScreenShotsPath": "../../screenshots"
}

15
src/Shared/E2ETesting/BrowserAssertFailedException.cs
View File

@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using Xunit.Sdk;
namespace OpenQA.Selenium
@ -12,16 +13,14 @@ namespace OpenQA.Selenium
// case.
public class BrowserAssertFailedException : XunitException
{
public BrowserAssertFailedException(IReadOnlyList<LogEntry> logs, Exception innerException)
: base(BuildMessage(logs), innerException)
public BrowserAssertFailedException(IReadOnlyList<LogEntry> logs, Exception innerException, string screenShotPath)
: base(BuildMessage(logs, screenShotPath), innerException)
{
}
private static string BuildMessage(IReadOnlyList<LogEntry> logs)
{
return
"Encountered browser errors while running assertion." + Environment.NewLine +
string.Join(Environment.NewLine, logs);
}
private static string BuildMessage(IReadOnlyList<LogEntry> logs, string screenShotPath) =>
(File.Exists(screenShotPath) ? $"Screen shot captured at '{screenShotPath}'" + Environment.NewLine : "") +
(logs.Count > 0 ? "Encountered browser errors" : "No browser errors found") + " while running the assertion." + Environment.NewLine +
string.Join(Environment.NewLine, logs);
}
}

60
src/Shared/E2ETesting/E2ETestOptions.cs Normal file
View File

@ -0,0 +1,60 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Linq;
using System.Reflection;
using Microsoft.Extensions.Configuration;
namespace Microsoft.AspNetCore.E2ETesting
{
public class E2ETestOptions
{
private const string TestingOptionsPrefix = "Microsoft.AspNetCore.E2ETesting";
public static readonly IConfiguration Configuration;
public static E2ETestOptions Instance;
static E2ETestOptions()
{
// Capture all the attributes that start with Microsoft.AspNetCore.E2ETesting and add them as a memory collection
// to the list of settings. We use GetExecutingAssembly, this works because E2ETestOptions is shared source.
var metadataAttributes = Assembly.GetExecutingAssembly()
.GetCustomAttributes<AssemblyMetadataAttribute>()
.Where(ama => ama.Key.StartsWith(TestingOptionsPrefix))
.ToDictionary(kvp => kvp.Key.Substring(TestingOptionsPrefix.Length + 1), kvp => kvp.Value);
try
{
// We save the configuration just to make resolved values easier to debug.
var builder = new ConfigurationBuilder()
.AddInMemoryCollection(metadataAttributes);
if (!metadataAttributes.TryGetValue("CI", out var value) || string.IsNullOrEmpty(value))
{
builder.AddJsonFile("e2eTestSettings.json", optional: true);
}
else
{
builder.AddJsonFile("e2eTestSettings.ci.json", optional: true);
}
Configuration = builder
.AddEnvironmentVariables("E2ETESTS")
.Build();
var instance = new E2ETestOptions();
Configuration.Bind(instance);
Instance = instance;
}
catch
{
}
}
public int DefaultWaitTimeoutInSeconds { get; set; } = 3;
public string ScreenShotsPath { get; set; }
public double DefaultAfterFailureWaitTimeoutInSeconds { get; set; } = 3;
}
}

17
src/Shared/E2ETesting/E2ETesting.props
View File

@ -2,6 +2,7 @@
<PropertyGroup>
<_DefaultProjectFilter>$(MSBuildProjectDirectory)\..\..</_DefaultProjectFilter>
<DefaultItemExcludes>$(DefaultItemExcludes);node_modules\**</DefaultItemExcludes>
<SeleniumScreenShotsFolderPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsTestResultsDir)','$(MSBuildProjectName)'))</SeleniumScreenShotsFolderPath>
<SeleniumProcessTrackingFolder Condition="'$(SeleniumProcessTrackingFolder)' == ''">$([MSBuild]::EnsureTrailingSlash('$(RepoRoot)'))artifacts\tmp\selenium\</SeleniumProcessTrackingFolder>
<SeleniumE2ETestsSupported Condition="'$(SeleniumE2ETestsSupported)' == '' and '$(TargetArchitecture)' != 'arm' and '$(OS)' == 'Windows_NT'">true</SeleniumE2ETestsSupported>
@ -30,6 +31,22 @@
<ItemGroup>
<Reference Include="Selenium.Support" />
<Reference Include="Selenium.WebDriver" />
<Reference Include="Microsoft.Extensions.Configuration" />
<Reference Include="Microsoft.Extensions.Configuration.Json" />
<Reference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
</ItemGroup>
<ItemGroup>
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
<_Parameter1>Microsoft.AspNetCore.E2ETesting.CI</_Parameter1>
<_Parameter2>$(ContinuousIntegrationBuild)</_Parameter2>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
<_Parameter1>Microsoft.AspNetCore.E2ETesting.ScreenshotsPath</_Parameter1>
<_Parameter2>$(SeleniumScreenShotsFolderPath)</_Parameter2>
</AssemblyAttribute>
</ItemGroup>
</Project>

42
src/Shared/E2ETesting/E2ETesting.targets
View File

@ -2,6 +2,16 @@
<!-- Version of this SDK is set in global.json -->
<Sdk Name="Yarn.MSBuild" />
<!-- Make sure the settings files get copied to the test output folder. -->
<ItemGroup>
<None Update="e2eTestSettings.*.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<Content Update="e2eTestSettings.*.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<!-- Ensuring that everything is ready before build -->
<Target Name="EnsureNodeJSRestored" Condition="'$(SeleniumE2ETestsSupported)' == 'true'" BeforeTargets="Build">
@ -53,20 +63,20 @@
<!-- Resolve content roots at build time -->
<Target Name="_ResolveTestProjectReferences" DependsOnTargets="ResolveReferences">
<PropertyGroup>
<_DefaultProjectRoot>$([System.IO.Path]::GetFullPath($(_DefaultProjectFilter)))</_DefaultProjectRoot>
</PropertyGroup>
<ItemGroup>
<_ContentRootProjectReferencesUnfiltered
Include="@(ReferencePath)"
Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference'" />
<_ContentRootProjectReferencesFilter
Include="@(_ContentRootProjectReferencesUnfiltered->StartsWith('$(_DefaultProjectRoot)'))" />
<_ContentRootProjectReferences
Include="@(_ContentRootProjectReferencesFilter)"
Condition="'%(Identity)' == 'True'" />
</ItemGroup>
</Target>
<PropertyGroup>
<_DefaultProjectRoot>$([System.IO.Path]::GetFullPath($(_DefaultProjectFilter)))</_DefaultProjectRoot>
</PropertyGroup>
<ItemGroup>
<_ContentRootProjectReferencesUnfiltered
Include="@(ReferencePath)"
Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference'" />
<_ContentRootProjectReferencesFilter
Include="@(_ContentRootProjectReferencesUnfiltered->StartsWith('$(_DefaultProjectRoot)'))" />
<_ContentRootProjectReferences
Include="@(_ContentRootProjectReferencesFilter)"
Condition="'%(Identity)' == 'True'" />
</ItemGroup>
</Target>
<Target Name="_AddTestProjectMetadataAttributes" BeforeTargets="BeforeCompile" DependsOnTargets="_ResolveTestProjectReferences">
<ItemGroup>
@ -114,4 +124,8 @@
</ItemGroup>
</Target>
<Target Name="_EnsureSeleniumScreenShotsFolder" BeforeTargets="Build">
<MakeDir Directories="$(SeleniumScreenShotsFolderPath)" />
</Target>
</Project>

59
src/Shared/E2ETesting/WaitAssert.cs
View File

@ -4,6 +4,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Runtime.ExceptionServices;
using OpenQA.Selenium;
using OpenQA.Selenium.Support.UI;
@ -15,7 +16,9 @@ namespace Microsoft.AspNetCore.E2ETesting
public static class WaitAssert
{
public static TimeSpan DefaultTimeout = TimeSpan.FromSeconds(3);
private static bool TestRunFailed = false;
public static TimeSpan DefaultTimeout = TimeSpan.FromSeconds(E2ETestOptions.Instance.DefaultWaitTimeoutInSeconds);
public static TimeSpan FailureTimeout = TimeSpan.FromSeconds(E2ETestOptions.Instance.DefaultAfterFailureWaitTimeoutInSeconds);
public static void Equal<T>(this IWebDriver driver, T expected, Func<T> actual)
=> WaitAssertCore(driver, () => Assert.Equal(expected, actual()));
@ -51,7 +54,7 @@ namespace Microsoft.AspNetCore.E2ETesting
{
if (timeout == default)
{
timeout = DefaultTimeout;
timeout = !TestRunFailed ? DefaultTimeout : FailureTimeout;
}
Exception lastException = null;
@ -64,7 +67,7 @@ namespace Microsoft.AspNetCore.E2ETesting
assertion();
return true;
}
catch(Exception e)
catch (Exception e)
{
lastException = e;
return false;
@ -73,19 +76,51 @@ namespace Microsoft.AspNetCore.E2ETesting
}
catch (WebDriverTimeoutException)
{
// At this point at least one test failed, so we mark the test as failed. Any assertions after this one
// will fail faster. There's a small race condition here between checking the value for TestRunFailed
// above and setting it here, but nothing bad can come out of it. Worst case scenario, one or more
// tests running concurrently might use the DefaultTimeout in their current assertion, which is fine.
TestRunFailed = true;
var fileId = $"{Guid.NewGuid():N}.png";
var screenShotPath = Path.Combine(Path.GetFullPath(E2ETestOptions.Instance.ScreenShotsPath), fileId);
var errors = driver.GetBrowserLogs(LogLevel.Severe);
if (errors.Count > 0)
TakeScreenShot(driver, screenShotPath);
var exceptionInfo = lastException != null ? ExceptionDispatchInfo.Capture(lastException) :
CaptureException(assertion);
throw new BrowserAssertFailedException(errors, exceptionInfo.SourceException, screenShotPath);
}
}
private static ExceptionDispatchInfo CaptureException(Action assertion)
{
try
{
assertion();
throw new InvalidOperationException("The assertion succeded after the timeout.");
}
catch (Exception ex)
{
return ExceptionDispatchInfo.Capture(ex);
}
}
private static void TakeScreenShot(IWebDriver driver, string screenShotPath)
{
if (driver is ITakesScreenshot takesScreenshot && E2ETestOptions.Instance.ScreenShotsPath != null)
{
try
{
throw new BrowserAssertFailedException(errors, lastException);
Directory.CreateDirectory(E2ETestOptions.Instance.ScreenShotsPath);
var screenShot = takesScreenshot.GetScreenshot();
screenShot.SaveAsFile(screenShotPath);
}
else if (lastException != null)
catch (Exception ex)
{
ExceptionDispatchInfo.Capture(lastException).Throw();
}
else
{
// Instead of reporting it as a timeout, report the Xunit exception
assertion();
Console.WriteLine($"Failed to take a screenshot {ex.ToString()}");
}
}
}