From 437134f1da454c47ab91fc7e8beab95fdbebc3d8 Mon Sep 17 00:00:00 2001 From: Javier Calvarro Nelson Date: Fri, 5 Apr 2019 09:00:11 +0200 Subject: [PATCH] [Infrastructure] Standarize E2E test asserts and increase wait time (#9080) * Standarize E2E test asserts and increase wait time --- .../test/Helpers/AspNetProcess.cs | 2 +- .../test/Helpers/ProjectFactoryFixture.cs | 8 ++ .../test/Helpers/WebDriverExtensions.cs | 94 ------------------- .../test/RazorComponentsTemplateTest.cs | 32 +++---- .../SpaTemplateTest/SpaTemplateTestBase.cs | 32 +++---- src/Shared/E2ETesting/WaitAssert.cs | 22 ++++- 6 files changed, 55 insertions(+), 135 deletions(-) delete mode 100644 src/ProjectTemplates/test/Helpers/WebDriverExtensions.cs diff --git a/src/ProjectTemplates/test/Helpers/AspNetProcess.cs b/src/ProjectTemplates/test/Helpers/AspNetProcess.cs index 90480320cf..3aa89b9b8b 100644 --- a/src/ProjectTemplates/test/Helpers/AspNetProcess.cs +++ b/src/ProjectTemplates/test/Helpers/AspNetProcess.cs @@ -75,7 +75,7 @@ namespace Templates.Test.Helpers if (driver.Title.Contains("Certificate error", StringComparison.OrdinalIgnoreCase)) { _output.WriteLine("Page contains certificate error. Attempting to get around this..."); - driver.Click(By.Id("moreInformationDropdownSpan")); + driver.FindElement(By.Id("moreInformationDropdownSpan")).Click(); var continueLink = driver.FindElement(By.Id("invalidcert_continue")); if (continueLink != null) { diff --git a/src/ProjectTemplates/test/Helpers/ProjectFactoryFixture.cs b/src/ProjectTemplates/test/Helpers/ProjectFactoryFixture.cs index 071ee5b3a8..0aaf323844 100644 --- a/src/ProjectTemplates/test/Helpers/ProjectFactoryFixture.cs +++ b/src/ProjectTemplates/test/Helpers/ProjectFactoryFixture.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.E2ETesting; using Xunit.Abstractions; namespace Templates.Test.Helpers @@ -27,6 +28,13 @@ namespace Templates.Test.Helpers DiagnosticsMessageSink = diagnosticsMessageSink; } + static ProjectFactoryFixture() + { + // There is no good place to put this, so this is the best one. + // This sets the defualt timeout for all the Selenium test assertions. + WaitAssert.DefaultTimeout = TimeSpan.FromSeconds(30); + } + public async Task GetOrCreateProject(string projectKey, ITestOutputHelper output) { await TemplatePackageInstaller.EnsureTemplatingEngineInitializedAsync(output); diff --git a/src/ProjectTemplates/test/Helpers/WebDriverExtensions.cs b/src/ProjectTemplates/test/Helpers/WebDriverExtensions.cs deleted file mode 100644 index cd44071b1e..0000000000 --- a/src/ProjectTemplates/test/Helpers/WebDriverExtensions.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using OpenQA.Selenium; -using OpenQA.Selenium.Interactions; -using OpenQA.Selenium.Support.UI; -using System; -using System.Linq; - -namespace Templates.Test.Helpers -{ - public static class WebDriverExtensions - { - // Maximum time any action performed by WebDriver will wait before failing. - // Any action will have to be completed in at most 10 seconds. - // Providing a smaller value won't improve the speed of the tests in any - // significant way and will make them more prone to fail on slower drivers. - internal const int DefaultMaxWaitTimeInSeconds = 10; - - public static string GetText(this ISearchContext driver, string cssSelector) - { - return driver.FindElement(By.CssSelector(cssSelector)).Text; - } - - public static void Click(this IWebDriver driver, By by) - { - Click(driver, null, by); - } - - public static void Click(this IWebDriver driver, ISearchContext searchContext, By by) - { - // This elaborate way of clicking is a workaround for https://developer.microsoft.com/en-us/microsoft-edge/platform/issues/5238133/ - new Actions(driver) - .MoveToElement((searchContext ?? driver).FindElement(by)) - .Click() - .Perform(); - } - - - public static void Click(this IWebDriver driver, ISearchContext searchContext, string cssSelector) - { - Click(driver, searchContext, By.CssSelector(cssSelector)); - } - - public static IWebElement FindElement(this ISearchContext searchContext, string cssSelector) - { - return searchContext.FindElement(By.CssSelector(cssSelector)); - } - - public static IWebElement Parent(this IWebElement webElement) - { - return webElement.FindElement(By.XPath("..")); - } - - public static IWebElement FindElement(this IWebDriver driver, ISearchContext searchContext, string cssSelector, int timeoutSeconds) - { - return FindElement(driver, searchContext, By.CssSelector(cssSelector), timeoutSeconds); - } - - public static IWebElement FindElement(this IWebDriver driver, ISearchContext searchContext, By by, int timeoutSeconds) - { - return new WebDriverWait(driver, TimeSpan.FromSeconds(timeoutSeconds)) - .Until(drv => searchContext.FindElement(by)); - } - - public static void WaitForUrl(this IWebDriver browser, string expectedUrl) - { - new WebDriverWait(browser, TimeSpan.FromSeconds(DefaultMaxWaitTimeInSeconds)) - .Until(driver => driver.Url.Contains(expectedUrl, StringComparison.OrdinalIgnoreCase)); - } - - public static void WaitForElement(this IWebDriver browser, string expectedElementCss) - { - new WebDriverWait(browser, TimeSpan.FromSeconds(DefaultMaxWaitTimeInSeconds)) - .Until(driver => driver.FindElements(By.CssSelector(expectedElementCss)).Count > 0); - } - - public static void WaitForText(this IWebDriver browser, string cssSelector, string expectedText) - { - new WebDriverWait(browser, TimeSpan.FromSeconds(DefaultMaxWaitTimeInSeconds)) - .Until(driver => { - try - { - var matchingElement = driver.FindElements(By.CssSelector(cssSelector)).FirstOrDefault(); - return matchingElement?.Text == expectedText; - } - catch (Exception) // We can get a "stale element" exception if the DOM mutates while we're holding a reference to its element - { - return false; - } - }); - } - } -} diff --git a/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs b/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs index 33b42aed69..0d08a60a59 100644 --- a/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs +++ b/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs @@ -75,35 +75,31 @@ namespace Templates.Test // Give components.server enough time to load so that it can replace // the prerendered content before we start making assertions. Thread.Sleep(5000); - Browser.WaitForElement("ul"); + Browser.Exists(By.TagName("ul")); // element gets project ID injected into it during template execution - Assert.Contains(Project.ProjectGuid, Browser.Title); + Browser.Equal(Project.ProjectName.Trim(), () => Browser.Title.Trim()); // Initially displays the home page - Assert.Equal("Hello, world!", Browser.GetText("h1")); + Browser.Equal("Hello, world!", () => Browser.FindElement(By.TagName("h1")).Text); // Can navigate to the counter page - Browser.Click(By.PartialLinkText("Counter")); - Browser.WaitForUrl("counter"); - Browser.WaitForText("h1", "Counter"); + Browser.FindElement(By.PartialLinkText("Counter")).Click(); + Browser.Contains("counter", () => Browser.Url); + Browser.Equal("Counter", () => Browser.FindElement(By.TagName("h1")).Text); // Clicking the counter button works - var counterComponent = Browser.FindElement("h1").Parent(); - var counterDisplay = Browser.FindElement("h1 + p"); - Assert.Equal("Current count: 0", counterDisplay.Text); - Browser.Click(counterComponent, "button"); - Browser.Equal("Current count: 1", () => Browser.FindElement("h1+p").Text); + Browser.Equal("Current count: 0", () => Browser.FindElement(By.CssSelector("h1 + p")).Text); + Browser.FindElement(By.CssSelector("p+button")).Click(); + Browser.Equal("Current count: 1", () => Browser.FindElement(By.CssSelector("h1 + p")).Text); // Can navigate to the 'fetch data' page - Browser.Click(By.PartialLinkText("Fetch data")); - Browser.WaitForUrl("fetchdata"); - Browser.WaitForText("h1", "Weather forecast"); + Browser.FindElement(By.PartialLinkText("Fetch data")).Click(); + Browser.Contains("fetchdata", () => Browser.Url); + Browser.Equal("Weather forecast", () => Browser.FindElement(By.TagName("h1")).Text); // Asynchronously loads and displays the table of weather forecasts - var fetchDataComponent = Browser.FindElement("h1").Parent(); - Browser.WaitForElement("table>tbody>tr"); - var table = Browser.FindElement(fetchDataComponent, "table", timeoutSeconds: 5); - Assert.Equal(5, table.FindElements(By.CssSelector("tbody tr")).Count); + Browser.Exists(By.CssSelector("table>tbody>tr")); + Browser.Equal(5, () => Browser.FindElements(By.CssSelector("p+table>tbody>tr")).Count); } } } diff --git a/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs b/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs index 50ebc7f0f7..c234d843db 100644 --- a/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs +++ b/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs @@ -3,6 +3,7 @@ using System; using System.IO; +using System.Linq; using System.Net; using System.Threading.Tasks; using Microsoft.AspNetCore.E2ETesting; @@ -168,37 +169,34 @@ namespace Templates.Test.SpaTemplateTest private void TestBasicNavigation(bool visitFetchData) { - Browser.WaitForElement("ul"); + Browser.Exists(By.TagName("ul")); // <title> element gets project ID injected into it during template execution - Assert.Contains(Project.ProjectGuid, Browser.Title); + Browser.Contains(Project.ProjectGuid, () => Browser.Title); // Initially displays the home page - Assert.Equal("Hello, world!", Browser.GetText("h1")); + Browser.Equal("Hello, world!", () => Browser.FindElement(By.TagName("h1")).Text); // Can navigate to the counter page - Browser.Click(By.PartialLinkText("Counter")); - Browser.WaitForUrl("counter"); + Browser.FindElement(By.PartialLinkText("Counter")).Click(); + Browser.Contains("counter", () => Browser.Url); - Assert.Equal("Counter", Browser.GetText("h1")); + Browser.Equal("Counter", () => Browser.FindElement(By.TagName("h1")).Text); // Clicking the counter button works - var counterComponent = Browser.FindElement("h1").Parent(); - Assert.Equal("0", counterComponent.GetText("strong")); - Browser.Click(counterComponent, "button"); - Assert.Equal("1", counterComponent.GetText("strong")); + Browser.Equal("0", () => Browser.FindElement(By.CssSelector("p>strong")).Text); + Browser.FindElement(By.CssSelector("p+button")).Click(); + Browser.Equal("1", () => Browser.FindElement(By.CssSelector("p>strong")).Text); if (visitFetchData) { // Can navigate to the 'fetch data' page - Browser.Click(By.PartialLinkText("Fetch data")); - Browser.WaitForUrl("fetch-data"); - Assert.Equal("Weather forecast", Browser.GetText("h1")); + Browser.FindElement(By.PartialLinkText("Fetch data")).Click(); + Browser.Contains("fetch-data", () => Browser.Url); + Browser.Equal("Weather forecast", () => Browser.FindElement(By.TagName("h1")).Text); // Asynchronously loads and displays the table of weather forecasts - var fetchDataComponent = Browser.FindElement("h1").Parent(); - Browser.WaitForElement("table>tbody>tr"); - var table = Browser.FindElement(fetchDataComponent, "table", timeoutSeconds: 5); - Assert.Equal(5, table.FindElements(By.CssSelector("tbody tr")).Count); + Browser.Exists(By.CssSelector("table>tbody>tr")); + Browser.Equal(5, () => Browser.FindElements(By.CssSelector("p+table>tbody>tr")).Count); } } diff --git a/src/Shared/E2ETesting/WaitAssert.cs b/src/Shared/E2ETesting/WaitAssert.cs index 28ebbaa35b..ded27f289f 100644 --- a/src/Shared/E2ETesting/WaitAssert.cs +++ b/src/Shared/E2ETesting/WaitAssert.cs @@ -4,7 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Collections.ObjectModel; +using System.Runtime.ExceptionServices; using OpenQA.Selenium; using OpenQA.Selenium.Support.UI; using Xunit; @@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.E2ETesting public static class WaitAssert { - private readonly static TimeSpan DefaultTimeout = TimeSpan.FromSeconds(3); + public static TimeSpan DefaultTimeout = TimeSpan.FromSeconds(3); public static void Equal<T>(this IWebDriver driver, T expected, Func<T> actual) => WaitAssertCore(driver, () => Assert.Equal(expected, actual())); @@ -41,6 +41,9 @@ namespace Microsoft.AspNetCore.E2ETesting public static void Single(this IWebDriver driver, Func<IEnumerable> actualValues) => WaitAssertCore(driver, () => Assert.Single(actualValues())); + public static void Exists(this IWebDriver driver, By finder) + => WaitAssertCore(driver, () => Assert.NotEmpty(driver.FindElements(finder))); + private static void WaitAssertCore(IWebDriver driver, Action assertion, TimeSpan timeout = default) { if (timeout == default) @@ -48,6 +51,7 @@ namespace Microsoft.AspNetCore.E2ETesting timeout = DefaultTimeout; } + Exception lastException = null; try { new WebDriverWait(driver, timeout).Until(_ => @@ -57,16 +61,24 @@ namespace Microsoft.AspNetCore.E2ETesting assertion(); return true; } - catch + catch(Exception e) { + lastException = e; return false; } }); } catch (WebDriverTimeoutException) { - // Instead of reporting it as a timeout, report the Xunit exception - assertion(); + if (lastException != null) + { + ExceptionDispatchInfo.Capture(lastException).Throw(); + } + else + { + // Instead of reporting it as a timeout, report the Xunit exception + assertion(); + } } } }