From d14e2730155790fc81b898e7336671b50cd934ea Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 14 Sep 2020 18:43:42 -0700 Subject: [PATCH] [5.0.0-rc2] Backport Fix chrome/selenium tests (#25330) (#25840) * Fix chrome/selenium tests (#25330) * Revert "Disable failing/hanging tests due to Chrome/Selenium issue (#25323)" This reverts commit 332f1504128a63431cb67830703922cdeefee525. * Update Selenium to latest * Update API * Try specifying a version * Update Selenium to 4.0.0-beta5 * Disable browser log tests * Fix components e2e tests and disable blazor standalone template test * Disable tests using browser log * Disable template test * Avoid using .NET formatted strings in tests * Annotate BasicTestApp suggesting that it needs the all globalization data * Culture specific formatting relies on the ICU data carried by the OS. This causes issues in our tests if WebAssembly carries a different set than the OS. Instead updating these tests to use hardcoded strings. * Additionally fixing an issue where some projects in the solution were using tasks from the .dotnet SDK rather than the local copy of the SDK. This was causing issues building locally. Co-authored-by: Pranav K --- eng/Versions.props | 6 +-- eng/targets/CSharp.Common.props | 9 +++- ...soft.AspNetCore.Components.E2ETests.csproj | 3 -- .../MultipleRootComponentsTest.cs | 2 +- .../ServerReconnectionTest.cs | 2 +- .../Tests/WebAssemblyAuthenticationTests.cs | 2 +- .../Tests/WebAssemblyICUShardingTest.cs | 20 ++++---- .../E2ETest/Tests/WebAssemblyLazyLoadTest.cs | 2 +- .../Tests/WebAssemblyLocalizationTest.cs | 1 - .../E2ETest/Tests/WebAssemblyLoggingTest.cs | 8 +-- .../BasicTestApp/BasicTestApp.csproj | 3 ++ .../testassets/TestServer/ServerStartup.cs | 2 +- .../BlazorTemplates.Tests.csproj | 4 -- .../BlazorWasmTemplateTest.cs | 4 +- src/Shared/E2ETesting/BrowserFixture.cs | 51 ++++++++++++++----- src/Shared/E2ETesting/selenium-config.json | 6 ++- 16 files changed, 74 insertions(+), 51 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 9a7690ff4f..70a923a445 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -237,10 +237,10 @@ 1.0.2 12.0.2 13.0.4 - 3.12.1 + 4.0.0-alpha05 17.17134.0 - 2.43.0 - 3.12.1 + 85.0.4183.8300 + 4.0.0-alpha05 1.4.0 4.0.0 2.0.593 diff --git a/eng/targets/CSharp.Common.props b/eng/targets/CSharp.Common.props index ff77d1b86a..097769897b 100644 --- a/eng/targets/CSharp.Common.props +++ b/eng/targets/CSharp.Common.props @@ -12,7 +12,12 @@ - + + <_ReferenceLocalRazorSDK + Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(UsingMicrosoftNETSdkBlazorWebAssembly)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != '' OR '$(_RazorSdkImportsMicrosoftNetSdk)' == 'true'">true + + + - true - diff --git a/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs b/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs index 6fc25c2c46..d1bc0ca7ec 100644 --- a/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs +++ b/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs @@ -43,7 +43,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests } } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void DoesNotStartMultipleConnections() { Navigate("/multiple-components"); diff --git a/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs b/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs index 9b92d33aab..aa8ed996e7 100644 --- a/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs +++ b/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs @@ -78,7 +78,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests Browser.False(() => Browser.FindElement(selector).Text == currentValue); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void ErrorsStopTheRenderingProcess() { Browser.FindElement(By.Id("cause-error")).Click(); diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs b/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs index 52b25edefc..10a16061f8 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs @@ -306,7 +306,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests ValidateLoggedIn(userName); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void CanNotRedirect_To_External_ReturnUrl() { Browser.Navigate().GoToUrl(new Uri(new Uri(Browser.Url), "/authentication/login?returnUrl=https%3A%2F%2Fwww.bing.com").AbsoluteUri); diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs index f4d06eeda4..1d49906642 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs @@ -1,9 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Globalization; -using System.Linq; using GlobalizationWasmApp; using Microsoft.AspNetCore.Components.E2ETest.Infrastructure; using Microsoft.AspNetCore.Components.E2ETest.Infrastructure.ServerFixtures; @@ -18,7 +16,6 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests // This app covers testing this along with verifying the behavior for fallback culture for localized resources. public class WebAssemblyICUShardingTest : ServerTestBase> { - private readonly DateTime DisplayTime = new DateTime(2020, 09, 02); public WebAssemblyICUShardingTest( BrowserFixture browserFixture, ToggleExecutionModeServerFixture serverFixture, @@ -39,7 +36,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests Assert.Equal(culture.ToString(), cultureDisplay.Text); var dateDisplay = Browser.Exists(By.Id("dateTime")); - Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text); + Assert.Equal("02/09/2020 00:00:00", dateDisplay.Text); var localizedDisplay = Browser.Exists(By.Id("localizedString")); Assert.Equal("Bonjour!", localizedDisplay.Text); @@ -57,7 +54,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests Assert.Equal(culture.ToString(), cultureDisplay.Text); var dateDisplay = Browser.Exists(By.Id("dateTime")); - Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text); + Assert.Equal("2020. 9. 2. 오전 12:00:00", dateDisplay.Text); var localizedDisplay = Browser.Exists(By.Id("localizedString")); // The app has a "ko" resx file. This test verifies that we can walk up the culture hierarchy correctly. @@ -76,7 +73,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests Assert.Equal(culture.ToString(), cultureDisplay.Text); var dateDisplay = Browser.Exists(By.Id("dateTime")); - Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text); + Assert.Equal("02.09.2020 00:00:00", dateDisplay.Text); var localizedDisplay = Browser.Exists(By.Id("localizedString")); Assert.Equal("Hello", localizedDisplay.Text); // No localized resources for this culture. @@ -94,7 +91,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests Assert.Equal(culture.ToString(), cultureDisplay.Text); var dateDisplay = Browser.Exists(By.Id("dateTime")); - Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text); + Assert.Equal("2/9/2020 12:00:00 ಪೂರ್ವಾಹ್ನ", dateDisplay.Text); var localizedDisplay = Browser.Exists(By.Id("localizedString")); Assert.Equal("ಹಲೋ", localizedDisplay.Text); @@ -105,15 +102,16 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests { // Arrange // This verifies that we complain if the app programtically configures a language. - var expected = "This application's globalization settings requires using the combined globalization data file."; Navigate($"{ServerPathBase}/?culture=fr&dotNetCulture=es", noReload: false); var errorUi = Browser.Exists(By.Id("blazor-error-ui")); Browser.Equal("block", () => errorUi.GetCssValue("display")); - var logs = Browser.GetBrowserLogs(LogLevel.Severe).Select(l => l.Message); - Assert.True(logs.Any(l => l.Contains(expected)), - $"Expected to see globalization error message in the browser logs: {string.Join(Environment.NewLine, logs)}."); + // Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803" + // var expected = "This application's globalization settings requires using the combined globalization data file."; + // var logs = Browser.GetBrowserLogs(LogLevel.Severe).Select(l => l.Message); + // Assert.True(logs.Any(l => l.Contains(expected)), + // $"Expected to see globalization error message in the browser logs: {string.Join(Environment.NewLine, logs)}."); } private void Initialize(CultureInfo culture) diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs index 8076ae61dc..317bae624b 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs @@ -111,7 +111,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests Assert.True(renderedElement.Displayed); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void ThrowsErrorForUnavailableAssemblies() { // Navigate to a page with lazy loaded assemblies for the first time diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyLocalizationTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyLocalizationTest.cs index 81c99ccf67..e41a665485 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyLocalizationTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyLocalizationTest.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using BasicTestApp; -using Microsoft.AspNetCore.Components.E2ETest; using Microsoft.AspNetCore.Components.E2ETest.Infrastructure; using Microsoft.AspNetCore.Components.E2ETest.Infrastructure.ServerFixtures; using Microsoft.AspNetCore.E2ETesting; diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs index c6440eecd8..f659d24942 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs @@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests Assert.Equal("none", errorUi.GetCssValue("display")); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsSimpleExceptionsUsingLogger() { Browser.FindElement(By.Id("throw-simple-exception")).Click(); @@ -44,7 +44,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests "at BasicTestApp.ErrorComponent.ThrowSimple"); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsInnerExceptionsUsingLogger() { Browser.FindElement(By.Id("throw-inner-exception")).Click(); @@ -57,7 +57,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests "at BasicTestApp.ErrorComponent.ThrowInner"); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsAggregateExceptionsUsingLogger() { Browser.FindElement(By.Id("throw-aggregate-exception")).Click(); @@ -72,7 +72,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests "System.InvalidTimeZoneException: Aggregate exception 3"); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsUsingCustomLogger() { Browser.MountTestComponent(); diff --git a/src/Components/test/testassets/BasicTestApp/BasicTestApp.csproj b/src/Components/test/testassets/BasicTestApp/BasicTestApp.csproj index 58d604dda8..f6f34a7e17 100644 --- a/src/Components/test/testassets/BasicTestApp/BasicTestApp.csproj +++ b/src/Components/test/testassets/BasicTestApp/BasicTestApp.csproj @@ -8,6 +8,9 @@ false + + + true diff --git a/src/Components/test/testassets/TestServer/ServerStartup.cs b/src/Components/test/testassets/TestServer/ServerStartup.cs index 83578c5dcb..bcd6e56e88 100644 --- a/src/Components/test/testassets/TestServer/ServerStartup.cs +++ b/src/Components/test/testassets/TestServer/ServerStartup.cs @@ -19,7 +19,7 @@ namespace TestServer public void ConfigureServices(IServiceCollection services) { services.AddMvc(); - services.AddServerSideBlazor(options => options.DetailedErrors = true); + services.AddServerSideBlazor(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj index 6a024814c2..2da0e11666 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj @@ -10,10 +10,6 @@ true true - - - true - false true diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs index a1922ddfe9..fa4c091d54 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs @@ -40,7 +40,7 @@ namespace Templates.Test return InitializeAsync(isolationContext: Guid.NewGuid().ToString()); } - [Fact] + [Fact(Skip = "Certificate issue: https://github.com/dotnet/aspnetcore/issues/25826")] public async Task BlazorWasmStandaloneTemplate_Works() { // Additional arguments are needed. See: https://github.com/dotnet/aspnetcore/issues/24278 @@ -178,7 +178,7 @@ namespace Templates.Test } } - [Fact] + [Fact(Skip = "Certificate issue: https://github.com/dotnet/aspnetcore/issues/25826")] public async Task BlazorWasmHostedPwaTemplate_Works() { // Additional arguments are needed. See: https://github.com/dotnet/aspnetcore/issues/24278 diff --git a/src/Shared/E2ETesting/BrowserFixture.cs b/src/Shared/E2ETesting/BrowserFixture.cs index 557fbf37fc..3e6fa36b67 100644 --- a/src/Shared/E2ETesting/BrowserFixture.cs +++ b/src/Shared/E2ETesting/BrowserFixture.cs @@ -11,7 +11,10 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Edge; +using OpenQA.Selenium.IE; using OpenQA.Selenium.Remote; +using OpenQA.Selenium.Safari; using Xunit; using Xunit.Abstractions; @@ -234,55 +237,75 @@ namespace Microsoft.AspNetCore.E2ETesting name = $"{name} - {context}"; } - var capabilities = new DesiredCapabilities(); + DriverOptions options; + + switch (sauce.BrowserName.ToLower()) + { + case "chrome": + options = new ChromeOptions(); + break; + case "safari": + options = new SafariOptions(); + break; + case "internet explorer": + options = new InternetExplorerOptions(); + break; + case "microsoftedge": + options = new EdgeOptions(); + break; + default: + throw new InvalidOperationException($"Browser name {sauce.BrowserName} not recognized"); + } // Required config - capabilities.SetCapability("username", sauce.Username); - capabilities.SetCapability("accessKey", sauce.AccessKey); - capabilities.SetCapability("tunnelIdentifier", sauce.TunnelIdentifier); - capabilities.SetCapability("name", name); + options.AddAdditionalOption("username", sauce.Username); + options.AddAdditionalOption("accessKey", sauce.AccessKey); + options.AddAdditionalOption("tunnelIdentifier", sauce.TunnelIdentifier); + options.AddAdditionalOption("name", name); if (!string.IsNullOrEmpty(sauce.BrowserName)) { - capabilities.SetCapability("browserName", sauce.BrowserName); + options.AddAdditionalOption("browserName", sauce.BrowserName); } if (!string.IsNullOrEmpty(sauce.PlatformVersion)) { - capabilities.SetCapability("platformName", sauce.PlatformName); - capabilities.SetCapability("platformVersion", sauce.PlatformVersion); + options.PlatformName = sauce.PlatformName; + options.AddAdditionalOption("platformVersion", sauce.PlatformVersion); } else { // In some cases (like macOS), SauceLabs expects us to set "platform" instead of "platformName". - capabilities.SetCapability("platform", sauce.PlatformName); + options.AddAdditionalOption("platform", sauce.PlatformName); } if (!string.IsNullOrEmpty(sauce.BrowserVersion)) { - capabilities.SetCapability("browserVersion", sauce.BrowserVersion); + options.BrowserVersion = sauce.BrowserVersion; } if (!string.IsNullOrEmpty(sauce.DeviceName)) { - capabilities.SetCapability("deviceName", sauce.DeviceName); + options.AddAdditionalOption("deviceName", sauce.DeviceName); } if (!string.IsNullOrEmpty(sauce.DeviceOrientation)) { - capabilities.SetCapability("deviceOrientation", sauce.DeviceOrientation); + options.AddAdditionalOption("deviceOrientation", sauce.DeviceOrientation); } if (!string.IsNullOrEmpty(sauce.AppiumVersion)) { - capabilities.SetCapability("appiumVersion", sauce.AppiumVersion); + options.AddAdditionalOption("appiumVersion", sauce.AppiumVersion); } if (!string.IsNullOrEmpty(sauce.SeleniumVersion)) { - capabilities.SetCapability("seleniumVersion", sauce.SeleniumVersion); + options.AddAdditionalOption("seleniumVersion", sauce.SeleniumVersion); } + var capabilities = options.ToCapabilities(); + await SauceConnectServer.StartAsync(output); var attempt = 0; diff --git a/src/Shared/E2ETesting/selenium-config.json b/src/Shared/E2ETesting/selenium-config.json index 95c078e2c7..0682e48d32 100644 --- a/src/Shared/E2ETesting/selenium-config.json +++ b/src/Shared/E2ETesting/selenium-config.json @@ -1,6 +1,8 @@ { "drivers": { - "chrome": {} + "chrome": { + "version" : "85.0.4183.87" + } }, - "ignoreExtraDrivers": true + "ignoreExtraDrivers": true } \ No newline at end of file