[5.0.0-rc2] Backport Fix chrome/selenium tests (#25330) (#25840)

* Fix chrome/selenium tests (#25330)

* Revert "Disable failing/hanging tests due to Chrome/Selenium issue (#25323)"

This reverts commit 332f1504128a63431cb67830703922cdeefee525.

* Update Selenium to latest

* Update API

* Try specifying a version

* Update Selenium to 4.0.0-beta5

* Disable browser log tests

* Fix components e2e tests and disable blazor standalone template test

* Disable tests using browser log

* Disable template test

* Avoid using .NET formatted strings in tests

* Annotate BasicTestApp suggesting that it needs the all globalization data

* Culture specific formatting relies on the ICU data carried by the OS. This
causes issues in our tests if WebAssembly carries a different set than the OS. Instead
updating these tests to use hardcoded strings.

* Additionally fixing an issue where some projects in the solution were using tasks from
the .dotnet SDK rather than the local copy of the SDK. This was causing issues building locally.

Co-authored-by: Pranav K <prkrishn@hotmail.com>

eng/Versions.props

@@ -237,10 +237,10 @@
     <NewtonsoftJsonBsonPackageVersion>1.0.2</NewtonsoftJsonBsonPackageVersion>
     <NewtonsoftJsonPackageVersion>12.0.2</NewtonsoftJsonPackageVersion>
     <NSwagApiDescriptionClientPackageVersion>13.0.4</NSwagApiDescriptionClientPackageVersion>
-    <SeleniumSupportPackageVersion>3.12.1</SeleniumSupportPackageVersion>
+    <SeleniumSupportPackageVersion>4.0.0-alpha05</SeleniumSupportPackageVersion>
     <SeleniumWebDriverMicrosoftDriverPackageVersion>17.17134.0</SeleniumWebDriverMicrosoftDriverPackageVersion>
-    <SeleniumWebDriverChromeDriverPackageVersion>2.43.0</SeleniumWebDriverChromeDriverPackageVersion>
+    <SeleniumWebDriverChromeDriverPackageVersion>85.0.4183.8300</SeleniumWebDriverChromeDriverPackageVersion>
-    <SeleniumWebDriverPackageVersion>3.12.1</SeleniumWebDriverPackageVersion>
+    <SeleniumWebDriverPackageVersion>4.0.0-alpha05</SeleniumWebDriverPackageVersion>
     <SerilogExtensionsLoggingPackageVersion>1.4.0</SerilogExtensionsLoggingPackageVersion>
     <SerilogSinksFilePackageVersion>4.0.0</SerilogSinksFilePackageVersion>
     <StackExchangeRedisPackageVersion>2.0.593</StackExchangeRedisPackageVersion>

eng/targets/CSharp.Common.props

@@ -12,7 +12,12 @@
     <PackageReference Include="MicroBuild.Core" Version="0.3.0" PrivateAssets="All" AllowExplicitReference="true" ExcludeAssets="All" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(UsingMicrosoftNETSdkBlazorWebAssembly)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != ''">
+  <PropertyGroup>
+    <_ReferenceLocalRazorSDK
+      Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(UsingMicrosoftNETSdkBlazorWebAssembly)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != '' OR '$(_RazorSdkImportsMicrosoftNetSdk)' == 'true'">true</_ReferenceLocalRazorSDK>
+  </PropertyGroup>
+
+  <ItemGroup Condition="'$(_ReferenceLocalRazorSDK)' == 'true'">
     <!--
       Use the Razor SDK as a project reference. The version of the .NET Core SDK we build with often contains a version of the Razor SDK
       several versions older than latest. We reference the project to ensure it's built before the other projects that use it. Since this
@@ -25,7 +30,7 @@
       UndefineProperties="TargetFramework;TargetFrameworks" />
   </ItemGroup>
 
-  <ImportGroup Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(UsingMicrosoftNETSdkBlazorWebAssembly)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != ''">
+  <ImportGroup Condition="'$(_ReferenceLocalRazorSDK)' == 'true'">
     <Import Project="$(RepoRoot)eng\targets\GetRazorSDKDirectory.props" />
     <Import Project="$(RepoRoot)src\Razor\Microsoft.NET.Sdk.Razor\src\build\netstandard2.0\Microsoft.NET.Sdk.Razor.props" />
   </ImportGroup>

src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj

@@ -14,9 +14,6 @@
     <SkipTests Condition="'$(SeleniumE2ETestsSupported)' != 'true'">true</SkipTests>
     <SkipTests Condition="'$(SeleniumE2ETestsSupported)' == 'true'">false</SkipTests>
 
-    <!-- Tests are disabled due to https://github.com/dotnet/aspnetcore/issues/25322 -->
-    <SkipTests>true</SkipTests>
-
     <!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
     <BaseOutputPath />
 

src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs

@@ -43,7 +43,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void DoesNotStartMultipleConnections()
         {
             Navigate("/multiple-components");

src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs

@@ -78,7 +78,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests
             Browser.False(() => Browser.FindElement(selector).Text == currentValue);
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void ErrorsStopTheRenderingProcess()
         {
             Browser.FindElement(By.Id("cause-error")).Click();

src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs

@@ -306,7 +306,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             ValidateLoggedIn(userName);
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void CanNotRedirect_To_External_ReturnUrl()
         {
             Browser.Navigate().GoToUrl(new Uri(new Uri(Browser.Url), "/authentication/login?returnUrl=https%3A%2F%2Fwww.bing.com").AbsoluteUri);

src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs

@@ -1,9 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
 using System.Globalization;
-using System.Linq;
 using GlobalizationWasmApp;
 using Microsoft.AspNetCore.Components.E2ETest.Infrastructure;
 using Microsoft.AspNetCore.Components.E2ETest.Infrastructure.ServerFixtures;
@@ -18,7 +16,6 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
     // This app covers testing this along with verifying the behavior for fallback culture for localized resources.
     public class WebAssemblyICUShardingTest : ServerTestBase<ToggleExecutionModeServerFixture<Program>>
     {
-        private readonly DateTime DisplayTime = new DateTime(2020, 09, 02);
         public WebAssemblyICUShardingTest(
             BrowserFixture browserFixture,
             ToggleExecutionModeServerFixture<Program> serverFixture,
@@ -39,7 +36,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             Assert.Equal(culture.ToString(), cultureDisplay.Text);
 
             var dateDisplay = Browser.Exists(By.Id("dateTime"));
-            Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text);
+            Assert.Equal("02/09/2020 00:00:00", dateDisplay.Text);
 
             var localizedDisplay = Browser.Exists(By.Id("localizedString"));
             Assert.Equal("Bonjour!", localizedDisplay.Text);
@@ -57,7 +54,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             Assert.Equal(culture.ToString(), cultureDisplay.Text);
 
             var dateDisplay = Browser.Exists(By.Id("dateTime"));
-            Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text);
+            Assert.Equal("2020. 9. 2. 오전 12:00:00", dateDisplay.Text);
 
             var localizedDisplay = Browser.Exists(By.Id("localizedString"));
             // The app has a "ko" resx file. This test verifies that we can walk up the culture hierarchy correctly.
@@ -76,7 +73,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             Assert.Equal(culture.ToString(), cultureDisplay.Text);
 
             var dateDisplay = Browser.Exists(By.Id("dateTime"));
-            Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text);
+            Assert.Equal("02.09.2020 00:00:00", dateDisplay.Text);
 
             var localizedDisplay = Browser.Exists(By.Id("localizedString"));
             Assert.Equal("Hello", localizedDisplay.Text); // No localized resources for this culture.
@@ -94,7 +91,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             Assert.Equal(culture.ToString(), cultureDisplay.Text);
 
             var dateDisplay = Browser.Exists(By.Id("dateTime"));
-            Assert.Equal(DisplayTime.ToString(culture), dateDisplay.Text);
+            Assert.Equal("2/9/2020 12:00:00 ಪೂರ್ವಾಹ್ನ", dateDisplay.Text);
 
             var localizedDisplay = Browser.Exists(By.Id("localizedString"));
             Assert.Equal("ಹಲೋ", localizedDisplay.Text);
@@ -105,15 +102,16 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
         {
             // Arrange
             // This verifies that we complain if the app programtically configures a language.
-            var expected = "This application's globalization settings requires using the combined globalization data file.";
             Navigate($"{ServerPathBase}/?culture=fr&dotNetCulture=es", noReload: false);
 
             var errorUi = Browser.Exists(By.Id("blazor-error-ui"));
             Browser.Equal("block", () => errorUi.GetCssValue("display"));
 
-            var logs = Browser.GetBrowserLogs(LogLevel.Severe).Select(l => l.Message);
+            // Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803"
-            Assert.True(logs.Any(l => l.Contains(expected)),
+            // var expected = "This application's globalization settings requires using the combined globalization data file.";
-                $"Expected to see globalization error message in the browser logs: {string.Join(Environment.NewLine, logs)}.");
+            // var logs = Browser.GetBrowserLogs(LogLevel.Severe).Select(l => l.Message);
+            // Assert.True(logs.Any(l => l.Contains(expected)),
+            //     $"Expected to see globalization error message in the browser logs: {string.Join(Environment.NewLine, logs)}.");
         }
 
         private void Initialize(CultureInfo culture)

src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs

@@ -111,7 +111,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             Assert.True(renderedElement.Displayed);
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void ThrowsErrorForUnavailableAssemblies()
         {
             // Navigate to a page with lazy loaded assemblies for the first time

src/Components/test/E2ETest/Tests/WebAssemblyLocalizationTest.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using BasicTestApp;
-using Microsoft.AspNetCore.Components.E2ETest;
 using Microsoft.AspNetCore.Components.E2ETest.Infrastructure;
 using Microsoft.AspNetCore.Components.E2ETest.Infrastructure.ServerFixtures;
 using Microsoft.AspNetCore.E2ETesting;

src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs

@@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
             Assert.Equal("none", errorUi.GetCssValue("display"));
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void LogsSimpleExceptionsUsingLogger()
         {
             Browser.FindElement(By.Id("throw-simple-exception")).Click();
@@ -44,7 +44,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
                 "at BasicTestApp.ErrorComponent.ThrowSimple");
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void LogsInnerExceptionsUsingLogger()
         {
             Browser.FindElement(By.Id("throw-inner-exception")).Click();
@@ -57,7 +57,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
                 "at BasicTestApp.ErrorComponent.ThrowInner");
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void LogsAggregateExceptionsUsingLogger()
         {
             Browser.FindElement(By.Id("throw-aggregate-exception")).Click();
@@ -72,7 +72,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
                 "System.InvalidTimeZoneException: Aggregate exception 3");
         }
 
-        [Fact]
+        [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")]
         public void LogsUsingCustomLogger()
         {
             Browser.MountTestComponent<LoggingComponent>();

src/Components/test/testassets/BasicTestApp/BasicTestApp.csproj

@@ -8,6 +8,9 @@
 
     <!-- Resx generation on Resources.resx only -->
     <GenerateResxSource>false</GenerateResxSource>
+
+    <!-- Project supports more than one language -->
+    <BlazorWebAssemblyLoadAllGlobalizationData>true</BlazorWebAssemblyLoadAllGlobalizationData>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/test/testassets/TestServer/ServerStartup.cs

@@ -19,7 +19,7 @@ namespace TestServer
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddMvc();
-            services.AddServerSideBlazor(options => options.DetailedErrors = true);
+            services.AddServerSideBlazor();
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj

@@ -10,10 +10,6 @@
 
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''" >true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
-
-    <!-- Tests are disabled due to https://github.com/dotnet/aspnetcore/issues/25322 -->
-    <SkipTests>true</SkipTests>
-
     <!--Do not run this test project on Helix.-->
     <BuildHelixPayload>false</BuildHelixPayload>
     <SkipHelixArm>true</SkipHelixArm>

src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs

@@ -40,7 +40,7 @@ namespace Templates.Test
             return InitializeAsync(isolationContext: Guid.NewGuid().ToString());
         }
 
-        [Fact]
+        [Fact(Skip = "Certificate issue: https://github.com/dotnet/aspnetcore/issues/25826")]
         public async Task BlazorWasmStandaloneTemplate_Works()
         {
             // Additional arguments are needed. See: https://github.com/dotnet/aspnetcore/issues/24278
@@ -178,7 +178,7 @@ namespace Templates.Test
             }
         }
 
-        [Fact]
+        [Fact(Skip = "Certificate issue: https://github.com/dotnet/aspnetcore/issues/25826")]
         public async Task BlazorWasmHostedPwaTemplate_Works()
         {
             // Additional arguments are needed. See: https://github.com/dotnet/aspnetcore/issues/24278

src/Shared/E2ETesting/BrowserFixture.cs

@@ -11,7 +11,10 @@ using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using OpenQA.Selenium;
 using OpenQA.Selenium.Chrome;
+using OpenQA.Selenium.Edge;
+using OpenQA.Selenium.IE;
 using OpenQA.Selenium.Remote;
+using OpenQA.Selenium.Safari;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -234,55 +237,75 @@ namespace Microsoft.AspNetCore.E2ETesting
                 name = $"{name} - {context}";
             }
 
-            var capabilities = new DesiredCapabilities();
+            DriverOptions options;
+
+            switch (sauce.BrowserName.ToLower())
+            {
+                case "chrome":
+                    options = new ChromeOptions();
+                    break;
+                case "safari":
+                    options = new SafariOptions();
+                    break;
+                case "internet explorer":
+                    options = new InternetExplorerOptions();
+                    break;
+                case "microsoftedge":
+                    options = new EdgeOptions();
+                    break;
+                default:
+                    throw new InvalidOperationException($"Browser name {sauce.BrowserName} not recognized");
+            }
 
             // Required config
-            capabilities.SetCapability("username", sauce.Username);
+            options.AddAdditionalOption("username", sauce.Username);
-            capabilities.SetCapability("accessKey", sauce.AccessKey);
+            options.AddAdditionalOption("accessKey", sauce.AccessKey);
-            capabilities.SetCapability("tunnelIdentifier", sauce.TunnelIdentifier);
+            options.AddAdditionalOption("tunnelIdentifier", sauce.TunnelIdentifier);
-            capabilities.SetCapability("name", name);
+            options.AddAdditionalOption("name", name);
 
             if (!string.IsNullOrEmpty(sauce.BrowserName))
             {
-                capabilities.SetCapability("browserName", sauce.BrowserName);
+                options.AddAdditionalOption("browserName", sauce.BrowserName);
             }
 
             if (!string.IsNullOrEmpty(sauce.PlatformVersion))
             {
-                capabilities.SetCapability("platformName", sauce.PlatformName);
+                options.PlatformName = sauce.PlatformName;
-                capabilities.SetCapability("platformVersion", sauce.PlatformVersion);
+                options.AddAdditionalOption("platformVersion", sauce.PlatformVersion);
             }
             else
             {
                 // In some cases (like macOS), SauceLabs expects us to set "platform" instead of "platformName".
-                capabilities.SetCapability("platform", sauce.PlatformName);
+                options.AddAdditionalOption("platform", sauce.PlatformName);
             }
 
             if (!string.IsNullOrEmpty(sauce.BrowserVersion))
             {
-                capabilities.SetCapability("browserVersion", sauce.BrowserVersion);
+                options.BrowserVersion = sauce.BrowserVersion;
             }
 
             if (!string.IsNullOrEmpty(sauce.DeviceName))
             {
-                capabilities.SetCapability("deviceName", sauce.DeviceName);
+                options.AddAdditionalOption("deviceName", sauce.DeviceName);
             }
 
             if (!string.IsNullOrEmpty(sauce.DeviceOrientation))
             {
-                capabilities.SetCapability("deviceOrientation", sauce.DeviceOrientation);
+                options.AddAdditionalOption("deviceOrientation", sauce.DeviceOrientation);
             }
 
             if (!string.IsNullOrEmpty(sauce.AppiumVersion))
             {
-                capabilities.SetCapability("appiumVersion", sauce.AppiumVersion);
+                options.AddAdditionalOption("appiumVersion", sauce.AppiumVersion);
             }
 
             if (!string.IsNullOrEmpty(sauce.SeleniumVersion))
             {
-                capabilities.SetCapability("seleniumVersion", sauce.SeleniumVersion);
+                options.AddAdditionalOption("seleniumVersion", sauce.SeleniumVersion);
             }
 
+            var capabilities = options.ToCapabilities();
+
             await SauceConnectServer.StartAsync(output);
 
             var attempt = 0;

src/Shared/E2ETesting/selenium-config.json

@@ -1,6 +1,8 @@
 {
   "drivers": {
-    "chrome": {}
+    "chrome": {
+      "version" : "85.0.4183.87"
+    }
   },
-  "ignoreExtraDrivers": true  
+  "ignoreExtraDrivers": true
 }