From 8fd86c38db09cb0c90236ee53fe5c29d97620b81 Mon Sep 17 00:00:00 2001 From: Ryan Brandenburg Date: Mon, 6 May 2019 15:33:02 -0700 Subject: [PATCH] Make some tests Flaky (#9935) Mark some tests flaky. --- src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs | 4 +++- src/ProjectTemplates/test/MvcTemplateTest.cs | 6 +++++- src/ProjectTemplates/test/RazorComponentsTemplateTest.cs | 3 +++ src/ProjectTemplates/test/RazorPagesTemplateTest.cs | 4 ++++ .../test/SpaTemplateTest/ReactReduxTemplateTest.cs | 3 +++ .../test/SpaTemplateTest/ReactTemplateTest.cs | 3 +++ .../test/SpaTemplateTest/SpaTemplateTestBase.cs | 4 +++- src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs | 3 ++- 8 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs b/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs index 46538b855b..abbb1b7176 100644 --- a/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs +++ b/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs @@ -11,6 +11,7 @@ using System.Net.Http; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing; using Microsoft.AspNetCore.Testing.xunit; using Xunit; using Xunit.Abstractions; @@ -55,7 +56,7 @@ namespace Microsoft.AspNetCore.Identity.Test using (var alg384 = SHA384.Create()) { byte[] hash; - if(isSha256) + if (isSha256) { hash = alg256.ComputeHash(respStream); } @@ -79,6 +80,7 @@ namespace Microsoft.AspNetCore.Identity.Test [Theory] [MemberData(nameof(ScriptWithFallbackSrcData))] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2267", FlakyOn.AzP.macOS)] public async Task IdentityUI_ScriptTags_FallbackSourceContent_Matches_CDNContent(ScriptTag scriptTag) { var slnDir = GetSolutionDir(); diff --git a/src/ProjectTemplates/test/MvcTemplateTest.cs b/src/ProjectTemplates/test/MvcTemplateTest.cs index 1cfc583412..b54b0dff68 100644 --- a/src/ProjectTemplates/test/MvcTemplateTest.cs +++ b/src/ProjectTemplates/test/MvcTemplateTest.cs @@ -8,6 +8,8 @@ using Templates.Test.Helpers; using System.Linq; using Xunit; using Xunit.Abstractions; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Testing.xunit; namespace Templates.Test { @@ -27,6 +29,7 @@ namespace Templates.Test [Theory] [InlineData(null)] [InlineData("F#")] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2267", FlakyOn.All)] public async Task MvcTemplate_NoAuthImplAsync(string languageOverride) { Project = await ProjectFactory.GetOrCreateProject("mvcnoauth" + (languageOverride == "F#" ? "fsharp" : "csharp"), Output); @@ -58,7 +61,7 @@ namespace Templates.Test PageUrls.PrivacyFullUrl }; - if(languageOverride == null) + if (languageOverride == null) { menuLinks = menuLinks.Append(PageUrls.PrivacyFullUrl); } @@ -100,6 +103,7 @@ namespace Templates.Test [Theory] [InlineData(true)] [InlineData(false)] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2267", FlakyOn.All)] public async Task MvcTemplate_IndividualAuthImplAsync(bool useLocalDB) { Project = await ProjectFactory.GetOrCreateProject("mvcindividual" + (useLocalDB ? "uld" : ""), Output); diff --git a/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs b/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs index 0d08a60a59..9477d08134 100644 --- a/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs +++ b/src/ProjectTemplates/test/RazorComponentsTemplateTest.cs @@ -5,6 +5,8 @@ using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.E2ETesting; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Testing.xunit; using OpenQA.Selenium; using Templates.Test.Helpers; using Xunit; @@ -24,6 +26,7 @@ namespace Templates.Test public Project Project { get; private set; } [Fact] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2407", FlakyOn.AzP.Windows)] public async Task RazorComponentsTemplateWorks() { Project = await ProjectFactory.GetOrCreateProject("blazorserverside", Output); diff --git a/src/ProjectTemplates/test/RazorPagesTemplateTest.cs b/src/ProjectTemplates/test/RazorPagesTemplateTest.cs index d7a432147e..69f74c1a0d 100644 --- a/src/ProjectTemplates/test/RazorPagesTemplateTest.cs +++ b/src/ProjectTemplates/test/RazorPagesTemplateTest.cs @@ -4,6 +4,8 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Testing.xunit; using Templates.Test.Helpers; using Xunit; using Xunit.Abstractions; @@ -25,6 +27,7 @@ namespace Templates.Test public ITestOutputHelper Output { get; } [Fact] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2327", FlakyOn.All)] public async Task RazorPagesTemplate_NoAuthImplAsync() { Project = await ProjectFactory.GetOrCreateProject("razorpagesnoauth", Output); @@ -95,6 +98,7 @@ namespace Templates.Test } [Theory] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2335", FlakyOn.All)] [InlineData(false)] [InlineData(true)] public async Task RazorPagesTemplate_IndividualAuthImplAsync(bool useLocalDB) diff --git a/src/ProjectTemplates/test/SpaTemplateTest/ReactReduxTemplateTest.cs b/src/ProjectTemplates/test/SpaTemplateTest/ReactReduxTemplateTest.cs index 44d6b67f32..4462f784c2 100644 --- a/src/ProjectTemplates/test/SpaTemplateTest/ReactReduxTemplateTest.cs +++ b/src/ProjectTemplates/test/SpaTemplateTest/ReactReduxTemplateTest.cs @@ -3,6 +3,8 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.E2ETesting; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Testing.xunit; using Templates.Test.Helpers; using Xunit; using Xunit.Abstractions; @@ -17,6 +19,7 @@ namespace Templates.Test.SpaTemplateTest } [Fact] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2407", FlakyOn.AzP.Windows)] public Task ReactReduxTemplate_Works_NetCore() => SpaTemplateImplAsync("reactredux", "reactredux", useLocalDb: false, usesAuth: false); } diff --git a/src/ProjectTemplates/test/SpaTemplateTest/ReactTemplateTest.cs b/src/ProjectTemplates/test/SpaTemplateTest/ReactTemplateTest.cs index d6c9c568b6..2a479c5f4b 100644 --- a/src/ProjectTemplates/test/SpaTemplateTest/ReactTemplateTest.cs +++ b/src/ProjectTemplates/test/SpaTemplateTest/ReactTemplateTest.cs @@ -3,6 +3,8 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.E2ETesting; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Testing.xunit; using Templates.Test.Helpers; using Xunit; using Xunit.Abstractions; @@ -21,6 +23,7 @@ namespace Templates.Test.SpaTemplateTest => SpaTemplateImplAsync("reactnoauth", "react", useLocalDb: false, usesAuth: false); [Fact] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2243", FlakyOn.All)] public Task ReactTemplate_IndividualAuth_NetCore() => SpaTemplateImplAsync("reactindividual", "react", useLocalDb: false, usesAuth: true); diff --git a/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs b/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs index 325ca911fb..c58560acd9 100644 --- a/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs +++ b/src/ProjectTemplates/test/SpaTemplateTest/SpaTemplateTestBase.cs @@ -244,7 +244,9 @@ namespace Templates.Test.SpaTemplateTest var entries = logs.GetLog(logKind); var badEntries = entries.Where(e => new LogLevel[] { LogLevel.Warning, LogLevel.Severe }.Contains(e.Level)); - badEntries = badEntries.Where(e => !e.Message.EndsWith("failed: WebSocket is closed before the connection is established.")); + badEntries = badEntries.Where(e => + !e.Message.Contains("failed: WebSocket is closed before the connection is established.") + && !e.Message.Contains("[WDS] Disconnected!")); Assert.True(badEntries.Count() == 0, "There were Warnings or Errors from the browser." + Environment.NewLine + string.Join(Environment.NewLine, badEntries)); } diff --git a/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs b/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs index b1eceb56d7..eead5acb99 100644 --- a/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs +++ b/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs @@ -66,6 +66,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys } [ConditionalFact] + [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2267", FlakyOn.All)] public async Task Server_ShutdownDuringRequest_Success() { Task responseTask; @@ -595,7 +596,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys private async Task SendRequestAsync(string uri) { - using (HttpClient client = new HttpClient() { Timeout = Utilities.DefaultTimeout } ) + using (HttpClient client = new HttpClient() { Timeout = Utilities.DefaultTimeout }) { return await client.GetStringAsync(uri); }