[Templates] Separate F# from C# tests, unskip tests, include reliability improvements. (#14261)

* Separates F# from C# tests. These tests have very different runtime
  characteristics and should not be bundled together as the same test
  scenario. For example, we want to quickly separate F# specific issues
  from C# specific ones.
* Unskipping all skipped tests as the issue tracking their flakyness
  was closed.
* Adding reliability improvements to network requests.
  * Adds retries.

src/ProjectTemplates/test/EmptyWebTemplateTest.cs

@@ -22,10 +22,19 @@ namespace Templates.Test
 
         public ITestOutputHelper Output { get; }
 
-        [Theory]
+        [Fact]
-        [InlineData(null)]
+        public async Task EmptyWebTemplateCSharp()
-        [InlineData("F#")]
+        {
-        public async Task EmptyWebTemplateAsync(string languageOverride)
+            await EmtpyTemplateCore(languageOverride: null);
+        }
+
+        [Fact]
+        public async Task EmptyWebTemplateFSharp()
+        {
+            await EmtpyTemplateCore("F#");
+        }
+
+        private async Task EmtpyTemplateCore(string languageOverride)
         {
             Project = await ProjectFactory.GetOrCreateProject("empty" + (languageOverride == "F#" ? "fsharp" : "csharp"), Output);
 

src/ProjectTemplates/test/Helpers/AspNetProcess.cs

@@ -58,7 +58,7 @@ namespace Templates.Test.Helpers
 
             var arguments = published ? $"exec {dllPath}" : "run";
             Process = ProcessEx.Run(output, workingDirectory, DotNetMuxer.MuxerPathOrDefault(), arguments, envVars: environmentVariables);
-            if(hasListeningUri)
+            if (hasListeningUri)
             {
                 ListeningUri = GetListeningUri(output);
             }
@@ -108,7 +108,7 @@ namespace Templates.Test.Helpers
                 HttpMethod.Get,
                 new Uri(ListeningUri, page.Url));
 
-            var response = await _httpClient.SendAsync(request);
+            var response = await RequestWithRetries(client => client.SendAsync(request), _httpClient);
 
             Assert.Equal(HttpStatusCode.OK, response.StatusCode);
             var parser = new HtmlParser();
@@ -141,7 +141,7 @@ namespace Templates.Test.Helpers
                     Assert.True(string.Equals(anchor.Href, expectedLink), $"Expected next link to be {expectedLink} but it was {anchor.Href}.");
                     var result = await RetryHelper.RetryRequest(async () =>
                     {
-                        return await _httpClient.GetAsync(anchor.Href);
+                        return await RequestWithRetries(client => client.GetAsync(anchor.Href), _httpClient);
                     }, logger: NullLogger.Instance);
 
                     Assert.True(IsSuccessStatusCode(result), $"{anchor.Href} is a broken link!");
@@ -149,6 +149,28 @@ namespace Templates.Test.Helpers
             }
         }
 
+        private async Task<T> RequestWithRetries<T>(Func<HttpClient, Task<T>> requester, HttpClient client, int retries = 3, TimeSpan initialDelay = default)
+        {
+            var currentDelay = initialDelay == default ? TimeSpan.FromSeconds(30) : initialDelay;
+            for (int i = 0; i <= retries; i++)
+            {
+                try
+                {
+                    return await requester(client);
+                }
+                catch (Exception)
+                {
+                    if (i == retries)
+                    {
+                        throw;
+                    }
+                    await Task.Delay(currentDelay);
+                    currentDelay *= 2;
+                }
+            }
+            throw new InvalidOperationException("Max retries reached.");
+        }
+
         private Uri GetListeningUri(ITestOutputHelper output)
         {
             // Wait until the app is accepting HTTP requests
@@ -190,7 +212,7 @@ namespace Templates.Test.Helpers
 
         internal Task<HttpResponseMessage> SendRequest(string path)
         {
-            return _httpClient.GetAsync(new Uri(ListeningUri, path));
+            return RequestWithRetries(client => client.GetAsync(new Uri(ListeningUri, path)), _httpClient);
         }
 
         public async Task AssertStatusCode(string requestUrl, HttpStatusCode statusCode, string acceptContentType = null)
@@ -204,7 +226,7 @@ namespace Templates.Test.Helpers
                 request.Headers.Add("Accept", acceptContentType);
             }
 
-            var response = await _httpClient.SendAsync(request);
+            var response = await RequestWithRetries(client => client.SendAsync(request), _httpClient);
             Assert.True(statusCode == response.StatusCode, $"Expected {requestUrl} to have status '{statusCode}' but it was '{response.StatusCode}'.");
         }
 

src/ProjectTemplates/test/MvcTemplateTest.cs

@@ -25,11 +25,14 @@ namespace Templates.Test
         public ProjectFactoryFixture ProjectFactory { get; }
         public ITestOutputHelper Output { get; }
 
-        [Theory]
+        [Fact(Skip = "https://github.com/aspnet/AspNetCore/issues/14022")]
-        [InlineData(null)]
+        public async Task MvcTemplate_NoAuthFSharp() => await MvcTemplateCore(languageOverride: "F#");
-        [InlineData("F#", Skip = "https://github.com/aspnet/AspNetCore/issues/14022")]
+
-        [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2267", FlakyOn.All)]
+        [Fact]
-        public async Task MvcTemplate_NoAuthImplAsync(string languageOverride)
+        public async Task MvcTemplate_NoAuthCSharp() => await MvcTemplateCore(languageOverride: null);
+
+
+        private async Task MvcTemplateCore(string languageOverride)
         {
             Project = await ProjectFactory.GetOrCreateProject("mvcnoauth" + (languageOverride == "F#" ? "fsharp" : "csharp"), Output);
 
@@ -98,8 +101,7 @@ namespace Templates.Test
         [Theory]
         [InlineData(true)]
         [InlineData(false)]
-        [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2267", FlakyOn.All)]
+        public async Task MvcTemplate_IndividualAuth(bool useLocalDB)
-        public async Task MvcTemplate_IndividualAuthImplAsync(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("mvcindividual" + (useLocalDB ? "uld" : ""), Output);
 

src/ProjectTemplates/test/RazorPagesTemplateTest.cs

@@ -26,8 +26,7 @@ namespace Templates.Test
         public ITestOutputHelper Output { get; }
 
         [Fact]
-        [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2327", FlakyOn.All)]
+        public async Task RazorPagesTemplate_NoAuth()
-        public async Task RazorPagesTemplate_NoAuthImplAsync()
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpagesnoauth", Output);
 
@@ -95,10 +94,9 @@ namespace Templates.Test
         }
 
         [Theory]
-        [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2335", FlakyOn.All)]
         [InlineData(false)]
         [InlineData(true)]
-        public async Task RazorPagesTemplate_IndividualAuthImplAsync(bool useLocalDB)
+        public async Task RazorPagesTemplate_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("razorpagesindividual" + (useLocalDB ? "uld" : ""), Output);
 

src/ProjectTemplates/test/WebApiTemplateTest.cs

@@ -22,10 +22,13 @@ namespace Templates.Test
 
         public Project Project { get; set; }
 
-        [Theory]
+        [Fact]
-        [InlineData(null)]
+        public async Task WebApiTemplateFSharp() => await WebApiTemplateCore(languageOverride: "F#");
-        [InlineData("F#")]
+
-        public async Task WebApiTemplateAsync(string languageOverride)
+        [Fact]
+        public async Task WebApiTemplateCSharp() => await WebApiTemplateCore(languageOverride: null);
+
+        private async Task WebApiTemplateCore(string languageOverride)
         {
             Project = await FactoryFixture.GetOrCreateProject("webapi" + (languageOverride == "F#" ? "fsharp" : "csharp"), Output);