From 2f4172e7e70b18da3b1a5d8a22a6fb397ca3b837 Mon Sep 17 00:00:00 2001
From: Mike Harder <mharder@microsoft.com>
Date: Wed, 5 Sep 2018 16:43:09 -0700
Subject: [PATCH] Reduce probability of MultipleAppTests failure (#1365)

- Decrease false-positive failure rate from 1 in 1,000 to 1 in 1,000,000
- Addresses #1350
---
 .../OutOfProcess/MultipleAppTests.cs          | 81 +++++++++++++------
 1 file changed, 58 insertions(+), 23 deletions(-)

diff --git a/test/IISExpress.FunctionalTests/OutOfProcess/MultipleAppTests.cs b/test/IISExpress.FunctionalTests/OutOfProcess/MultipleAppTests.cs
index cf647f46e7..cc270d9afa 100644
--- a/test/IISExpress.FunctionalTests/OutOfProcess/MultipleAppTests.cs
+++ b/test/IISExpress.FunctionalTests/OutOfProcess/MultipleAppTests.cs
@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Collections.Generic;
 using System.Net;
 using System.Net.Http;
@@ -25,42 +26,76 @@ namespace Microsoft.AspNetCore.Server.IISIntegration.FunctionalTests
         [ConditionalTheory]
         [InlineData(AncmVersion.AspNetCoreModule)]
         [InlineData(AncmVersion.AspNetCoreModuleV2)]
-        public async Task Startup(AncmVersion ancmVersion)
+        public Task Startup(AncmVersion ancmVersion)
         {
-            const int numApps = 10;
+            // ANCM v1 currently does *not* retry if an app fails to start the first time due to a port collision.
+            // So, this test is expected to fail on v1 approximately 1 in 1,000 times (probably of at least one collision
+            // when 10 sites choose a random port from the range 1025-48000).  Adding one retry should reduce the failure
+            // rate from 1 in 1,000 to 1 in 1,000,000.  The previous product code (with "srand(GetTickCount())") should still
+            // fail the test reliably.
+            // https://github.com/aspnet/IISIntegration/issues/1350
+            // 
+            // ANCM v2 does retry on port collisions, so no retries should be required.
+            var attempts = (ancmVersion == AncmVersion.AspNetCoreModule) ? 2 : 1;
 
-            using (var deployers = new DisposableList<ApplicationDeployer>())
+            return Retry(async () =>
             {
-                var deploymentResults = new List<DeploymentResult>();
+                const int numApps = 10;
 
-                // Deploy all apps
-                for (var i = 0; i < numApps; i++)
+                using (var deployers = new DisposableList<ApplicationDeployer>())
                 {
-                    var deploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel: IntegrationTesting.HostingModel.OutOfProcess);
-                    deploymentParameters.AncmVersion = ancmVersion;
+                    var deploymentResults = new List<DeploymentResult>();
 
-                    var deployer = CreateDeployer(deploymentParameters);
-                    deployers.Add(deployer);
-                    deploymentResults.Add(await deployer.DeployAsync());
+                    // Deploy all apps
+                    for (var i = 0; i < numApps; i++)
+                    {
+                        var deploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel: IntegrationTesting.HostingModel.OutOfProcess);
+                        deploymentParameters.AncmVersion = ancmVersion;
+
+                        var deployer = CreateDeployer(deploymentParameters);
+                        deployers.Add(deployer);
+                        deploymentResults.Add(await deployer.DeployAsync());
+                    }
+
+                    // Start all requests as quickly as possible, so apps are started as quickly as possible,
+                    // to test possible race conditions when multiple apps start at the same time.
+                    var requestTasks = new List<Task<HttpResponseMessage>>();
+                    foreach (var deploymentResult in deploymentResults)
+                    {
+                        requestTasks.Add(deploymentResult.HttpClient.GetAsync("/HelloWorld"));
+                    }
+
+                    // Verify all apps started and return expected response
+                    foreach (var requestTask in requestTasks)
+                    {
+                        var response = await requestTask;
+                        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+                        var responseText = await response.Content.ReadAsStringAsync();
+                        Assert.Equal("Hello World", responseText);
+                    }
                 }
+            },
+            attempts: attempts);
+        }
 
-                // Start all requests as quickly as possible, so apps are started as quickly as possible,
-                // to test possible race conditions when multiple apps start at the same time.
-                var requestTasks = new List<Task<HttpResponseMessage>>();
-                foreach (var deploymentResult in deploymentResults)
+        private async Task Retry(Func<Task> func, int attempts)
+        {
+            var exceptions = new List<Exception>();
+
+            for (var attempt = 0; attempt < attempts; attempt++)
+            {
+                try
                 {
-                    requestTasks.Add(deploymentResult.HttpClient.GetAsync("/HelloWorld"));
+                    await func();
+                    return;
                 }
-
-                // Verify all apps started and return expected response
-                foreach (var requestTask in requestTasks)
+                catch (Exception e)
                 {
-                    var response = await requestTask;
-                    Assert.Equal(HttpStatusCode.OK, response.StatusCode);
-                    var responseText = await response.Content.ReadAsStringAsync();
-                    Assert.Equal("Hello World", responseText);
+                    exceptions.Add(e);
                 }
             }
+
+            throw new AggregateException(exceptions);
         }
     }
 }