Jhkim/add test gracefulshutdown (#120)

Add new test for Graceful shutdown and Filtering MS request headers

test/AspNetCoreModule.Test/Framework/InitializeTestMachine.cs

@@ -13,7 +13,8 @@ namespace AspNetCoreModule.Test.Framework
         public const string ANCMTestFlagsEnvironmentVariable = "%ANCMTestFlags%";
         public const string ANCMTestFlagsDefaultContext = "AdminAnd64Bit";
         public const string ANCMTestFlagsTestSkipContext = "SkipTest";
-        public const string ANCMTestFlagsUsePrivateAspNetCoreFileContext = "UsePrivateAspNetCoreFile";
+        public const string ANCMTestFlagsUsePrivateAspNetCoreFileContext = "UsePrivate";
+        public const string ANCMTestFlagsUseIISExpressContext = "UseIISExpress";
 
         private static bool? _usePrivateAspNetCoreFile = null;
         public static bool? UsePrivateAspNetCoreFile
@@ -115,10 +116,16 @@ namespace AspNetCoreModule.Test.Framework
                     IISConfigUtility.IsIISReady = false;
                     if (IISConfigUtility.IsIISInstalled == true)
                     {
-                        if (Environment.GetEnvironmentVariable("ANCMTEST_USE_IISEXPRESS") != null && Environment.GetEnvironmentVariable("ANCMTEST_USE_IISEXPRESS").Equals("true", StringComparison.InvariantCultureIgnoreCase))
+                        var envValue = Environment.ExpandEnvironmentVariables(ANCMTestFlagsEnvironmentVariable);
+                        if (envValue.ToLower().Contains(ANCMTestFlagsUseIISExpressContext.ToLower()))
                         {
+                            TestUtility.LogInformation("UseIISExpress is set");
                             throw new System.ApplicationException("'ANCMTestServerType' environment variable is set to 'true'");
                         }
+                        else
+                        {
+                            TestUtility.LogInformation("UseIISExpress is not set");
+                        }
                         
                         // check websocket is installed
                         if (File.Exists(Path.Combine(IISConfigUtility.Strings.IIS64BitPath, "iiswsock.dll")))

test/AspNetCoreModule.Test/FunctionalTest.cs

@@ -39,15 +39,21 @@ namespace AspNetCoreModule.Test
         [ANCMTestSkipCondition("%ANCMTestFlags%")]
         [OSSkipCondition(OperatingSystems.Linux)]
         [OSSkipCondition(OperatingSystems.MacOSX)]
-        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 25, 19)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 25, 19, false)]
-        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 25, 19)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 25, 19, true)]
-        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 5, 4)]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 25, 19, false)]
-        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 5, 4)]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 25, 19, true)]
-        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 0, 0)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 5, 4, true)]
-        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 0, 0)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 5, 4, false)]
-        public Task ShutdownTimeLimitTest(IISConfigUtility.AppPoolBitness appPoolBitness, int valueOfshutdownTimeLimit, int expectedClosingTime)
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 5, 4, true)]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 5, 4, false)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 0, 0, false)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, 0, 0, true)]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 0, 0, false)]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, 0, 0, true)]
+        public Task ShutdownTimeLimitTest(IISConfigUtility.AppPoolBitness appPoolBitness, int valueOfshutdownTimeLimit, int expectedClosingTime, bool isGraceFullShutdownEnabled)
         {
-            return DoShutdownTimeLimitTest(appPoolBitness, valueOfshutdownTimeLimit, expectedClosingTime);
+            return DoShutdownTimeLimitTest(appPoolBitness, valueOfshutdownTimeLimit, expectedClosingTime, isGraceFullShutdownEnabled);
         }
         
         [ConditionalTheory]
@@ -275,6 +281,21 @@ namespace AspNetCoreModule.Test
             return DoSendHTTPSRequestTest(appPoolBitness);
         }
 
+        [ConditionalTheory]
+        [ANCMTestSkipCondition("%ANCMTestFlags%")]
+        [OSSkipCondition(OperatingSystems.Linux)]
+        [OSSkipCondition(OperatingSystems.MacOSX)]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, "MS-ASPNETCORE", "f")]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, "mS-ASPNETCORE", "fo")]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, "MS-ASPNETCORE-", "foo")]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, "mS-ASPNETCORE-f", "fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo")]
+        [InlineData(IISConfigUtility.AppPoolBitness.enable32Bit, "MS-ASPNETCORE-foo", "foo")]
+        [InlineData(IISConfigUtility.AppPoolBitness.noChange, "MS-ASPNETCORE-foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo", "bar")]
+        public Task FilterOutMSRequestHeadersTest(IISConfigUtility.AppPoolBitness appPoolBitness, string requestHeader, string requestHeaderValue)
+        {
+            return DoFilterOutMSRequestHeadersTest(appPoolBitness, requestHeader, requestHeaderValue);
+        }
+
         [ConditionalTheory]
         [ANCMTestSkipCondition("%ANCMTestFlags%")]
         [OSSkipCondition(OperatingSystems.Linux)]

test/AspNetCoreModule.Test/FunctionalTestHelper.cs

@@ -711,37 +711,54 @@ namespace AspNetCoreModule.Test
             }
         }
 
-        public static async Task DoShutdownTimeLimitTest(IISConfigUtility.AppPoolBitness appPoolBitness, int valueOfshutdownTimeLimit, int expectedClosingTime)
+        public static async Task DoShutdownTimeLimitTest(IISConfigUtility.AppPoolBitness appPoolBitness, int valueOfshutdownTimeLimit, int expectedClosingTime, bool isGraceFullShutdownEnabled)
         {
             using (var testSite = new TestWebSite(appPoolBitness, "DoShutdownTimeLimitTest"))
             {
                 using (var iisConfig = new IISConfigUtility(testSite.IisServerType, testSite.IisExpressConfigPath))
                 {
-                    // Set new value (10 second) to make the backend process get the Ctrl-C signal and measure when the recycle happens
+                    DateTime startTime = DateTime.Now;
-                    iisConfig.SetANCMConfig(testSite.SiteName, testSite.AspNetCoreApp.Name, "shutdownTimeLimit", valueOfshutdownTimeLimit);
+
-                    iisConfig.SetANCMConfig(
+                    // Make shutdownDelay time with hard coded value such as 20 seconds and test vairious shutdonwTimeLimit, either less than 20 seconds or bigger then 20 seconds
-                        testSite.SiteName, 
+                    int shutdownDelayTime = 20000;
-                        testSite.AspNetCoreApp.Name, 
+                    iisConfig.SetANCMConfig(testSite.SiteName, testSite.AspNetCoreApp.Name, "shutdownTimeLimit", valueOfshutdownTimeLimit);
-                        "environmentVariable", 
+                    iisConfig.SetANCMConfig(testSite.SiteName, testSite.AspNetCoreApp.Name, "environmentVariable", new string[] { "ANCMTestShutdownDelay", shutdownDelayTime.ToString() });
-                        new string[] { "ANCMTestShutdownDelay", "20000" }
+                    string expectedGracefulShutdownResponseStatusCode = "202";
-                        );
+                    if (!isGraceFullShutdownEnabled)
+                    {
+                        iisConfig.SetANCMConfig(testSite.SiteName, testSite.AspNetCoreApp.Name, "environmentVariable", new string[] { "GracefulShutdown", "disabled" });
+                        expectedGracefulShutdownResponseStatusCode = "200";
+                    }
+
+                    string response = await GetResponse(testSite.AspNetCoreApp.GetUri(""), HttpStatusCode.OK);
+                    Assert.True(response == "Running");
                     
-                    await VerifyResponseBody(testSite.AspNetCoreApp.GetUri(), "Running", HttpStatusCode.OK);
                     string backendProcessId = await GetResponse(testSite.AspNetCoreApp.GetUri("GetProcessId"), HttpStatusCode.OK);
                     var backendProcess = Process.GetProcessById(Convert.ToInt32(backendProcessId));
 
-                    // Set a new value such as 100 to make the backend process being recycled
+                    // Set a new configuration value to make the backend process being recycled
-                    DateTime startTime = DateTime.Now;
+                    DateTime startTime2 = DateTime.Now;
                     iisConfig.SetANCMConfig(testSite.SiteName, testSite.AspNetCoreApp.Name, "shutdownTimeLimit", 100);
                     backendProcess.WaitForExit(30000);
                     DateTime endTime = DateTime.Now;
-                    var difference = endTime - startTime;
+                    var difference = endTime - startTime2;
                     Assert.True(difference.Seconds >= expectedClosingTime);
                     Assert.True(difference.Seconds < expectedClosingTime + 3);
-                    Assert.True(backendProcessId != await GetResponse(testSite.AspNetCoreApp.GetUri("GetProcessId"), HttpStatusCode.OK));
+                    string newBackendProcessId = await GetResponse(testSite.AspNetCoreApp.GetUri("GetProcessId"), HttpStatusCode.OK);
+                    Assert.True(backendProcessId != newBackendProcessId);
                     await VerifyResponseBody(testSite.AspNetCoreApp.GetUri(), "Running", HttpStatusCode.OK);
-                }
 
+                    // if expectedClosing time is less than the shutdownDelay time, gracefulshutdown is supposed to fail and failure event is expected
+                    if (expectedClosingTime*1000 + 1000 == shutdownDelayTime)
+                    {
+                        Assert.True(TestUtility.RetryHelper((arg1, arg2) => VerifyANCMGracefulShutdownEvent(arg1, arg2), startTime, backendProcessId));
+                        Assert.True(TestUtility.RetryHelper((arg1, arg2) => VerifyANCMGracefulShutdownEvent(arg1, arg2), startTime, expectedGracefulShutdownResponseStatusCode));
+                    }
+                    else
+                    {
+                        Assert.True(TestUtility.RetryHelper((arg1, arg2) => VerifyANCMGracefulShutdownFailureEvent(arg1, arg2), startTime, backendProcessId));
+                    }
+                }
                 testSite.AspNetCoreApp.RestoreFile("web.config");
             }
         }
@@ -1180,6 +1197,58 @@ namespace AspNetCoreModule.Test
             }
         }
 
+        public static async Task DoFilterOutMSRequestHeadersTest(IISConfigUtility.AppPoolBitness appPoolBitness, string requestHeader, string requestHeaderValue)
+        {
+            using (var testSite = new TestWebSite(appPoolBitness, "DoSendHTTPSRequestTest", startIISExpress: false))
+            {
+                using (var iisConfig = new IISConfigUtility(testSite.IisServerType, testSite.IisExpressConfigPath))
+                {
+                    string hostName = "";
+                    string subjectName = "localhost";
+                    string ipAddress = "*";
+                    string hexIPAddress = "0x00";
+                    int sslPort = InitializeTestMachine.SiteId + 6300;
+
+                    // Add https binding and get https uri information
+                    iisConfig.AddBindingToSite(testSite.SiteName, ipAddress, sslPort, hostName, "https");
+
+                    // Create a self signed certificate
+                    string thumbPrint = iisConfig.CreateSelfSignedCertificate(subjectName);
+
+                    // Export the self signed certificate to rootCA
+                    iisConfig.ExportCertificateTo(thumbPrint, sslStoreTo: @"Cert:\LocalMachine\Root");
+
+                    // Configure http.sys ssl certificate mapping to IP:Port endpoint with the newly created self signed certificage
+                    iisConfig.SetSSLCertificate(sslPort, hexIPAddress, thumbPrint);
+
+                    // starting IISExpress was deffered after creating test applications and now it is ready to start it 
+                    testSite.StartIISExpress();
+
+                    // Verify http request
+                    string requestHeaders = string.Empty;
+                    requestHeaders = await GetResponseAndHeaders(testSite.AspNetCoreApp.GetUri("DumpRequestHeaders"), new string[] { "Accept-Encoding", "gzip", requestHeader, requestHeaderValue }, HttpStatusCode.OK);
+                    requestHeaders = requestHeaders.Replace(" ", "");
+                    Assert.False(requestHeaders.ToUpper().Contains(requestHeader.ToLower()+":"));
+
+                    // Verify https request
+                    Uri targetHttpsUri = testSite.AspNetCoreApp.GetUri(null, sslPort, protocol: "https");
+                    requestHeader = await GetResponseAndHeaders(targetHttpsUri, new string[] { "Accept-Encoding", "gzip", requestHeader, requestHeaderValue }, HttpStatusCode.OK);
+                    requestHeaders = requestHeaders.Replace(" ", "");
+                    Assert.False(requestHeaders.ToUpper().Contains(requestHeader.ToLower() + ":"));
+
+                    // Remove the SSL Certificate mapping
+                    iisConfig.RemoveSSLCertificate(sslPort, hexIPAddress);
+
+                    // Remove the newly created self signed certificate
+                    iisConfig.DeleteCertificate(thumbPrint);
+
+                    // Remove the exported self signed certificate on rootCA
+                    iisConfig.DeleteCertificate(thumbPrint, @"Cert:\LocalMachine\Root");
+                }
+                testSite.AspNetCoreApp.RestoreFile("web.config");
+            }
+        }
+
         public static async Task DoClientCertificateMappingTest(IISConfigUtility.AppPoolBitness appPoolBitness, bool useHTTPSMiddleWare)
         {
             using (var testSite = new TestWebSite(appPoolBitness, "DoClientCertificateMappingTest", startIISExpress: false))
@@ -1512,6 +1581,16 @@ namespace AspNetCoreModule.Test
             return VerifyEventLog(1001, startFrom, includeThis);
         }
 
+        private static bool VerifyANCMGracefulShutdownEvent(DateTime startFrom, string includeThis)
+        {
+            return VerifyEventLog(1006, startFrom, includeThis);
+        }
+
+        private static bool VerifyANCMGracefulShutdownFailureEvent(DateTime startFrom, string includeThis)
+        {
+            return VerifyEventLog(1005, startFrom, includeThis);
+        }
+
         private static bool VerifyApplicationEventLog(int eventID, DateTime startFrom, string includeThis)
         {
             return VerifyEventLog(eventID, startFrom, includeThis);

test/AspNetCoreModule.TestSites.Standard/Program.cs

@@ -14,6 +14,9 @@ namespace AspnetCoreModule.TestSites.Standard
 {
     public static class Program
     {
+        public static IApplicationLifetime AappLifetime;
+        public static int GracefulShutdownDelayTime = 0;
+
         private static X509Certificate2 _x509Certificate2;
 
         public static void Main(string[] args)
@@ -141,7 +144,34 @@ namespace AspnetCoreModule.TestSites.Standard
             }
 
             var host = builder.Build();
-            host.Run();
+            AappLifetime = (IApplicationLifetime)host.Services.GetService(typeof(IApplicationLifetime));
+
+            string gracefulShutdownDelay = Environment.GetEnvironmentVariable("GracefulShutdownDelayTime");
+            if (!string.IsNullOrEmpty(gracefulShutdownDelay))
+            {
+                GracefulShutdownDelayTime = Convert.ToInt32(gracefulShutdownDelay);
+            }
+            
+            AappLifetime.ApplicationStarted.Register(
+                () => Thread.Sleep(1000)
+            );
+            AappLifetime.ApplicationStopping.Register(
+                () => Thread.Sleep(Startup.SleeptimeWhileClosing / 2)
+            );
+            AappLifetime.ApplicationStopped.Register(
+                () => {
+                    Thread.Sleep(Startup.SleeptimeWhileClosing / 2);
+                    Startup.SleeptimeWhileClosing = 0;   // All of SleeptimeWhileClosing is used now
+                    }
+            );
+            try
+            {
+                host.Run();                
+            }
+            catch
+            {
+                // ignore
+            }
             
             if (Startup.SleeptimeWhileClosing != 0)
             {

test/AspNetCoreModule.TestSites.Standard/Startup.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -304,6 +305,20 @@ namespace AspnetCoreModule.TestSites.Standard
                         }
                     }
                 }
+
+                // Handle shutdown event from ANCM
+                if (HttpMethods.IsPost(context.Request.Method) &&
+                    //context.Request.Path.Equals(ANCMRequestPath) &&
+                    string.Equals("shutdown", context.Request.Headers["MS-ASPNETCORE-EVENT"], StringComparison.OrdinalIgnoreCase))
+                {
+                    response = "shutdown";
+                    string shutdownMode = Environment.GetEnvironmentVariable("GracefulShutdown");
+                    if (String.IsNullOrEmpty(shutdownMode) || !shutdownMode.ToLower().StartsWith("disabled"))
+                    {
+                        context.Response.StatusCode = StatusCodes.Status202Accepted;
+                        Program.AappLifetime.StopApplication();
+                    }
+                }
                 return context.Response.WriteAsync(response);
             });
         }