Update shutdown logic for hosted applications

- Guarantee the ordering of ApplicationLifetime events
- Guarantee that the callbacks of each event is completed before the next event is triggered

src/Microsoft.AspNetCore.Hosting/Internal/ApplicationLifetime.cs

@@ -39,13 +39,19 @@ namespace Microsoft.AspNetCore.Hosting.Internal
         /// </summary>
         public void StopApplication()
         {
-            try
+            // Lock on CTS to synchronize multiple calls to StopApplication. This guarantees that the first call 
+            // to StopApplication and its callbacks run to completion before subsequent calls to StopApplication, 
+            // which will no-op since the first call already requested cancellation, get a chance to execute.
+            lock (_stoppingSource)
             {
-                _stoppingSource.Cancel(throwOnFirstException: false);
+                try
-            }
+                {
-            catch (Exception)
+                    _stoppingSource.Cancel(throwOnFirstException: false);
-            {
+                }
-                // TODO: LOG
+                catch (Exception)
+                {
+                    // TODO: LOG
+                }
             }
         }
 

test/Microsoft.AspNetCore.Hosting.Tests/WebHostTests.cs

@@ -166,6 +166,67 @@ namespace Microsoft.AspNetCore.Hosting
             Assert.Equal(1, _startInstances[0].DisposeCalls);
         }
 
+        [Fact]
+        public void WebHostApplicationLifetimeEventsOrderedCorrectlyDuringShutdown()
+        {
+            var host = CreateBuilder()
+                .UseServer(this)
+                .UseStartup("Microsoft.AspNetCore.Hosting.Tests")
+                .Build();
+
+            var lifetime = host.Services.GetRequiredService<IApplicationLifetime>();
+            var applicationStartedEvent = new ManualResetEventSlim(false);
+            var applicationStoppingEvent = new ManualResetEventSlim(false);
+            var applicationStoppedEvent = new ManualResetEventSlim(false);
+            var applicationStartedCompletedBeforeApplicationStopping = false;
+            var applicationStoppingCompletedBeforeApplicationStopped = false;
+            var applicationStoppedCompletedBeforeRunCompleted = false;
+
+            lifetime.ApplicationStarted.Register(() =>
+            {
+                applicationStartedEvent.Set();
+            });
+
+            lifetime.ApplicationStopping.Register(() =>
+            {
+                // Check whether the applicationStartedEvent has been set
+                applicationStartedCompletedBeforeApplicationStopping = applicationStartedEvent.IsSet;
+
+                // Simulate work.
+                Thread.Sleep(1000);
+
+                applicationStoppingEvent.Set();
+            });
+
+            lifetime.ApplicationStopped.Register(() =>
+            {
+                // Check whether the applicationStoppingEvent has been set
+                applicationStoppingCompletedBeforeApplicationStopped = applicationStoppingEvent.IsSet;
+                applicationStoppedEvent.Set();
+            });
+
+            var runHostAndVerifyApplicationStopped = Task.Run(() =>
+            {
+                host.Run();
+                // Check whether the applicationStoppingEvent has been set
+                applicationStoppedCompletedBeforeRunCompleted = applicationStoppedEvent.IsSet;
+            });
+
+            // Wait until application has started to shut down the host
+            Assert.True(applicationStartedEvent.Wait(5000));
+
+            // Trigger host shutdown on a separate thread
+            Task.Run(() => lifetime.StopApplication());
+
+            // Wait for all events and host.Run() to complete
+            Assert.True(runHostAndVerifyApplicationStopped.Wait(5000));
+
+            // Verify Ordering
+            Assert.True(applicationStartedCompletedBeforeApplicationStopping);
+            Assert.True(applicationStoppingCompletedBeforeApplicationStopped);
+            Assert.True(applicationStoppedCompletedBeforeRunCompleted);
+        }
+
         [Fact]
         public void WebHostDisposesServiceProvider()
         {