change the logic on handling configuration change and adds shutdown logic (#610)

src/AspNetCore/Inc/applicationinfo.h

@@ -134,6 +134,20 @@ public:
         return m_pApplication;
     }
 
+    VOID
+    ClearAndDereferenceApplication()
+    {
+        APPLICATION* pApplication = m_pApplication;
+        m_pApplication = NULL;
+        pApplication->DereferenceApplication();
+    }
+
+    VOID
+    ClearApplication()
+    {
+        m_pApplication = NULL;
+    }
+
     HRESULT
     EnsureApplicationCreated();
 
@@ -211,6 +225,18 @@ public:
         pApplicationInfo->DereferenceApplicationInfo();
     }
 
+    static
+    VOID
+    ReferenceCopyToTable(
+        APPLICATION_INFO *        pEntry,
+        PVOID                     pvData
+    )
+    {
+        APPLICATION_INFO_HASH *pHash = static_cast<APPLICATION_INFO_HASH *>(pvData);
+        DBG_ASSERT(pHash);
+        pHash->InsertRecord(pEntry);
+    }
+
 private:
 
     APPLICATION_INFO_HASH(const APPLICATION_INFO_HASH &);

src/AspNetCore/Inc/applicationmanager.h

@@ -3,13 +3,20 @@
 
 #pragma once
 
-#define DEFAULT_HASH_BUCKETS 293
+#define DEFAULT_HASH_BUCKETS 17
 
 //
 // This class will manage the lifecycle of all Asp.Net Core applciation
 // It should be global singleton.
 // Should always call GetInstance to get the object instance
 //
+
+struct CONFIG_CHANGE_CONTEXT
+{
+    PCWSTR   pstrPath;
+    MULTISZ  MultiSz;
+};
+
 class APPLICATION_MANAGER
 {
 public:
@@ -41,15 +48,42 @@ public:
         }
     }
 
+    static
+    BOOL
+    FindConfigChangedApplication(
+        _In_ APPLICATION_INFO *     pEntry,
+        _In_ PVOID                  pvContext
+    );
+
+    static
+    VOID
+    RecycleApplication(
+        _In_ APPLICATION_INFO *     pEntry,
+        _In_ APP_HOSTING_MODEL      hostingModel
+    );
+
+    static
+    void
+    DoRecycleApplication(
+        LPVOID lpParam
+    );
+
+    static
+    VOID
+    ShutdownApplication(
+        _In_ APPLICATION_INFO *     pEntry,
+        _In_ PVOID                  pvContext
+    );
+
     HRESULT
-    GetApplicationInfo(
+    GetOrCreateApplicationInfo(
         _In_ IHttpServer*          pServer,
         _In_ ASPNETCORE_CONFIG*    pConfig,
         _Out_ APPLICATION_INFO **  ppApplicationInfo
     );
 
     HRESULT
-    RecycleApplication(
+    RecycleApplicationFromManager(
         _In_ LPCWSTR pszApplicationId
     );
 
@@ -120,8 +154,7 @@ private:
     // 
     APPLICATION_MANAGER() : m_pApplicationInfoHash(NULL), 
         m_pFileWatcher(NULL),
-        m_hostingModel(HOSTING_UNKNOWN),
+        m_hostingModel(HOSTING_UNKNOWN)
-        m_fInShutdown(FALSE)
     {
         InitializeSRWLock(&m_srwLock);
     }
@@ -131,5 +164,4 @@ private:
     static APPLICATION_MANAGER *sm_pApplicationManager;
     SRWLOCK                     m_srwLock;
     APP_HOSTING_MODEL          m_hostingModel;
-    bool                       m_fInShutdown;
 };

src/AspNetCore/Inc/globalmodule.h

@@ -28,8 +28,8 @@ public:
     );
 
     GLOBAL_NOTIFICATION_STATUS
-    OnGlobalApplicationStop(
+    OnGlobalConfigurationChange(
-        _In_ IHttpApplicationStopProvider * pProvider
+        _In_ IGlobalConfigurationChangeProvider * pProvider
     );
 
 private:

src/AspNetCore/Src/applicationinfo.cpp

@@ -89,6 +89,10 @@ APPLICATION_INFO::StartMonitoringAppOffline()
     return hr;
 }
 
+//
+// Called by the file watcher when the app_offline.htm's file status has been changed.
+// If it finds it, we will call recycle on the application.
+//
 VOID
 APPLICATION_INFO::UpdateAppOfflineFileHandle()
 {
@@ -98,15 +102,30 @@ APPLICATION_INFO::UpdateAppOfflineFileHandle()
         &strFilePath);
     APP_OFFLINE_HTM *pOldAppOfflineHtm = NULL;
     APP_OFFLINE_HTM *pNewAppOfflineHtm = NULL;
+    BOOL             fLocked = FALSE;
 
     if (INVALID_FILE_ATTRIBUTES == GetFileAttributes(strFilePath.QueryStr()) &&
         GetLastError() == ERROR_FILE_NOT_FOUND)
     {
+        // Check if app offline was originally present.
+        // if it was, log that app_offline has been dropped.
+        if (m_fAppOfflineFound)
+        {
+            STACK_STRU(strEventMsg, 256);
+            if (SUCCEEDED(strEventMsg.SafeSnwprintf(
+                ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_REMOVED_MSG)))
+            {
+                UTILITY::LogEvent(g_hEventLog,
+                    EVENTLOG_INFORMATION_TYPE,
+                    ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_REMOVED,
+                    strEventMsg.QueryStr());
+            }
+        }
+
         m_fAppOfflineFound = FALSE;
     }
     else
     {
-        m_fAppOfflineFound = TRUE;
         pNewAppOfflineHtm = new APP_OFFLINE_HTM(strFilePath.QueryStr());
 
         if (pNewAppOfflineHtm != NULL)
@@ -132,9 +151,18 @@ APPLICATION_INFO::UpdateAppOfflineFileHandle()
             }
         }
 
+        m_fAppOfflineFound = TRUE;
+
         // recycle the application
         if (m_pApplication != NULL)
         {
+            // Lock here to avoid races with the application manager calling shutdown on the application.
+            AcquireSRWLockExclusive(&m_srwLock);
+            fLocked = TRUE;
+            if (m_pApplication == NULL)
+            {
+                goto Finished;
+            }
             STACK_STRU(strEventMsg, 256);
             if (SUCCEEDED(strEventMsg.SafeSnwprintf(
                 ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_MSG,
@@ -146,12 +174,16 @@ APPLICATION_INFO::UpdateAppOfflineFileHandle()
                     strEventMsg.QueryStr());
             }
 
-            m_pApplication->ShutDown();
+            APPLICATION_MANAGER::RecycleApplication(this, m_pConfiguration->QueryHostingModel());
-            m_pApplication->DereferenceApplication();
-            m_pApplication = NULL;
 
         }
     }
+
+Finished:
+    if (fLocked)
+    {
+        ReleaseSRWLockExclusive(&m_srwLock);
+    }
 }
 
 HRESULT

src/AspNetCore/Src/applicationmanager.cxx

@@ -3,10 +3,15 @@
 
 #include "precomp.hxx"
 
+// The application manager is a singleton across ANCM.
 APPLICATION_MANAGER* APPLICATION_MANAGER::sm_pApplicationManager = NULL;
 
+//
+// Retrieves the application info from the application manager
+// Will create the application info if it isn't initalized
+//
 HRESULT
-APPLICATION_MANAGER::GetApplicationInfo(
+APPLICATION_MANAGER::GetOrCreateApplicationInfo(
     _In_ IHttpServer*          pServer,
     _In_ ASPNETCORE_CONFIG*    pConfig,
     _Out_ APPLICATION_INFO **  ppApplicationInfo
@@ -21,21 +26,25 @@ APPLICATION_MANAGER::GetApplicationInfo(
     PCWSTR                 pszApplicationId = NULL;
     STACK_STRU ( strEventMsg, 256 );
 
+    DBG_ASSERT(pServer);
+    DBG_ASSERT(pConfig);
+    DBG_ASSERT(ppApplicationInfo);
+
     *ppApplicationInfo = NULL;
 
-    DBG_ASSERT(pServer != NULL);
+    // The configuration path is unique for each application and is used for the 
-    DBG_ASSERT(pConfig != NULL);
+    // key in the applicationInfoHash.
-
     pszApplicationId = pConfig->QueryConfigPath()->QueryStr(); 
-
     hr = key.Initialize(pszApplicationId);
     if (FAILED(hr))
     {
         goto Finished;
     }
 
+    // When accessing the m_pApplicationInfoHash, we need to acquire the application manager
+    // lock to avoid races on setting state.
     AcquireSRWLockShared(&m_srwLock);
-    if (m_fInShutdown)
+    if (g_fInShutdown)
     {
         ReleaseSRWLockShared(&m_srwLock);
         hr = HRESULT_FROM_WIN32(ERROR_SERVER_SHUTDOWN_IN_PROGRESS);
@@ -46,6 +55,7 @@ APPLICATION_MANAGER::GetApplicationInfo(
 
     if (*ppApplicationInfo == NULL)
     {
+        // Check which hosting model we want to support 
         switch (pConfig->QueryHostingModel())
         {
         case HOSTING_IN_PROCESS:
@@ -74,7 +84,7 @@ APPLICATION_MANAGER::GetApplicationInfo(
 
         AcquireSRWLockExclusive(&m_srwLock);
         fExclusiveLock = TRUE;
-        if (m_fInShutdown)
+        if (g_fInShutdown)
         {
             // Already in shuting down. No need to create the application
             hr = HRESULT_FROM_WIN32(ERROR_SERVER_SHUTDOWN_IN_PROGRESS);
@@ -124,10 +134,12 @@ APPLICATION_MANAGER::GetApplicationInfo(
         }
 
         *ppApplicationInfo = pApplicationInfo;
-        ReleaseSRWLockExclusive(&m_srwLock);
-        fExclusiveLock = FALSE;
-
         pApplicationInfo->StartMonitoringAppOffline();
+
+        // Release the locko after starting to monitor for app offline to avoid races with it being dropped.
+        ReleaseSRWLockExclusive(&m_srwLock);
+
+        fExclusiveLock = FALSE;
         pApplicationInfo = NULL;
     }
 
@@ -189,39 +201,131 @@ Finished:
     return hr;
 }
 
+
+//
+// If the application's configuration was changed,
+// append the configuration path to the config change context.
+//
+BOOL
+APPLICATION_MANAGER::FindConfigChangedApplication(
+    _In_ APPLICATION_INFO *     pEntry,
+    _In_ PVOID                  pvContext)
+{
+    DBG_ASSERT(pEntry);
+    DBG_ASSERT(pvContext);
+
+    // Config Change context contains the original config path that changed
+    // and a multiStr containing 
+    CONFIG_CHANGE_CONTEXT* pContext = static_cast<CONFIG_CHANGE_CONTEXT*>(pvContext);
+    STRU* pstruConfigPath = pEntry->QueryConfig()->QueryConfigPath();
+
+    // check if the application path contains our app/subapp by seeing if the config path
+    // starts with the notification path.
+    BOOL fChanged = pstruConfigPath->StartsWith(pContext->pstrPath, true);
+    if (fChanged)
+    {
+        DWORD dwLen = (DWORD)wcslen(pContext->pstrPath);
+        WCHAR wChar = pstruConfigPath->QueryStr()[dwLen];
+
+        // We need to check that the last character of the config path
+        // is either a null terminator or a slash.
+        // This checks the case where the config path was
+        // MACHINE/WEBROOT/site and your site path is MACHINE/WEBROOT/siteTest
+        if (wChar != L'\0' && wChar != L'/')
+        {
+            // not current app or sub app
+            fChanged = FALSE;
+        }
+        else
+        {
+            pContext->MultiSz.Append(*pstruConfigPath);
+        }
+    }
+    return fChanged;
+}
+
+//
+// Finds any applications affected by a configuration change and calls Recycle on them
+// InProcess:  Triggers g_httpServer->RecycleProcess() and keep the application inside of the manager.
+//             This will cause a shutdown event to occur through the global stop listening event.
+// OutOfProcess: Removes all applications in the application manager and calls Recycle, which will call Shutdown,
+//             on each application.
+// 
 HRESULT
-APPLICATION_MANAGER::RecycleApplication(
+APPLICATION_MANAGER::RecycleApplicationFromManager(
     _In_ LPCWSTR pszApplicationId
 )
 {
-    HRESULT          hr = S_OK;
+    HRESULT                 hr = S_OK;
-    APPLICATION_INFO_KEY  key;
+    APPLICATION_INFO_KEY    key;
-    DWORD            dwPreviousCounter = 0;
+    DWORD                   dwPreviousCounter = 0;
+    APPLICATION_INFO_HASH*  table = NULL;
+    CONFIG_CHANGE_CONTEXT   context;
+    BOOL                    fKeepTable = FALSE;
+
+    if (g_fInShutdown)
+    {
+        // We are already shutting down, ignore this event as a global configuration change event
+        // can occur after global stop listening for some reason.
+        return hr;
+    }
+
+    AcquireSRWLockExclusive(&m_srwLock);
+    if (g_fInShutdown)
+    {
+        ReleaseSRWLockExclusive(&m_srwLock);
+        return hr;
+    }
 
     hr = key.Initialize(pszApplicationId);
     if (FAILED(hr))
     {
         goto Finished;
     }
-    AcquireSRWLockExclusive(&m_srwLock);
+    // Make a shallow copy of existing hashtable as we may need to remove nodes
+    // This will be used for finding differences in which applications are affected by a config change.
+    table = new APPLICATION_INFO_HASH();
+
+    if (table == NULL)
+    {
+        hr = E_OUTOFMEMORY;
+        goto Finished;
+    }
+
+    // few application expected, small bucket size for hash table
+    if (FAILED(hr = table->Initialize(17 /*prime*/)))
+    {
+        goto Finished;
+    }
+
+    context.pstrPath = pszApplicationId;
+
+    // Keep track of the preview count of applications to know whether there are applications to delete
     dwPreviousCounter = m_pApplicationInfoHash->Count();
 
-    m_pApplicationInfoHash->DeleteKey(&key);
+    // We don't want to hold the application manager lock for long time as it will block all incoming requests
+    // Don't call application shutdown inside the lock
+    m_pApplicationInfoHash->Apply(APPLICATION_INFO_HASH::ReferenceCopyToTable, static_cast<PVOID>(table));
+    DBG_ASSERT(dwPreviousCounter == table->Count());
     
-    if(dwPreviousCounter != m_pApplicationInfoHash->Count())
+    // Removed the applications which are impacted by the configurtion change
+    m_pApplicationInfoHash->DeleteIf(FindConfigChangedApplication, (PVOID)&context);
+
+    if (dwPreviousCounter != m_pApplicationInfoHash->Count())
     {
-        // Application got recycled. Log an event
+        if (m_hostingModel == HOSTING_IN_PROCESS)
-        STACK_STRU(strEventMsg, 256);
-        if (SUCCEEDED(strEventMsg.SafeSnwprintf(
-            ASPNETCORE_EVENT_RECYCLE_CONFIGURATION_MSG,
-            pszApplicationId)))
         {
-            UTILITY::LogEvent(g_hEventLog,
+            // When we are inprocess, we need to keep the application the
-                EVENTLOG_INFORMATION_TYPE,
+            // application manager that is being deleted. This is because we will always need to recycle the worker
-                ASPNETCORE_EVENT_RECYCLE_CONFIGURATION,
+            // process and any requests that hit this worker process must be rejected (while out of process can
-                strEventMsg.QueryStr());
+            // start a new dotnet process). We will immediately call Recycle after this call.
-        }
+            DBG_ASSERT(m_pApplicationInfoHash->Count() == 0);
+            delete m_pApplicationInfoHash;
 
+            // We don't want to delete the table as m_pApplicationInfoHash = table
+            fKeepTable = TRUE;
+            m_pApplicationInfoHash = table;
+        }
     }
 
     if (m_pApplicationInfoHash->Count() == 0)
@@ -229,40 +333,178 @@ APPLICATION_MANAGER::RecycleApplication(
         m_hostingModel = HOSTING_UNKNOWN;
     }
 
-
-    if (g_fAspnetcoreRHLoadedError)
-    {
-        // We had assembly loading failure
-        // this error blocked the start of all applications
-        // Let's recycle the worker process if user redeployed any application
-        g_pHttpServer->RecycleProcess(L"AspNetCore Recycle Process on Demand due to assembly loading failure");
-    }
-
     ReleaseSRWLockExclusive(&m_srwLock);
 
+    // If we receive a request at this point.
+    // OutOfProcess: we will create a new application with new configuration
+    // InProcess: the request would have to be rejected, as we are about to call g_HttpServer->RecycleProcess
+    // on the worker proocess
+    if (!context.MultiSz.IsEmpty())
+    {
+        PCWSTR path = context.MultiSz.First();
+        // Iterate through each of the paths that were shut down,
+        // calling RecycleApplication on each of them.
+        while (path != NULL)
+        {
+            APPLICATION_INFO* pRecord;
+
+            // Application got recycled. Log an event
+            STACK_STRU(strEventMsg, 256);
+            if (SUCCEEDED(strEventMsg.SafeSnwprintf(
+                ASPNETCORE_EVENT_RECYCLE_CONFIGURATION_MSG,
+                path)))
+            {
+                UTILITY::LogEvent(g_hEventLog,
+                    EVENTLOG_INFORMATION_TYPE,
+                    ASPNETCORE_EVENT_RECYCLE_CONFIGURATION,
+                    strEventMsg.QueryStr());
+            }
+
+            hr = key.Initialize(path);
+            if (FAILED(hr))
+            {
+                goto Finished;
+            }
+
+            table->FindKey(&key, &pRecord);
+            DBG_ASSERT(pRecord != NULL);
+
+            // RecycleApplication is called on a separate thread. 
+            RecycleApplication(pRecord, m_hostingModel);
+            pRecord->DereferenceApplicationInfo();
+            path = context.MultiSz.Next(path);
+        }
+    }
+
 Finished:
+    if (table != NULL && !fKeepTable)
+    {
+        table->Clear();
+        delete table;
+    }
+
+    if (FAILED(hr))
+    {
+        // Failed to recycle an application. Log an event
+        STACK_STRU(strEventMsg, 256);
+        if  (SUCCEEDED(strEventMsg.SafeSnwprintf(
+                ASPNETCORE_EVENT_RECYCLE_FAILURE_CONFIGURATION_MSG,
+                pszApplicationId)))
+        {
+            UTILITY::LogEvent(g_hEventLog,
+                EVENTLOG_ERROR_TYPE,
+                ASPNETCORE_EVENT_RECYCLE_APP_FAILURE,
+                strEventMsg.QueryStr());
+        }
+        // Need to recycle the process as we cannot recycle the application
+        g_pHttpServer->RecycleProcess(L"AspNetCore Recycle Process on Demand Due Application Recycle Error");
+    }
 
     return hr;
 }
 
+//
+// Shutsdown all applications in the application hashtable
+// Only called by OnGlobalStopListening.
+// 
 VOID
 APPLICATION_MANAGER::ShutDown()
 {
-    m_fInShutdown = TRUE;
+    // We are guaranteed to only have one outstanding OnGlobalStopListening event at a time
+    // However, it is possible to receive multiple OnGlobalStopListening events
+    // Protect against this by checking if we already shut down.
+    g_fInShutdown = TRUE;
     if (m_pApplicationInfoHash != NULL)
     {
+        if (m_pFileWatcher != NULL)
+        {
+            delete  m_pFileWatcher;
+            m_pFileWatcher = NULL;
+        }
+
+        DBG_ASSERT(m_pApplicationInfoHash);
+        // During shutdown we lock until we delete the application 
         AcquireSRWLockExclusive(&m_srwLock);
 
-        // clean up the hash table so that the application will be informed on shutdown
+        // Call shutdown on each application in the application manager
+        m_pApplicationInfoHash->Apply(ShutdownApplication, NULL);
         m_pApplicationInfoHash->Clear();
+        delete m_pApplicationInfoHash;
+        m_pApplicationInfoHash = NULL;
 
         ReleaseSRWLockExclusive(&m_srwLock);
     }
-
+}
-    // stop filewatcher monitoring thread
+
-    if (m_pFileWatcher != NULL)
+//
-    {
+// Calls shutdown on each application. ApplicationManager's lock is held for duration of
-        delete  m_pFileWatcher;
+// each shutdown call, guaranteeing another application cannot be created.
-        m_pFileWatcher = NULL;
+//
-    }
+// static
+VOID
+APPLICATION_MANAGER::ShutdownApplication(
+    _In_ APPLICATION_INFO *     pEntry,
+    _In_ PVOID                  pvContext
+)
+{
+    UNREFERENCED_PARAMETER(pvContext);
+    APPLICATION* pApplication = pEntry->QueryApplication();
+    pApplication->ReferenceApplication();
+
+    // Remove the application from the applicationInfo. 
+    pEntry->ClearAndDereferenceApplication();
+    pApplication->ShutDown();
+    pApplication->DereferenceApplication();
+}
+
+//
+// Function used by DoRecycleApplication thread to do the real shutdown
+//
+// static
+VOID
+APPLICATION_MANAGER::DoRecycleApplication(
+    LPVOID lpParam)
+{
+    APPLICATION* pApplication = static_cast<APPLICATION*>(lpParam);
+
+    // Recycle will call shutdown for out of process
+    pApplication->Recycle();
+
+    // Decrement the ref count as we reference it in RecycleApplication.
+    pApplication->DereferenceApplication();
+}
+
+//
+// Function used to recycle an application
+//
+// static
+VOID
+APPLICATION_MANAGER::RecycleApplication(
+    _In_ APPLICATION_INFO *     pEntry,
+    _In_ APP_HOSTING_MODEL      hostingModel
+)
+{
+
+    APPLICATION* pApplication = pEntry->QueryApplication();
+
+    // Reference the application first
+    pApplication->ReferenceApplication();
+
+    if (hostingModel == APP_HOSTING_MODEL::HOSTING_OUT_PROCESS)
+    {
+        pEntry->ClearApplication();
+    }
+
+    // Reset application pointer to NULL
+    // The destructor of ApplictionInfo will not call ShutDown again
+    HANDLE hThread = CreateThread(
+        NULL,       // default security attributes
+        0,          // default stack size
+        (LPTHREAD_START_ROUTINE)DoRecycleApplication,
+        pApplication,       // thread function arguments
+        0,          // default creation flags
+        NULL);      // receive thread identifier
+
+
+    CloseHandle(hThread);
 }

src/AspNetCore/Src/dllmain.cpp

@@ -13,6 +13,7 @@ HINSTANCE           g_hModule;
 HMODULE             g_hAspnetCoreRH = NULL;
 BOOL                g_fAspnetcoreRHAssemblyLoaded = FALSE;
 BOOL                g_fAspnetcoreRHLoadedError = FALSE;
+BOOL                g_fInShutdown = FALSE;
 DWORD               g_dwAspNetCoreDebugFlags = 0;
 DWORD               g_dwActiveServerProcesses = 0;
 SRWLOCK             g_srwLock;
@@ -47,6 +48,10 @@ BOOL WINAPI DllMain(HMODULE hModule,
         DisableThreadLibraryCalls(hModule);
         break;
     case DLL_PROCESS_DETACH:
+        // IIS can cause dll detatch to occur before we receive global notifications
+        // For example, when we switch the bitness of the worker process, 
+        // this is a bug in IIS. To try to avoid AVs, we will set a global flag
+        g_fInShutdown = TRUE;
         StaticCleanup();
     default:
         break;
@@ -137,6 +142,8 @@ HRESULT
         {
             fDisableANCM = (dwData != 0);
         }
+
+        RegCloseKey(hKey);
     }
 
     if (fDisableANCM)
@@ -202,7 +209,7 @@ HRESULT
 
     hr = pModuleInfo->SetGlobalNotifications(
                               pGlobalModule,
-                              GL_APPLICATION_STOP | // Configuration change trigers IIS application stop
+                              GL_CONFIGURATION_CHANGE | // Configuration change trigers IIS application stop
                               GL_STOP_LISTENING);   // worker process stop or recycle
 
     if (FAILED(hr))

src/AspNetCore/Src/globalmodule.cpp

@@ -17,14 +17,18 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalStopListening(
 {
     UNREFERENCED_PARAMETER(pProvider);
 
-    if (m_pApplicationManager != NULL)
+    if (g_fInShutdown)
     {
-        // we should let application manager to shudown all allication
+        // Avoid receiving two shutudown notifications.
-        // and dereference it as some requests may still reference to application manager
+        return GL_NOTIFICATION_CONTINUE;
-        m_pApplicationManager->ShutDown();
-        m_pApplicationManager = NULL;
     }
 
+    DBG_ASSERT(m_pApplicationManager);
+    // we should let application manager to shutdown all allication
+    // and dereference it as some requests may still reference to application manager
+    m_pApplicationManager->ShutDown();
+    m_pApplicationManager = NULL;
+
     // Return processing to the pipeline.
     return GL_NOTIFICATION_CONTINUE;
 }
@@ -34,14 +38,16 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalStopListening(
 // Recycled the corresponding core app if its configuration changed
 //
 GLOBAL_NOTIFICATION_STATUS
-ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
+ASPNET_CORE_GLOBAL_MODULE::OnGlobalConfigurationChange(
-    _In_ IHttpApplicationStopProvider * pProvider
+    _In_ IGlobalConfigurationChangeProvider * pProvider
 )
 {
+    if (g_fInShutdown)
+    {
+        return GL_NOTIFICATION_CONTINUE;
+    }
     // Retrieve the path that has changed.
-    IHttpApplication* pApplication = pProvider->GetApplication();
+    PCWSTR pwszChangePath = pProvider->GetChangePath();
-
-    PCWSTR pwszChangePath = pApplication->GetAppConfigPath();
 
     // Test for an error.
     if (NULL != pwszChangePath &&
@@ -50,7 +56,7 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
     {
         if (m_pApplicationManager != NULL)
         {
-            m_pApplicationManager->RecycleApplication(pwszChangePath);
+            m_pApplicationManager->RecycleApplicationFromManager(pwszChangePath);
         }
     }
 

src/AspNetCore/Src/precomp.hxx

@@ -146,6 +146,7 @@ HRESULT_FROM_GETLASTERROR()
 extern PVOID        g_pModuleId;
 extern BOOL         g_fAspnetcoreRHAssemblyLoaded;
 extern BOOL         g_fAspnetcoreRHLoadedError;
+extern BOOL         g_fInShutdown;
 extern BOOL         g_fEnableReferenceCountTracing;
 extern DWORD        g_dwActiveServerProcesses;
 extern HINSTANCE    g_hModule;

src/AspNetCore/Src/proxymodule.cxx

@@ -84,7 +84,7 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler(
     APPLICATION* pApplication = NULL;
     STACK_STRU(struFileName, 256);
 
-    hr = ASPNETCORE_CONFIG::GetConfig(g_pHttpServer, g_pModuleId, pHttpContext, &pConfig);
+    hr = ASPNETCORE_CONFIG::GetConfig(g_pHttpServer, g_pModuleId, pHttpContext, g_hEventLog, &pConfig);
     if (FAILED(hr))
     {
         goto Finished;
@@ -97,7 +97,7 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler(
         goto Finished;
     }
 
-    hr = pApplicationManager->GetApplicationInfo(
+    hr = pApplicationManager->GetOrCreateApplicationInfo(
         g_pHttpServer,
         pConfig,
         &m_pApplicationInfo);
@@ -149,7 +149,8 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler(
 
     // make sure application is in running state
     // cannot recreate the application as we cannot reload clr for inprocess
-    if (pApplication->QueryStatus() != APPLICATION_STATUS::RUNNING)
+    if (pApplication->QueryStatus() != APPLICATION_STATUS::RUNNING &&
+        pApplication->QueryStatus() != APPLICATION_STATUS::STARTING)
     {
         hr = HRESULT_FROM_WIN32(ERROR_SERVER_DISABLED);
         goto Finished;
@@ -173,6 +174,7 @@ Finished:
         pHttpContext->GetResponse()->SetStatus(500, "Internal Server Error", 0, hr);
         retVal = RQ_NOTIFICATION_FINISH_REQUEST;
     }
+
     return retVal;
 }
 

src/CommonLib/application.h

@@ -6,8 +6,10 @@
 enum APPLICATION_STATUS
 {
     UNKNOWN = 0,
+    STARTING,
     RUNNING,
-    FAUL
+    SHUTDOWN,
+    FAIL
 };
 
 class  ASPNETCORE_CONFIG;
@@ -23,6 +25,10 @@ public:
     VOID
     ShutDown() = 0;
 
+    virtual
+    VOID
+    Recycle() = 0;
+
     virtual
     ~APPLICATION();
 
@@ -42,7 +48,7 @@ public:
 
 protected:
     mutable LONG            m_cRefs;
-    APPLICATION_STATUS      m_status;
+    volatile APPLICATION_STATUS m_status;
     IHttpServer*            m_pHttpServer;
     ASPNETCORE_CONFIG*      m_pConfig;
 };

src/CommonLib/aspnetcore_msg.mc

@@ -158,6 +158,26 @@ Language=English
 %1
 .
 
+Messageid=1022
+SymbolicName=ASPNETCORE_EVENT_RECYCLE_APP_FAILURE
+Language=English
+%1
+.
+
+Messageid=1023
+SymbolicName=ASPNETCORE_EVENT_APP_IN_SHUTDOWN
+Language=English
+%1
+.
+
+Messageid=1024
+SymbolicName=ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_REMOVED
+Language=English
+%1
+.
+
+
+
 ;
 ;#endif     // _ASPNETCORE_MODULE_MSG_H_
 ;

src/CommonLib/aspnetcoreconfig.cxx

@@ -46,12 +46,16 @@ ASPNETCORE_CONFIG::GetConfig(
     _In_  IHttpServer             *pHttpServer,
     _In_  HTTP_MODULE_ID           pModuleId,
     _In_  IHttpContext            *pHttpContext,
+    _In_  HANDLE                   hEventLog,
     _Out_ ASPNETCORE_CONFIG      **ppAspNetCoreConfig
 )
 {
     HRESULT                 hr = S_OK;
     IHttpApplication       *pHttpApplication = pHttpContext->GetApplication();
     ASPNETCORE_CONFIG      *pAspNetCoreConfig = NULL;
+    STRU                    struHostFxrDllLocation;
+    PWSTR*                  pwzArgv;
+    DWORD                   dwArgCount;
 
     if (ppAspNetCoreConfig == NULL)
     {
@@ -85,6 +89,29 @@ ASPNETCORE_CONFIG::GetConfig(
         goto Finished;
     }
 
+    // Modify config for inprocess.
+    if (pAspNetCoreConfig->QueryHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS)
+    {
+        if (FAILED(hr = HOSTFXR_UTILITY::GetHostFxrParameters(
+            hEventLog,
+            pAspNetCoreConfig->QueryProcessPath()->QueryStr(),
+            pAspNetCoreConfig->QueryApplicationPhysicalPath()->QueryStr(),
+            pAspNetCoreConfig->QueryArguments()->QueryStr(),
+            &struHostFxrDllLocation,
+            &dwArgCount,
+            &pwzArgv)))
+        {
+            goto Finished;
+        }
+
+        if (FAILED(hr = pAspNetCoreConfig->SetHostFxrFullPath(struHostFxrDllLocation.QueryStr())))
+        {
+            goto Finished;
+        }
+
+        pAspNetCoreConfig->SetHostFxrArguments(dwArgCount, pwzArgv);
+    }
+
     hr = pHttpApplication->GetModuleContextContainer()->
         SetModuleContext(pAspNetCoreConfig, pModuleId);
     if (FAILED(hr))
@@ -94,8 +121,8 @@ ASPNETCORE_CONFIG::GetConfig(
             delete pAspNetCoreConfig;
 
             pAspNetCoreConfig = (ASPNETCORE_CONFIG*)pHttpApplication->
-                                 GetModuleContextContainer()->
+                GetModuleContextContainer()->
-                                 GetModuleContext(pModuleId);
+                GetModuleContext(pModuleId);
 
             _ASSERT(pAspNetCoreConfig != NULL);
 
@@ -539,7 +566,7 @@ Finished:
         pWindowsAuthenticationElement = NULL;
     }
 
-    if (pAnonymousAuthenticationElement!= NULL)
+    if (pAnonymousAuthenticationElement != NULL)
     {
         pAnonymousAuthenticationElement->Release();
         pAnonymousAuthenticationElement = NULL;

src/CommonLib/aspnetcoreconfig.h

@@ -69,6 +69,7 @@ public:
         _In_  IHttpServer             *pHttpServer,
         _In_  HTTP_MODULE_ID           pModuleId,
         _In_  IHttpContext            *pHttpContext,
+        _In_  HANDLE                   hEventLog,
         _Out_ ASPNETCORE_CONFIG       **ppAspNetCoreConfig
     );
 

src/CommonLib/resources.h

@@ -28,8 +28,11 @@
 #define ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_STDERR_MSG    L"Application '%s' with physical root '%s' hit unexpected managed background thread exit, ErrorCode = '0x%x. First 4KB characters of captured stderr logs on startup:\r\n%s"
 #define ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_STDOUT_MSG    L"Application '%s' with physical root '%s' hit unexpected managed background thread exit, ErrorCode = '0x%x. Last 4KB characters of captured stdout and stderr logs:\r\n%s"
 #define ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_MSG           L"Application '%s' with physical root '%s' hit unexpected managed background thread exit, ErrorCode = '0x%x. Please check the stderr logs for more information."
-#define ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_MSG              L"Application '%s' is recycled due to app_offline file was detected."
+#define ASPNETCORE_EVENT_APP_IN_SHUTDOWN_MSG                 L"Application shutting down."
-#define ASPNETCORE_EVENT_RECYCLE_CONFIGURATION_MSG           L"Application '%s' is recycled due to configuration changed"
+#define ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_MSG              L"Application '%s' was recycled after detecting the app_offline file."
+#define ASPNETCORE_EVENT_RECYCLE_APPOFFLINE_REMOVED_MSG      L"App_offline.htm has been removed from the application. Application will be recycled."
+#define ASPNETCORE_EVENT_RECYCLE_CONFIGURATION_MSG           L"Application '%s' was recycled due to configuration change"
+#define ASPNETCORE_EVENT_RECYCLE_FAILURE_CONFIGURATION_MSG   L"Failed to recycle application due to a configuration change at '%s'. Recycling worker process."
 #define ASPNETCORE_EVENT_MODULE_DISABLED_MSG                 L"AspNetCore Module is disabled"
 #define ASPNETCORE_EVENT_INPROCESS_FULL_FRAMEWORK_APP_MSG    L"Application '%s' was compiled for .NET Framework. Please compile for .NET core to run the inprocess application or change the process mode to out of process. ErrorCode = '0x%x'."
 #define ASPNETCORE_EVENT_PORTABLE_APP_DOTNET_MISSING_MSG     L"Could not find dotnet.exe on the system PATH environment variable for portable application '%s'. Check that a valid path to dotnet is on the PATH and the bitness of dotnet matches the bitness of the IIS worker process. ErrorCode = '0x%x'."

src/Microsoft.AspNetCore.Server.IISIntegration/NativeMethods.cs

@@ -59,6 +59,12 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
         [DllImport(AspNetCoreModuleDll)]
         internal unsafe static extern HttpApiTypes.HTTP_REQUEST_V2* http_get_raw_request(IntPtr pInProcessHandler);
 
+        [DllImport(AspNetCoreModuleDll)]
+        internal unsafe static extern void http_stop_calls_into_managed();
+
+        [DllImport(AspNetCoreModuleDll)]
+        internal unsafe static extern void http_stop_incoming_requests();
+
         [DllImport(AspNetCoreModuleDll)]
         internal unsafe static extern HttpApiTypes.HTTP_RESPONSE_V2* http_get_raw_response(IntPtr pInProcessHandler);
 

src/Microsoft.AspNetCore.Server.IISIntegration/Server/IISHttpContext.cs

@@ -47,6 +47,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
 
         protected Exception _applicationException;
         private readonly MemoryPool<byte> _memoryPool;
+        private readonly IISHttpServer _server;
 
         private GCHandle _thisHandle;
         private MemoryHandle _inputHandle;
@@ -59,13 +60,14 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
         private const string NegotiateString = "Negotiate";
         private const string BasicString = "Basic";
 
-        internal unsafe IISHttpContext(MemoryPool<byte> memoryPool, IntPtr pInProcessHandler, IISOptions options)
+        internal unsafe IISHttpContext(MemoryPool<byte> memoryPool, IntPtr pInProcessHandler, IISOptions options, IISHttpServer server)
             : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.http_get_raw_request(pInProcessHandler))
         {
             _thisHandle = GCHandle.Alloc(this);
 
             _memoryPool = memoryPool;
             _pInProcessHandler = pInProcessHandler;
+            _server = server;
 
             NativeMethods.http_set_managed_context(pInProcessHandler, (IntPtr)_thisHandle);
             unsafe
@@ -200,6 +202,11 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
             }
         }
         
+        internal IISHttpServer Server
+        {
+            get { return _server; }
+        }
+
         private async Task InitializeResponseAwaited()
         {
             await FireOnStarting();

src/Microsoft.AspNetCore.Server.IISIntegration/Server/IISHttpContextOfT.cs

@@ -14,8 +14,8 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
     {
         private readonly IHttpApplication<TContext> _application;
 
-        public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISOptions options)
+        public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISOptions options, IISHttpServer server)
-            : base(memoryPool, pInProcessHandler, options)
+            : base(memoryPool, pInProcessHandler, options, server)
         {
             _application = application;
         }

src/Microsoft.AspNetCore.Server.IISIntegration/Server/IISHttpServer.cs

@@ -28,6 +28,11 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
         private readonly IAuthenticationSchemeProvider _authentication;
         private readonly IISOptions _options;
 
+        private volatile int _stopping;
+        private bool Stopping => _stopping == 1;
+        private int _outstandingRequests;
+        private readonly TaskCompletionSource<object> _shutdownSignal = new TaskCompletionSource<object>(TaskCreationOptions.RunContinuationsAsynchronously);
+
         public IFeatureCollection Features { get; } = new FeatureCollection();
         public IISHttpServer(IApplicationLifetime applicationLifetime, IAuthenticationSchemeProvider authentication, IOptions<IISOptions> options)
         {
@@ -45,7 +50,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
         {
             _httpServerHandle = GCHandle.Alloc(this);
 
-            _iisContextFactory = new IISContextFactory<TContext>(_memoryPool, application, _options);
+            _iisContextFactory = new IISContextFactory<TContext>(_memoryPool, application, _options, this);
 
             // Start the server by registering the callback
             NativeMethods.register_callbacks(_requestHandler, _shutdownHandler, _onAsyncCompletion, (IntPtr)_httpServerHandle, (IntPtr)_httpServerHandle);
@@ -55,15 +60,54 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
 
         public Task StopAsync(CancellationToken cancellationToken)
         {
-            // TODO: Drain pending requests
+            void RegisterCancelation()
+            {
+                cancellationToken.Register(() =>
+                {
+                    NativeMethods.http_stop_calls_into_managed();
+                    _shutdownSignal.TrySetResult(null);
+                });
+            }
+            if (Interlocked.Exchange(ref _stopping, 1) == 1)
+            {
+                RegisterCancelation();
 
-            // Stop all further calls back into managed code by unhooking the callback
+                return _shutdownSignal.Task;
+            }
 
-            return Task.CompletedTask;
+            // First call back into native saying "DON'T SEND ME ANY MORE REQUESTS"
+            NativeMethods.http_stop_incoming_requests();
+
+            try
+            {
+                // Wait for active requests to drain
+                if (_outstandingRequests > 0)
+                {
+                    RegisterCancelation();
+                }
+                else
+                {
+                    // We have drained all requests. Block any callbacks into managed at this point.
+                    NativeMethods.http_stop_calls_into_managed();
+                    _shutdownSignal.TrySetResult(null);
+                }
+            }
+            catch (Exception ex)
+            {
+                _shutdownSignal.TrySetException(ex);
+            }
+
+            return _shutdownSignal.Task;
         }
 
         public void Dispose()
         {
+            _stopping = 1;
+
+            // Block any more calls into managed from native as we are unloading.
+            NativeMethods.http_stop_calls_into_managed();
+            _shutdownSignal.TrySetResult(null);
+
             if (_httpServerHandle.IsAllocated)
             {
                 _httpServerHandle.Free();
@@ -76,6 +120,7 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
         {
             // Unwrap the server so we can create an http context and process the request
             var server = (IISHttpServer)GCHandle.FromIntPtr(pvRequestContext).Target;
+            Interlocked.Increment(ref server._outstandingRequests);
 
             var context = server._iisContextFactory.CreateHttpContext(pInProcessHandler);
 
@@ -105,6 +150,13 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
             // Post completion after completing the request to resume the state machine
             context.PostCompletion(ConvertRequestCompletionResults(completedTask.Result));
 
+            if (Interlocked.Decrement(ref context.Server._outstandingRequests) == 0 && context.Server.Stopping)
+            {
+                // All requests have been drained.
+                NativeMethods.http_stop_calls_into_managed();
+                context.Server._shutdownSignal.TrySetResult(null);
+            }
+
             // Dispose the context
             context.Dispose();
         }
@@ -120,19 +172,27 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
             private readonly IHttpApplication<T> _application;
             private readonly MemoryPool<byte> _memoryPool;
             private readonly IISOptions _options;
+            private readonly IISHttpServer _server;
 
-            public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> application, IISOptions options)
+            public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> application, IISOptions options, IISHttpServer server)
             {
                 _application = application;
                 _memoryPool = memoryPool;
                 _options = options;
+                _server = server;
             }
 
             public IISHttpContext CreateHttpContext(IntPtr pInProcessHandler)
             {
-                return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options);
+                return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server);
             }
         }
+
+        ~IISHttpServer()
+        {
+            // If this finalize is invoked, try our best to block all calls into managed.
+            NativeMethods.http_stop_calls_into_managed();
+        }
     }
 
     // Over engineering to avoid allocations...

src/Microsoft.AspNetCore.Server.IISIntegration/Server/OutputProducer.cs

@@ -110,7 +110,6 @@ namespace Microsoft.AspNetCore.Server.IISIntegration
                 if (_flushTcs == null || _flushTcs.Task.IsCompleted)
                 {
                     _flushTcs = new TaskCompletionSource<object>();
-
                     awaitable.OnCompleted(_flushCompleted);
                 }
             }

src/RequestHandler/inprocess/inprocessapplication.cpp

@@ -7,15 +7,15 @@ IN_PROCESS_APPLICATION::IN_PROCESS_APPLICATION(
     ASPNETCORE_CONFIG*  pConfig) :
     APPLICATION(pHttpServer, pConfig),
     m_ProcessExitCode(0),
-    m_fManagedAppLoaded(FALSE),
-    m_fLoadManagedAppError(FALSE),
-    m_fInitialized(FALSE),
-    m_fRecycleProcessCalled(FALSE),
     m_hLogFileHandle(INVALID_HANDLE_VALUE),
     m_hErrReadPipe(INVALID_HANDLE_VALUE),
     m_hErrWritePipe(INVALID_HANDLE_VALUE),
+    m_dwStdErrReadTotal(0),
     m_fDoneStdRedirect(FALSE),
-    m_dwStdErrReadTotal(0)
+    m_fBlockCallbacksIntoManaged(FALSE),
+    m_fInitialized(FALSE),
+    m_fShutdownCalledFromNative(FALSE),
+    m_fShutdownCalledFromManaged(FALSE)
 {
     // is it guaranteed that we have already checked app offline at this point?
     // If so, I don't think there is much to do here.
@@ -24,111 +24,158 @@ IN_PROCESS_APPLICATION::IN_PROCESS_APPLICATION(
     InitializeSRWLock(&m_srwLock);
 
     // TODO we can probably initialized as I believe we are the only ones calling recycle.
-    m_fInitialized = TRUE;
+    m_status = APPLICATION_STATUS::STARTING;
-    m_status = APPLICATION_STATUS::RUNNING;
 }
 
 IN_PROCESS_APPLICATION::~IN_PROCESS_APPLICATION()
 {
-    Recycle();
+    // TODO check if anything else needs to be cleaned up
 }
 
 __override
 VOID
 IN_PROCESS_APPLICATION::ShutDown()
 {
-    //todo
+    DWORD    dwThreadStatus = 0;
-}
+    DWORD    dwTimeout = m_pConfig->QueryShutdownTimeLimitInMS();
+    HANDLE   handle = NULL;
+    WIN32_FIND_DATA fileData;
+    BOOL     fLocked = FALSE;
 
-// This is the same function as before, TODO configrm if we need to change anything for configuration.
+    if (IsDebuggerPresent())
-VOID
-IN_PROCESS_APPLICATION::Recycle(
-    VOID
-)
-{
-    if (m_fInitialized)
     {
-        DWORD    dwThreadStatus = 0;
+        dwTimeout = INFINITE;
-        DWORD    dwTimeout = m_pConfig->QueryShutdownTimeLimitInMS();
+    }
-        HANDLE   handle = NULL;
+    else
-        WIN32_FIND_DATA fileData;
+    {
+        dwTimeout = m_pConfig->QueryShutdownTimeLimitInMS();
+    }
 
-        AcquireSRWLockExclusive(&m_srwLock);
+    if (m_fShutdownCalledFromNative ||
+        m_status == APPLICATION_STATUS::STARTING ||
+        m_status == APPLICATION_STATUS::FAIL)
+    {
+        goto Finished;
+    }
 
-        if (!m_pHttpServer->IsCommandLineLaunch() &&
+    AcquireSRWLockExclusive(&m_srwLock);
-            !m_fRecycleProcessCalled &&
+    fLocked = TRUE;
-            (m_pHttpServer->GetAdminManager() != NULL))
-        {
-            // IIS scenario.
-            // notify IIS first so that new request will be routed to new worker process
-            m_pHttpServer->RecycleProcess(L"AspNetCore Recycle Process on Demand");
-        }
 
-        m_fRecycleProcessCalled = TRUE;
+    if (m_fShutdownCalledFromNative ||
+        m_status == APPLICATION_STATUS::STARTING ||
+        m_status == APPLICATION_STATUS::FAIL)
+    {
+        goto Finished;
+    }
 
-        // First call into the managed server and shutdown
+    // We need to keep track of when both managed and native initiate shutdown
-        if (m_ShutdownHandler != NULL)
+    // to avoid AVs. If shutdown has already been initiated in managed, we don't want to call into
-        {
+    // managed. We still need to wait on main exiting no matter what. m_fShutdownCalledFromNative
-            m_ShutdownHandler(m_ShutdownHandlerContext);
+    // is used for detecting redundant calls and blocking more requests to OnExecuteRequestHandler.
-            m_ShutdownHandler = NULL;
+    m_fShutdownCalledFromNative = TRUE;
-        }
+    m_status = APPLICATION_STATUS::SHUTDOWN;
 
+    if (!m_fShutdownCalledFromManaged)
+    {
+        // Initiate a recycle such that another worker process is created to replace this one.
+
+        m_ShutdownHandler(m_ShutdownHandlerContext);
+        m_ShutdownHandler = NULL;
+
+        ReleaseSRWLockExclusive(&m_srwLock);
+        fLocked = FALSE;
+
+        // Release the lock before we wait on the thread to exit. 
         if (m_hThread != NULL &&
             GetExitCodeThread(m_hThread, &dwThreadStatus) != 0 &&
             dwThreadStatus == STILL_ACTIVE)
         {
-            // wait for gracefullshut down, i.e., the exit of the background thread or timeout
+            // wait for graceful shutdown, i.e., the exit of the background thread or timeout
             if (WaitForSingleObject(m_hThread, dwTimeout) != WAIT_OBJECT_0)
             {
                 // if the thread is still running, we need kill it first before exit to avoid AV
                 if (GetExitCodeThread(m_hThread, &dwThreadStatus) != 0 && dwThreadStatus == STILL_ACTIVE)
                 {
+                    // Calling back into managed at this point is prone to have AVs
+                    // Calling terminate thread here may be our best solution.
                     TerminateThread(m_hThread, STATUS_CONTROL_C_EXIT);
                 }
             }
         }
-
-        CloseHandle(m_hThread);
-        m_hThread = NULL;
-        s_Application = NULL;
-
-        ReleaseSRWLockExclusive(&m_srwLock);
-
-        CloseStdErrHandles();
-
-        if (m_pStdFile != NULL)
-        {
-            fflush(stdout);
-            fflush(stderr);
-            fclose(m_pStdFile);
-        }
-
-        if (m_hLogFileHandle != INVALID_HANDLE_VALUE)
-        {
-            m_Timer.CancelTimer();
-            CloseHandle(m_hLogFileHandle);
-            m_hLogFileHandle = INVALID_HANDLE_VALUE;
-        }
-
-        // delete empty log file
-        handle = FindFirstFile(m_struLogFilePath.QueryStr(), &fileData);
-        if (handle != INVALID_HANDLE_VALUE &&
-            fileData.nFileSizeHigh == 0 &&
-            fileData.nFileSizeLow == 0) // skip check of nFileSizeHigh
-        {
-            FindClose(handle);
-            // no need to check whether the deletion succeeds
-            // as nothing can be done
-            DeleteFile(m_struLogFilePath.QueryStr());
-        }
-
-        if (m_pHttpServer && m_pHttpServer->IsCommandLineLaunch())
-        {
-            // IISExpress scenario
-            // Can only call exit to terminate current process
-            exit(0);
-        }
     }
+
+    CloseHandle(m_hThread);
+    m_hThread = NULL;
+    s_Application = NULL;
+
+    CloseStdErrHandles();
+
+    if (m_pStdFile != NULL)
+    {
+        fflush(stdout);
+        fflush(stderr);
+        fclose(m_pStdFile);
+    }
+
+    if (m_hLogFileHandle != INVALID_HANDLE_VALUE)
+    {
+        m_Timer.CancelTimer();
+        CloseHandle(m_hLogFileHandle);
+        m_hLogFileHandle = INVALID_HANDLE_VALUE;
+    }
+
+    // delete empty log file
+    handle = FindFirstFile(m_struLogFilePath.QueryStr(), &fileData);
+    if (handle != INVALID_HANDLE_VALUE &&
+        fileData.nFileSizeHigh == 0 &&  
+        fileData.nFileSizeLow == 0) // skip check of nFileSizeHigh
+    {
+        FindClose(handle);
+        // no need to check whether the deletion succeeds
+        // as nothing can be done
+        DeleteFile(m_struLogFilePath.QueryStr());
+    }
+
+Finished:
+
+    if (fLocked)
+    {
+        ReleaseSRWLockExclusive(&m_srwLock);
+    }
+    return;
+}
+
+VOID
+IN_PROCESS_APPLICATION::Recycle(
+    VOID
+)
+{
+    // We need to guarantee that recycle is only called once, as calling pHttpServer->RecycleProcess
+    // multiple times can lead to AVs. 
+    if (m_fRecycleCalled)
+    {
+        return;
+    }
+    AcquireSRWLockExclusive(&m_srwLock);
+    if (m_fRecycleCalled)
+    {
+        goto Finished;
+    }
+    if (!m_pHttpServer->IsCommandLineLaunch())
+    {
+        // IIS scenario.
+        // notify IIS first so that new request will be routed to new worker process
+        m_pHttpServer->RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
+    }
+    else
+    {
+        // IISExpress scenario
+        // Can only call exit to terminate current process
+        ShutDown();
+        exit(0);
+    }
+Finished:
+    ReleaseSRWLockExclusive(&m_srwLock);
 }
 
 REQUEST_NOTIFICATION_STATUS
@@ -140,18 +187,33 @@ IN_PROCESS_APPLICATION::OnAsyncCompletion(
 {
     REQUEST_NOTIFICATION_STATUS dwRequestNotificationStatus = RQ_NOTIFICATION_CONTINUE;
 
+    ReferenceApplication();
+
     if (pInProcessHandler->QueryIsManagedRequestComplete())
     {
         // means PostCompletion has been called and this is the associated callback.
         dwRequestNotificationStatus = pInProcessHandler->QueryAsyncCompletionStatus();
-        // TODO cleanup whatever disconnect listener there is
+    }
-        return dwRequestNotificationStatus;
+    else if (m_fBlockCallbacksIntoManaged)
+    {
+        // this can potentially happen in ungraceful shutdown.
+        // Or something really wrong happening with async completions
+        // At this point, managed is in a shutting down state and we cannot send a request to it.
+        pInProcessHandler->QueryHttpContext()->GetResponse()->SetStatus(503,
+            "Server has been shutdown",
+            0,
+            (ULONG)HRESULT_FROM_WIN32(ERROR_SHUTDOWN_IN_PROGRESS));
+        dwRequestNotificationStatus = RQ_NOTIFICATION_FINISH_REQUEST;
     }
     else
     {
         // Call the managed handler for async completion.
-        return m_AsyncCompletionHandler(pInProcessHandler->QueryManagedHttpContext(), hrCompletionStatus, cbCompletion);
+        dwRequestNotificationStatus = m_AsyncCompletionHandler(pInProcessHandler->QueryManagedHttpContext(), hrCompletionStatus, cbCompletion);
     }
+
+    DereferenceApplication();
+
+    return dwRequestNotificationStatus;
 }
 
 REQUEST_NOTIFICATION_STATUS
@@ -160,27 +222,47 @@ IN_PROCESS_APPLICATION::OnExecuteRequest(
     _In_ IN_PROCESS_HANDLER* pInProcessHandler
 )
 {
-    if (m_RequestHandler != NULL)
+    REQUEST_NOTIFICATION_STATUS dwRequestNotificationStatus = RQ_NOTIFICATION_CONTINUE;
-    {
+    PFN_REQUEST_HANDLER pRequestHandler = NULL;
-        return m_RequestHandler(pInProcessHandler, m_RequestHandlerContext);
-    }
 
-    //
+    ReferenceApplication();
-    // return error as the application did not register callback
+    pRequestHandler = m_RequestHandler;
-    //
+
-    if (ANCMEvents::ANCM_EXECUTE_REQUEST_FAIL::IsEnabled(pHttpContext->GetTraceContext()))
+    if (pRequestHandler == NULL)
     {
-        ANCMEvents::ANCM_EXECUTE_REQUEST_FAIL::RaiseEvent(pHttpContext->GetTraceContext(),
+        //
-            NULL,
+        // return error as the application did not register callback
+        //
+        if (ANCMEvents::ANCM_EXECUTE_REQUEST_FAIL::IsEnabled(pHttpContext->GetTraceContext()))
+        {
+            ANCMEvents::ANCM_EXECUTE_REQUEST_FAIL::RaiseEvent(pHttpContext->GetTraceContext(),
+                NULL,
+                (ULONG)E_APPLICATION_ACTIVATION_EXEC_FAILURE);
+        }
+
+        pHttpContext->GetResponse()->SetStatus(500,
+            "Internal Server Error",
+            0,
             (ULONG)E_APPLICATION_ACTIVATION_EXEC_FAILURE);
+
+        dwRequestNotificationStatus = RQ_NOTIFICATION_FINISH_REQUEST;
+    }
+    else if (m_status != APPLICATION_STATUS::RUNNING || m_fBlockCallbacksIntoManaged)
+    {
+        pHttpContext->GetResponse()->SetStatus(503,
+            "Server is currently shutting down.",
+            0,
+            (ULONG)HRESULT_FROM_WIN32(ERROR_SHUTDOWN_IN_PROGRESS));
+        dwRequestNotificationStatus = RQ_NOTIFICATION_FINISH_REQUEST;
+    }
+    else
+    {
+        dwRequestNotificationStatus = pRequestHandler(pInProcessHandler, m_RequestHandlerContext);
     }
 
-    pHttpContext->GetResponse()->SetStatus(500,
+    DereferenceApplication();
-        "Internal Server Error",
-        0,
-        (ULONG)E_APPLICATION_ACTIVATION_EXEC_FAILURE);
 
-    return RQ_NOTIFICATION_FINISH_REQUEST;
+    return dwRequestNotificationStatus;
 }
 
 VOID
@@ -203,7 +285,7 @@ IN_PROCESS_APPLICATION::SetCallbackHandles(
     SetStdHandle(STD_ERROR_HANDLE, INVALID_HANDLE_VALUE);
     // Initialization complete
     SetEvent(m_pInitalizeEvent);
-
+    m_fInitialized = TRUE;
 }
 
 VOID
@@ -478,14 +560,20 @@ IN_PROCESS_APPLICATION::LoadManagedApplication
     DWORD      dwResult;
     BOOL       fLocked = FALSE;
 
-    if (m_fManagedAppLoaded || m_fLoadManagedAppError)
+    ReferenceApplication();
+
+    if (m_status != APPLICATION_STATUS::STARTING)
     {
         // Core CLR has already been loaded.
         // Cannot load more than once even there was a failure
-        if (m_fLoadManagedAppError)
+        if (m_status == APPLICATION_STATUS::FAIL)
         {
             hr = E_APPLICATION_ACTIVATION_EXEC_FAILURE;
         }
+        else if (m_status == APPLICATION_STATUS::SHUTDOWN)
+        {
+            hr = HRESULT_FROM_WIN32(ERROR_SHUTDOWN_IS_SCHEDULED);
+        }
 
         goto Finished;
     }
@@ -495,12 +583,16 @@ IN_PROCESS_APPLICATION::LoadManagedApplication
 
     AcquireSRWLockExclusive(&m_srwLock);
     fLocked = TRUE;
-    if (m_fManagedAppLoaded || m_fLoadManagedAppError)
+    if (m_status != APPLICATION_STATUS::STARTING)
     {
-        if (m_fLoadManagedAppError)
+        if (m_status == APPLICATION_STATUS::FAIL)
         {
             hr = E_APPLICATION_ACTIVATION_EXEC_FAILURE;
         }
+        else if (m_status == APPLICATION_STATUS::SHUTDOWN)
+        {
+            hr = HRESULT_FROM_WIN32(ERROR_SHUTDOWN_IS_SCHEDULED);
+        }
 
         goto Finished;
     }
@@ -566,16 +658,14 @@ IN_PROCESS_APPLICATION::LoadManagedApplication
         goto Finished;
     }
 
-    m_fManagedAppLoaded = TRUE;
+    m_status = APPLICATION_STATUS::RUNNING;
 
 Finished:
 
     if (FAILED(hr))
     {
         STACK_STRU(strEventMsg, 256);
-        // Question: in case of application loading failure, should we allow retry on 
+        m_status = APPLICATION_STATUS::FAIL;
-        // following request or block the activation at all
-        m_fLoadManagedAppError = TRUE; // m_hThread != NULL ?
 
         if (SUCCEEDED(strEventMsg.SafeSnwprintf(
             ASPNETCORE_EVENT_LOAD_CLR_FALIURE_MSG,
@@ -595,6 +685,8 @@ Finished:
         ReleaseSRWLockExclusive(&m_srwLock);
     }
 
+    DereferenceApplication();
+
     return hr;
 }
 
@@ -683,90 +775,100 @@ IN_PROCESS_APPLICATION::ExecuteApplication(
 
 Finished:
 
-    if (!m_fRecycleProcessCalled)
+    //
+    // this method is called by the background thread and should never exit unless shutdown
+    // If main returned and shutdown was not called in managed, we want to block native from calling into
+    // managed. To do this, we can say that shutdown was called from managed.
+    // Don't bother locking here as there will always be a race between receiving a native shutdown
+    // notification and unexpected managed exit.
+    //
+    m_status = APPLICATION_STATUS::SHUTDOWN;
+    m_fShutdownCalledFromManaged = TRUE;
+    FreeLibrary(hModule);
+
+    if (!m_fShutdownCalledFromNative)
     {
-        //
+        LogErrorsOnMainExit(hr);
-        // Ungraceful shutdown, try to log an error message.
+        if (m_fInitialized)
-        // This will be a common place for errors as it means the hostfxr_main returned
-        // or there was an exception.
-        //
-
-        CHAR            pzFileContents[4096] = { 0 };
-        DWORD           dwNumBytesRead;
-        STRU            struStdErrLog;
-        LARGE_INTEGER   li = { 0 };
-        STRU            strEventMsg;
-        BOOL            fLogged = FALSE;
-        DWORD           dwFilePointer = 0;
-
-        if (m_pConfig->QueryStdoutLogEnabled())
         {
-            // Put stdout/stderr logs into 
+            //
-            if (m_hLogFileHandle != INVALID_HANDLE_VALUE)
+            // If the inprocess server was initialized, we need to cause recycle to be called on the worker process.
-            {
+            //
-                if (GetFileSizeEx(m_hLogFileHandle, &li) && li.LowPart > 0 && li.HighPart == 0)
+            Recycle();
-                {
+        }
-                    if (li.LowPart > 4096)
+    }
-                    {
+
-                        dwFilePointer = SetFilePointer(m_hLogFileHandle, -4096, NULL, FILE_END);
+    return hr;
-                    }
+}
-                    else
+
-                    {
+VOID
-                        dwFilePointer = SetFilePointer(m_hLogFileHandle, 0, NULL, FILE_BEGIN);
+IN_PROCESS_APPLICATION::LogErrorsOnMainExit(
-                    }
+    HRESULT hr
-                    if (dwFilePointer != INVALID_SET_FILE_POINTER)
+)
-                    {
+{
-                        if (ReadFile(m_hLogFileHandle, pzFileContents, 4096, &dwNumBytesRead, NULL))
+    //
-                        {
+    // Ungraceful shutdown, try to log an error message.
-                            if (SUCCEEDED(struStdErrLog.CopyA(m_pzFileContents, m_dwStdErrReadTotal)) &&
+    // This will be a common place for errors as it means the hostfxr_main returned
-                                SUCCEEDED(strEventMsg.SafeSnwprintf(
+    // or there was an exception.
-                                    ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_STDOUT_MSG,
+    //
-                                    m_pConfig->QueryApplicationPath()->QueryStr(),
+    CHAR            pzFileContents[4096] = { 0 };
-                                    m_pConfig->QueryApplicationPhysicalPath()->QueryStr(),
+    DWORD           dwNumBytesRead;
-                                    hr,
+    STRU            struStdErrLog;
-                                    struStdErrLog.QueryStr())))
+    LARGE_INTEGER   li = { 0 };
-                            {
+    STRU            strEventMsg;
-                                UTILITY::LogEvent(g_hEventLog,
+    BOOL            fLogged = FALSE;
-                                    EVENTLOG_ERROR_TYPE,
+    DWORD           dwFilePointer = 0;
-                                    ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT,
+
-                                    strEventMsg.QueryStr());
+    if (m_pConfig->QueryStdoutLogEnabled())
-                                fLogged = TRUE;
+    {
+        // Put stdout/stderr logs into 
+        if (m_hLogFileHandle != INVALID_HANDLE_VALUE)
+        {
+            if (GetFileSizeEx(m_hLogFileHandle, &li) && li.LowPart > 0 && li.HighPart == 0)
+            {
+                if (li.LowPart > 4096)
+                {
+                    dwFilePointer = SetFilePointer(m_hLogFileHandle, -4096, NULL, FILE_END);
+                }
+                else
+                {
+                    dwFilePointer = SetFilePointer(m_hLogFileHandle, 0, NULL, FILE_BEGIN);
+                }
+                if (dwFilePointer != INVALID_SET_FILE_POINTER)
+                {
+                    if (ReadFile(m_hLogFileHandle, pzFileContents, 4096, &dwNumBytesRead, NULL))
+                    {
+                        if (SUCCEEDED(struStdErrLog.CopyA(m_pzFileContents, m_dwStdErrReadTotal)) &&
+                            SUCCEEDED(strEventMsg.SafeSnwprintf(
+                                ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_STDOUT_MSG,
+                                m_pConfig->QueryApplicationPath()->QueryStr(),
+                                m_pConfig->QueryApplicationPhysicalPath()->QueryStr(),
+                                hr,
+                                struStdErrLog.QueryStr())))
+                        {
+                            UTILITY::LogEvent(g_hEventLog,
+                                EVENTLOG_ERROR_TYPE,
+                                ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT,
+                                strEventMsg.QueryStr());
+                            fLogged = TRUE;
 
-                            }
                         }
                     }
                 }
             }
         }
-        else
+    }
+    else
+    {
+        if (m_dwStdErrReadTotal > 0)
         {
-            if (m_dwStdErrReadTotal > 0)
+            if (SUCCEEDED(struStdErrLog.CopyA(m_pzFileContents, m_dwStdErrReadTotal)) &&
-            {
+                SUCCEEDED(strEventMsg.SafeSnwprintf(
-                if (SUCCEEDED(struStdErrLog.CopyA(m_pzFileContents, m_dwStdErrReadTotal)) &&
+                    ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_STDERR_MSG,
-                    SUCCEEDED(strEventMsg.SafeSnwprintf(
+                    m_pConfig->QueryApplicationPath()->QueryStr(),
-                        ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_STDERR_MSG,
+                    m_pConfig->QueryApplicationPhysicalPath()->QueryStr(),
-                        m_pConfig->QueryApplicationPath()->QueryStr(),
+                    hr,
-                        m_pConfig->QueryApplicationPhysicalPath()->QueryStr(),
+                    struStdErrLog.QueryStr())))
-                        hr,
-                        struStdErrLog.QueryStr())))
-                {
-                    UTILITY::LogEvent(g_hEventLog,
-                        EVENTLOG_ERROR_TYPE,
-                        ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT,
-                        strEventMsg.QueryStr());
-                    fLogged = TRUE;
-                }
-            }
-        }
-
-        if (!fLogged)
-        {
-            // If we didn't log, log the generic message.
-            if (SUCCEEDED(strEventMsg.SafeSnwprintf(
-                ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_MSG,
-                m_pConfig->QueryApplicationPath()->QueryStr(),
-                m_pConfig->QueryApplicationPhysicalPath()->QueryStr(),
-                hr)))
             {
                 UTILITY::LogEvent(g_hEventLog,
                     EVENTLOG_ERROR_TYPE,
@@ -775,11 +877,24 @@ Finished:
                 fLogged = TRUE;
             }
         }
-
-        // The destructor of inprocessapplication will call recycle. No need to recycle here.
     }
 
-    return hr;
+    if (!fLogged)
+    {
+        // If we didn't log, log the generic message.
+        if (SUCCEEDED(strEventMsg.SafeSnwprintf(
+            ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT_MSG,
+            m_pConfig->QueryApplicationPath()->QueryStr(),
+            m_pConfig->QueryApplicationPhysicalPath()->QueryStr(),
+            hr)))
+        {
+            UTILITY::LogEvent(g_hEventLog,
+                EVENTLOG_ERROR_TYPE,
+                ASPNETCORE_EVENT_INPROCESS_THREAD_EXIT,
+                strEventMsg.QueryStr());
+            fLogged = TRUE;
+        }
+    }
 }
 
 //

src/RequestHandler/inprocess/inprocessapplication.h

@@ -19,18 +19,19 @@ public:
         ShutDown();
 
     VOID
-        SetCallbackHandles(
+    SetCallbackHandles(
-            _In_ PFN_REQUEST_HANDLER request_callback,
+        _In_ PFN_REQUEST_HANDLER request_callback,
-            _In_ PFN_SHUTDOWN_HANDLER shutdown_callback,
+        _In_ PFN_SHUTDOWN_HANDLER shutdown_callback,
-            _In_ PFN_MANAGED_CONTEXT_HANDLER managed_context_callback,
+        _In_ PFN_MANAGED_CONTEXT_HANDLER managed_context_callback,
-            _In_ VOID* pvRequstHandlerContext,
+        _In_ VOID* pvRequstHandlerContext,
-            _In_ VOID* pvShutdownHandlerContext
+        _In_ VOID* pvShutdownHandlerContext
-        );
+    );
 
+    __override
     VOID
-        Recycle(
+    Recycle(
-            VOID
+        VOID
-        );
+    );
 
     // Executes the .NET Core process
     HRESULT
@@ -53,6 +54,11 @@ public:
             VOID
         );
 
+    VOID
+        LogErrorsOnMainExit(
+            HRESULT hr
+        );
+
     REQUEST_NOTIFICATION_STATUS
         OnAsyncCompletion(
             DWORD                   cbCompletion,
@@ -67,6 +73,22 @@ public:
             IN_PROCESS_HANDLER* pInProcessHandler
         );
 
+    VOID
+    StopCallsIntoManaged(
+        VOID
+    )
+    {
+        m_fBlockCallbacksIntoManaged = TRUE;
+    }
+
+    VOID
+    StopIncomingRequests(
+        VOID
+    )
+    {
+        m_fShutdownCalledFromManaged = TRUE;
+    }
+
     static
         IN_PROCESS_APPLICATION*
         GetInstance(
@@ -102,12 +124,13 @@ private:
     // The exit code of the .NET Core process
     INT                             m_ProcessExitCode;
 
-    BOOL                            m_fManagedAppLoaded;
-    BOOL                            m_fLoadManagedAppError;
-    BOOL                            m_fInitialized;
     BOOL                            m_fIsWebSocketsConnection;
     BOOL                            m_fDoneStdRedirect;
-    BOOL                            m_fRecycleProcessCalled;
+    volatile BOOL                   m_fBlockCallbacksIntoManaged;
+    volatile BOOL                   m_fShutdownCalledFromNative;
+    volatile BOOL                   m_fShutdownCalledFromManaged;
+    BOOL                            m_fRecycleCalled;
+    BOOL                            m_fInitialized;
 
     FILE*                           m_pStdFile;
     STTIMER                         m_Timer;
@@ -138,9 +161,9 @@ private:
         );
 
     HRESULT
-    SetEnvironementVariablesOnWorkerProcess(
+        SetEnvironementVariablesOnWorkerProcess(
-        VOID
+            VOID
-    );
+        );
 
     static
         INT

src/RequestHandler/managedexports.cxx

@@ -423,4 +423,18 @@ http_get_authentication_information(
     return S_OK;
 }
 
+EXTERN_C __MIDL_DECLSPEC_DLLEXPORT
+VOID
+http_stop_calls_into_mananged()
+{
+    IN_PROCESS_APPLICATION::GetInstance()->StopCallsIntoManaged();
+}
+
+EXTERN_C __MIDL_DECLSPEC_DLLEXPORT
+VOID
+http_stop_incoming_requests()
+{
+    IN_PROCESS_APPLICATION::GetInstance()->StopIncomingRequests();
+}
+
 // End of export

src/RequestHandler/outofprocess/outprocessapplication.cpp

@@ -7,7 +7,7 @@ OUT_OF_PROCESS_APPLICATION::OUT_OF_PROCESS_APPLICATION(
 {
     m_status = APPLICATION_STATUS::RUNNING;
     m_pProcessManager = NULL;
-    //todo
+    InitializeSRWLock(&rwlock);
 }
 
 OUT_OF_PROCESS_APPLICATION::~OUT_OF_PROCESS_APPLICATION()
@@ -57,10 +57,22 @@ __override
 VOID
 OUT_OF_PROCESS_APPLICATION::ShutDown()
 {
-    if (m_pProcessManager != NULL)
+    AcquireSRWLockExclusive(&rwlock);
     {
-        m_pProcessManager->ShutdownAllProcesses();
+        if (m_pProcessManager != NULL)
-        m_pProcessManager->DereferenceProcessManager();
+        {
-        m_pProcessManager = NULL;
+            m_pProcessManager->ShutdownAllProcesses();
+            m_pProcessManager->DereferenceProcessManager();
+            m_pProcessManager = NULL;
+        }
     }
+    ReleaseSRWLockExclusive(&rwlock);
 }
+
+__override
+VOID
+OUT_OF_PROCESS_APPLICATION::Recycle()
+{
+    ShutDown();
+}
+

src/RequestHandler/outofprocess/outprocessapplication.h

@@ -20,6 +20,11 @@ public:
     VOID
     ShutDown();
 
+    __override
+    VOID
+    Recycle();
+
 private:
     PROCESS_MANAGER * m_pProcessManager;
+    SRWLOCK           rwlock;
 };