Enable appverifier and dumps on jenkins (#1491)

.vsts-pipelines/templates/build-steps.yml

@@ -16,13 +16,6 @@ phases:
           artifactName: logs
           artifactType: Container
           pathtoPublish: artifacts/logs
-      - task: PublishBuildArtifacts@1
-        displayName: Upload dumps
-        condition: eq(variables['system.pullrequest.isfork'], false)
-        inputs:
-          artifactName: dumps
-          artifactType: Container
-          pathtoPublish: artifacts/dumps
 
 - template: .vsts-pipelines/templates/phases/default-build.yml@buildtools
   parameters:

build/buildpipeline/pipeline.groovy

@@ -1,6 +1,7 @@
 import org.dotnet.ci.pipelines.Pipeline
 
 def windowsPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows.groovy')
+def windowsAppverifPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows-appverif.groovy')
 
 def configurations = [
     'Debug',
@@ -16,4 +17,5 @@ configurations.each { configuration ->
     windowsPipeline.triggerPipelineOnEveryGithubPR("Windows ${configuration} x64 Build", params)
     windowsPipeline.triggerPipelineOnGithubPush(params)
 
+    windowsAppverifPipeline.triggerPipelineOnEveryGithubPR("Windows AppVerifier ${configuration} x64 Build", params)
 }

build/buildpipeline/windows-appverif.groovy

@@ -0,0 +1,15 @@
+@Library('dotnet-ci') _
+
+// 'node' indicates to Jenkins that the enclosed block runs on a node that matches
+// the label 'windows-with-vs'
+simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') {
+    stage ('Checking out source') {
+        checkout scm
+        bat 'git submodule update --init --recursive'
+    }
+    stage ('Build') {
+        def logFolder = getLogFolder()
+        def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'"
+        bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 Setup;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";"
+    }
+}

build/buildpipeline/windows.groovy

@@ -10,6 +10,6 @@ simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') {
     stage ('Build') {
         def logFolder = getLogFolder()
         def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'"
-        bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;${environment};&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\""
+        bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 SetupDumps;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";"
     }
 }

test/Common.FunctionalTests/AppOfflineTests.cs

@@ -101,6 +101,10 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
         [ConditionalFact]
         [RequiresIIS(IISCapability.ShutdownToken)]
         public async Task AppOfflineDroppedWhileSiteStarting_SiteShutsDown_InProcess()
+        {
+            // This test often hits a race between debug logging and stdout redirection closing the handle
+            // we are fine having this race
+            using (AppVerifier.Disable(DeployerSelector.ServerType, 0x300))
             {
                 var deploymentResult = await DeployApp(HostingModel.InProcess);
 
@@ -136,6 +140,8 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
                 }
 
                 Assert.True(false);
+
+            }
         }
 
         [ConditionalFact]

test/Common.FunctionalTests/Utilities/AppVerifier.cs

@@ -0,0 +1,83 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+    public class AppVerifier
+    {
+        private static readonly TimeSpan AppVerifierCommandTimeout = TimeSpan.FromSeconds(5);
+
+        public static IDisposable Disable(ServerType serverType, int code)
+        {
+            // Set in SetupTestEnvironment.ps1
+            var enabledCodes = (Environment.GetEnvironmentVariable("APPVERIFIER_ENABLED_CODES") ?? "").Split(' ');
+            string processName;
+            switch (serverType)
+            {
+                case ServerType.IISExpress:
+                    processName = "iisexpress.exe";
+                    break;
+                case ServerType.IIS:
+                    processName = "w3wp.exe";
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(serverType), serverType, null);
+            }
+
+            if (!enabledCodes.Contains(code.ToString()))
+            {
+                return null;
+            }
+
+            RunProcessAndWaitForExit("appverif.exe", $"-configure {code} -for {processName} -with ErrorReport=0", AppVerifierCommandTimeout);
+            return new AppVerifierToken(processName, code.ToString());
+        }
+
+        private static void RunProcessAndWaitForExit(string fileName, string arguments, TimeSpan timeout)
+        {
+            var startInfo = new ProcessStartInfo
+            {
+                FileName = fileName,
+                Arguments = arguments,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+            };
+
+            var process = Process.Start(startInfo);
+
+            if (!process.WaitForExit((int)timeout.TotalMilliseconds))
+            {
+                process.Kill();
+            }
+
+            if (process.ExitCode != 0)
+            {
+                throw new InvalidOperationException($"Exit code {process.ExitCode} when running {fileName} {arguments}. Stdout: {process.StandardOutput.ReadToEnd()}");
+            }
+        }
+
+        public class AppVerifierToken : IDisposable
+        {
+            private readonly string _processName;
+
+            private readonly string _codes;
+
+            public AppVerifierToken(string processName, string codes)
+            {
+                _processName = processName;
+                _codes = codes;
+            }
+
+            public void Dispose()
+            {
+                //
+                RunProcessAndWaitForExit("appverif.exe", $"-configure {_codes} -for {_processName} -with ErrorReport={Environment.GetEnvironmentVariable("APPVERIFIER_LEVEL")}", AppVerifierCommandTimeout);
+            }
+        }
+    }
+}

test/IIS.FunctionalTests/ServicesTests.cs

@@ -31,9 +31,13 @@ namespace IIS.FunctionalTests
         [InlineData(HostingModel.InProcess)]
         [InlineData(HostingModel.OutOfProcess)]
         public async Task ApplicationPreloadStartsApp(HostingModel hostingModel)
+        {
+            // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
+            using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
             {
                 var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
-            baseDeploymentParameters.TransformArguments((args, contentRoot)=> $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
+                baseDeploymentParameters.TransformArguments(
+                    (args, contentRoot) => $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
                 EnablePreload(baseDeploymentParameters);
 
                 var result = await DeployAsync(baseDeploymentParameters);
@@ -42,12 +46,16 @@ namespace IIS.FunctionalTests
                 StopServer();
                 EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
             }
+        }
 
         [ConditionalTheory]
         [RequiresIIS(IISCapability.ApplicationInitialization)]
         [InlineData(HostingModel.InProcess)]
         [InlineData(HostingModel.OutOfProcess)]
         public async Task ApplicationInitializationPageIsRequested(HostingModel hostingModel)
+        {
+            // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
+            using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
             {
                 var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
                 EnablePreload(baseDeploymentParameters);
@@ -66,6 +74,7 @@ namespace IIS.FunctionalTests
                 StopServer();
                 EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
             }
+        }
 
         private static void EnablePreload(IISDeploymentParameters baseDeploymentParameters)
         {

tools/SetupTestEnvironment.ps1

@@ -1,39 +1,10 @@
 param($Mode)
 
-function Setup-appverif($application)
+$DumpFolder = "$env:ASPNETCORE_TEST_LOG_DIR\dumps"
+if (!($DumpFolder))
 {
-    appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application
-    $onlyLog = 0x1E1;
-    $codes = @(
-        # Exceptions
-        0x650,
-        # Handles
-        0x300, 0x301, 0x302, 0x303, 0x304, 0x305,
-        # Heaps
-        0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014,
-        # Leak
-        0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906,
-        # Locks
-        0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215,
-        # Memory
-        0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E,
-        # SRWLock
-        0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257,
-        # TSL
-        0x350, 0x351, 0x352,
-        # ThreadPool
-        0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D
-    );
-    appverif.exe -configure $codes -for $application -with ErrorReport=$onlyLog
-}
-
-function Shutdown-appverif($application)
-{
-    appverif.exe -export log -for $application -with To=$LogsFolder\$application.xml Log=0
-    appverif.exe -disable * -for $application
-}
-
     $DumpFolder = "$PSScriptRoot\..\artifacts\dumps"
+}
 if (!(Test-Path $DumpFolder))
 {
     New-Item $DumpFolder -ItemType Directory;
@@ -51,27 +22,56 @@ $werHive = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting";
 $ldHive = "$werHive\LocalDumps";
 
 
-$cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
+function Setup-appverif($application)
-if (!(Test-Path $cdb))
 {
-    $downloadedFile = [System.IO.Path]::GetTempFileName();
+    appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application
-    $downloadedFile = "$downloadedFile.exe";
+    $level = 0x1E1;
-    Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
+    $codes = @(
-    & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
+        # Exceptions
+        0x650,
+        # Handles
+        0x300, 0x301, 0x302, 0x303, 0x304, # 0x305,
+        # Heaps
+        0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014,
+        # Leak
+        0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906,
+        # Locks
+        0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215,
+        # Memory
+        0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E,
+        # SRWLock
+        0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257,
+        # TSL
+        0x350, 0x351, 0x352,
+        # ThreadPool
+        0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D
+    );
+
+    setx APPVERIFIER_ENABLED_CODES "$codes";
+    setx APPVERIFIER_LEVEL $level;
+    appverif.exe -configure $codes -for $application -with ErrorReport=$level
+
+    # 0x305, - disabled because coreclr.dll!SetThreadName(void *) ofthen passes invalid handle (0xffffff)
+    appverif.exe -configure 0x305 -for $application -with ErrorReport=0
 }
 
-if ($Mode -eq "Setup")
+function Shutdown-appverif($application)
 {
-    Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
+    setx APPVERIFIER_ENABLED_CODES "`"`"";
+    setx APPVERIFIER_LEVEL "`"`"";
 
-    Setup-appverif w3wp.exe
+    appverif.exe -disable * -for $application
-    Setup-appverif iisexpress.exe
+}
 
+function Setup-Dumps()
+{
     if (!(Test-Path $ldHive ))
     {
         New-Item -Path $werHive -Name LocalDumps
     }
 
+    Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
+
     New-ItemProperty $werHive -Name "DontShowUI" -Value 1 -PropertyType "DWORD" -Force;
 
     New-ItemProperty $ldHive -Name "DumpFolder" -Value $DumpFolder -PropertyType "ExpandString" -Force;
@@ -81,7 +81,7 @@ if ($Mode -eq "Setup")
     Restart-Service WerSvc
 }
 
-if ($Mode -eq "Shutdown")
+function Shutdown-Dumps()
 {
     Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe;
 
@@ -89,8 +89,14 @@ if ($Mode -eq "Shutdown")
 
     New-ItemProperty $werHive -Name "DontShowUI" -Value 0 -PropertyType "DWORD" -Force;
 
-    Shutdown-appverif w3wp.exe
+    $cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
-    Shutdown-appverif iisexpress.exe
+    if (!(Test-Path $cdb))
+    {
+        $downloadedFile = [System.IO.Path]::GetTempFileName();
+        $downloadedFile = "$downloadedFile.exe";
+        Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
+        & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
+    }
 
     foreach ($dump in (Get-ChildItem -Path $DumpFolder -Filter "*.dmp"))
     {
@@ -102,4 +108,28 @@ if ($Mode -eq "Shutdown")
     }
 }
 
+if ($Mode -eq "Setup")
+{
+    Setup-appverif w3wp.exe
+    Setup-appverif iisexpress.exe
+
+    Setup-Dumps;
+}
+
+if ($Mode -eq "SetupDumps")
+{
+    Shutdown-appverif w3wp.exe
+    Shutdown-appverif iisexpress.exe
+
+    Setup-Dumps;
+}
+
+if ($Mode -eq "Shutdown")
+{
+    Shutdown-appverif w3wp.exe
+    Shutdown-appverif iisexpress.exe
+
+    Shutdown-Dumps;
+}
+
 Exit 0;