diff --git a/.vsts-pipelines/templates/build-steps.yml b/.vsts-pipelines/templates/build-steps.yml index 2d6544e107..36219f533b 100644 --- a/.vsts-pipelines/templates/build-steps.yml +++ b/.vsts-pipelines/templates/build-steps.yml @@ -16,13 +16,6 @@ phases: artifactName: logs artifactType: Container pathtoPublish: artifacts/logs - - task: PublishBuildArtifacts@1 - displayName: Upload dumps - condition: eq(variables['system.pullrequest.isfork'], false) - inputs: - artifactName: dumps - artifactType: Container - pathtoPublish: artifacts/dumps - template: .vsts-pipelines/templates/phases/default-build.yml@buildtools parameters: diff --git a/build/buildpipeline/pipeline.groovy b/build/buildpipeline/pipeline.groovy index f3be6756b9..e1c8b612e9 100644 --- a/build/buildpipeline/pipeline.groovy +++ b/build/buildpipeline/pipeline.groovy @@ -1,6 +1,7 @@ import org.dotnet.ci.pipelines.Pipeline def windowsPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows.groovy') +def windowsAppverifPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows-appverif.groovy') def configurations = [ 'Debug', @@ -16,4 +17,5 @@ configurations.each { configuration -> windowsPipeline.triggerPipelineOnEveryGithubPR("Windows ${configuration} x64 Build", params) windowsPipeline.triggerPipelineOnGithubPush(params) + windowsAppverifPipeline.triggerPipelineOnEveryGithubPR("Windows AppVerifier ${configuration} x64 Build", params) } diff --git a/build/buildpipeline/windows-appverif.groovy b/build/buildpipeline/windows-appverif.groovy new file mode 100644 index 0000000000..0c1a6affe8 --- /dev/null +++ b/build/buildpipeline/windows-appverif.groovy @@ -0,0 +1,15 @@ +@Library('dotnet-ci') _ + +// 'node' indicates to Jenkins that the enclosed block runs on a node that matches +// the label 'windows-with-vs' +simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') { + stage ('Checking out source') { + checkout scm + bat 'git submodule update --init --recursive' + } + stage ('Build') { + def logFolder = getLogFolder() + def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'" + bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 Setup;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";" + } +} diff --git a/build/buildpipeline/windows.groovy b/build/buildpipeline/windows.groovy index eb1a95bca3..35bb4737ae 100644 --- a/build/buildpipeline/windows.groovy +++ b/build/buildpipeline/windows.groovy @@ -10,6 +10,6 @@ simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') { stage ('Build') { def logFolder = getLogFolder() def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'" - bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;${environment};&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\"" + bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 SetupDumps;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";" } } diff --git a/test/Common.FunctionalTests/AppOfflineTests.cs b/test/Common.FunctionalTests/AppOfflineTests.cs index c314a9dfab..7d8d661ef0 100644 --- a/test/Common.FunctionalTests/AppOfflineTests.cs +++ b/test/Common.FunctionalTests/AppOfflineTests.cs @@ -102,40 +102,46 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests [RequiresIIS(IISCapability.ShutdownToken)] public async Task AppOfflineDroppedWhileSiteStarting_SiteShutsDown_InProcess() { - var deploymentResult = await DeployApp(HostingModel.InProcess); - - for (int i = 0; i < 10; i++) + // This test often hits a race between debug logging and stdout redirection closing the handle + // we are fine having this race + using (AppVerifier.Disable(DeployerSelector.ServerType, 0x300)) { - // send first request and add app_offline while app is starting - var runningTask = AssertAppOffline(deploymentResult); + var deploymentResult = await DeployApp(HostingModel.InProcess); - // This test tries to hit a race where we drop app_offline file while - // in process application is starting, application start takes at least 400ms - // so we back off for 100ms to allow request to reach request handler - // Test itself is racy and can result in two scenarios - // 1. ANCM detects app_offline before it starts the request - if AssertAppOffline succeeds we've hit it - // 2. Intended scenario where app starts and then shuts down - // In first case we remove app_offline and try again - await Task.Delay(RetryDelay); - - AddAppOffline(deploymentResult.ContentRoot); - - try + for (int i = 0; i < 10; i++) { - await runningTask.DefaultTimeout(); + // send first request and add app_offline while app is starting + var runningTask = AssertAppOffline(deploymentResult); - // if AssertAppOffline succeeded ANCM have picked up app_offline before starting the app - // try again - RemoveAppOffline(deploymentResult.ContentRoot); - } - catch - { - deploymentResult.AssertWorkerProcessStop(); - return; + // This test tries to hit a race where we drop app_offline file while + // in process application is starting, application start takes at least 400ms + // so we back off for 100ms to allow request to reach request handler + // Test itself is racy and can result in two scenarios + // 1. ANCM detects app_offline before it starts the request - if AssertAppOffline succeeds we've hit it + // 2. Intended scenario where app starts and then shuts down + // In first case we remove app_offline and try again + await Task.Delay(RetryDelay); + + AddAppOffline(deploymentResult.ContentRoot); + + try + { + await runningTask.DefaultTimeout(); + + // if AssertAppOffline succeeded ANCM have picked up app_offline before starting the app + // try again + RemoveAppOffline(deploymentResult.ContentRoot); + } + catch + { + deploymentResult.AssertWorkerProcessStop(); + return; + } } + + Assert.True(false); + } - - Assert.True(false); } [ConditionalFact] diff --git a/test/Common.FunctionalTests/Utilities/AppVerifier.cs b/test/Common.FunctionalTests/Utilities/AppVerifier.cs new file mode 100644 index 0000000000..ab059d789d --- /dev/null +++ b/test/Common.FunctionalTests/Utilities/AppVerifier.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; + +namespace Microsoft.AspNetCore.Server.IntegrationTesting +{ + public class AppVerifier + { + private static readonly TimeSpan AppVerifierCommandTimeout = TimeSpan.FromSeconds(5); + + public static IDisposable Disable(ServerType serverType, int code) + { + // Set in SetupTestEnvironment.ps1 + var enabledCodes = (Environment.GetEnvironmentVariable("APPVERIFIER_ENABLED_CODES") ?? "").Split(' '); + string processName; + switch (serverType) + { + case ServerType.IISExpress: + processName = "iisexpress.exe"; + break; + case ServerType.IIS: + processName = "w3wp.exe"; + break; + default: + throw new ArgumentOutOfRangeException(nameof(serverType), serverType, null); + } + + if (!enabledCodes.Contains(code.ToString())) + { + return null; + } + + RunProcessAndWaitForExit("appverif.exe", $"-configure {code} -for {processName} -with ErrorReport=0", AppVerifierCommandTimeout); + return new AppVerifierToken(processName, code.ToString()); + } + + private static void RunProcessAndWaitForExit(string fileName, string arguments, TimeSpan timeout) + { + var startInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + var process = Process.Start(startInfo); + + if (!process.WaitForExit((int)timeout.TotalMilliseconds)) + { + process.Kill(); + } + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"Exit code {process.ExitCode} when running {fileName} {arguments}. Stdout: {process.StandardOutput.ReadToEnd()}"); + } + } + + public class AppVerifierToken : IDisposable + { + private readonly string _processName; + + private readonly string _codes; + + public AppVerifierToken(string processName, string codes) + { + _processName = processName; + _codes = codes; + } + + public void Dispose() + { + // + RunProcessAndWaitForExit("appverif.exe", $"-configure {_codes} -for {_processName} -with ErrorReport={Environment.GetEnvironmentVariable("APPVERIFIER_LEVEL")}", AppVerifierCommandTimeout); + } + } + } +} diff --git a/test/IIS.FunctionalTests/ServicesTests.cs b/test/IIS.FunctionalTests/ServicesTests.cs index b91fc8aa20..47e53b8d40 100644 --- a/test/IIS.FunctionalTests/ServicesTests.cs +++ b/test/IIS.FunctionalTests/ServicesTests.cs @@ -32,15 +32,20 @@ namespace IIS.FunctionalTests [InlineData(HostingModel.OutOfProcess)] public async Task ApplicationPreloadStartsApp(HostingModel hostingModel) { - var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true); - baseDeploymentParameters.TransformArguments((args, contentRoot)=> $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\""); - EnablePreload(baseDeploymentParameters); + // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team + using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900)) + { + var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true); + baseDeploymentParameters.TransformArguments( + (args, contentRoot) => $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\""); + EnablePreload(baseDeploymentParameters); - var result = await DeployAsync(baseDeploymentParameters); + var result = await DeployAsync(baseDeploymentParameters); - await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200); - StopServer(); - EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result)); + await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200); + StopServer(); + EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result)); + } } [ConditionalTheory] @@ -49,22 +54,26 @@ namespace IIS.FunctionalTests [InlineData(HostingModel.OutOfProcess)] public async Task ApplicationInitializationPageIsRequested(HostingModel hostingModel) { - var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true); - EnablePreload(baseDeploymentParameters); + // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team + using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900)) + { + var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true); + EnablePreload(baseDeploymentParameters); - baseDeploymentParameters.ServerConfigActionList.Add( - (config, _) => { - config - .RequiredElement("system.webServer") - .GetOrAdd("applicationInitialization") - .GetOrAdd("add", "initializationPage", "/CreateFile"); - }); + baseDeploymentParameters.ServerConfigActionList.Add( + (config, _) => { + config + .RequiredElement("system.webServer") + .GetOrAdd("applicationInitialization") + .GetOrAdd("add", "initializationPage", "/CreateFile"); + }); - var result = await DeployAsync(baseDeploymentParameters); + var result = await DeployAsync(baseDeploymentParameters); - await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200); - StopServer(); - EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result)); + await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200); + StopServer(); + EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result)); + } } private static void EnablePreload(IISDeploymentParameters baseDeploymentParameters) diff --git a/tools/SetupTestEnvironment.ps1 b/tools/SetupTestEnvironment.ps1 index dcdaa84f57..625b040e39 100644 --- a/tools/SetupTestEnvironment.ps1 +++ b/tools/SetupTestEnvironment.ps1 @@ -1,39 +1,10 @@ param($Mode) -function Setup-appverif($application) +$DumpFolder = "$env:ASPNETCORE_TEST_LOG_DIR\dumps" +if (!($DumpFolder)) { - appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application - $onlyLog = 0x1E1; - $codes = @( - # Exceptions - 0x650, - # Handles - 0x300, 0x301, 0x302, 0x303, 0x304, 0x305, - # Heaps - 0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014, - # Leak - 0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906, - # Locks - 0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215, - # Memory - 0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E, - # SRWLock - 0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257, - # TSL - 0x350, 0x351, 0x352, - # ThreadPool - 0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D - ); - appverif.exe -configure $codes -for $application -with ErrorReport=$onlyLog + $DumpFolder = "$PSScriptRoot\..\artifacts\dumps" } - -function Shutdown-appverif($application) -{ - appverif.exe -export log -for $application -with To=$LogsFolder\$application.xml Log=0 - appverif.exe -disable * -for $application -} - -$DumpFolder = "$PSScriptRoot\..\artifacts\dumps" if (!(Test-Path $DumpFolder)) { New-Item $DumpFolder -ItemType Directory; @@ -51,27 +22,56 @@ $werHive = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting"; $ldHive = "$werHive\LocalDumps"; -$cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe" -if (!(Test-Path $cdb)) +function Setup-appverif($application) { - $downloadedFile = [System.IO.Path]::GetTempFileName(); - $downloadedFile = "$downloadedFile.exe"; - Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile; - & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q; + appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application + $level = 0x1E1; + $codes = @( + # Exceptions + 0x650, + # Handles + 0x300, 0x301, 0x302, 0x303, 0x304, # 0x305, + # Heaps + 0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014, + # Leak + 0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906, + # Locks + 0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215, + # Memory + 0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E, + # SRWLock + 0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257, + # TSL + 0x350, 0x351, 0x352, + # ThreadPool + 0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D + ); + + setx APPVERIFIER_ENABLED_CODES "$codes"; + setx APPVERIFIER_LEVEL $level; + appverif.exe -configure $codes -for $application -with ErrorReport=$level + + # 0x305, - disabled because coreclr.dll!SetThreadName(void *) ofthen passes invalid handle (0xffffff) + appverif.exe -configure 0x305 -for $application -with ErrorReport=0 } -if ($Mode -eq "Setup") +function Shutdown-appverif($application) { - Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe; + setx APPVERIFIER_ENABLED_CODES "`"`""; + setx APPVERIFIER_LEVEL "`"`""; - Setup-appverif w3wp.exe - Setup-appverif iisexpress.exe + appverif.exe -disable * -for $application +} +function Setup-Dumps() +{ if (!(Test-Path $ldHive )) { New-Item -Path $werHive -Name LocalDumps } + Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe; + New-ItemProperty $werHive -Name "DontShowUI" -Value 1 -PropertyType "DWORD" -Force; New-ItemProperty $ldHive -Name "DumpFolder" -Value $DumpFolder -PropertyType "ExpandString" -Force; @@ -81,7 +81,7 @@ if ($Mode -eq "Setup") Restart-Service WerSvc } -if ($Mode -eq "Shutdown") +function Shutdown-Dumps() { Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe; @@ -89,8 +89,14 @@ if ($Mode -eq "Shutdown") New-ItemProperty $werHive -Name "DontShowUI" -Value 0 -PropertyType "DWORD" -Force; - Shutdown-appverif w3wp.exe - Shutdown-appverif iisexpress.exe + $cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe" + if (!(Test-Path $cdb)) + { + $downloadedFile = [System.IO.Path]::GetTempFileName(); + $downloadedFile = "$downloadedFile.exe"; + Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile; + & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q; + } foreach ($dump in (Get-ChildItem -Path $DumpFolder -Filter "*.dmp")) { @@ -102,4 +108,28 @@ if ($Mode -eq "Shutdown") } } +if ($Mode -eq "Setup") +{ + Setup-appverif w3wp.exe + Setup-appverif iisexpress.exe + + Setup-Dumps; +} + +if ($Mode -eq "SetupDumps") +{ + Shutdown-appverif w3wp.exe + Shutdown-appverif iisexpress.exe + + Setup-Dumps; +} + +if ($Mode -eq "Shutdown") +{ + Shutdown-appverif w3wp.exe + Shutdown-appverif iisexpress.exe + + Shutdown-Dumps; +} + Exit 0; \ No newline at end of file