Enable appverifier and dumps on jenkins (#1491)
This commit is contained in:
Pavel Krymets
GitHub
parent
9b1fd75b7e
commit
1fb57d9ed9
|
|
@ -16,13 +16,6 @@ phases:
|
|||
artifactName: logs
|
||||
artifactType: Container
|
||||
pathtoPublish: artifacts/logs
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: Upload dumps
|
||||
condition: eq(variables['system.pullrequest.isfork'], false)
|
||||
inputs:
|
||||
artifactName: dumps
|
||||
artifactType: Container
|
||||
pathtoPublish: artifacts/dumps
|
||||
|
||||
- template: .vsts-pipelines/templates/phases/default-build.yml@buildtools
|
||||
parameters:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import org.dotnet.ci.pipelines.Pipeline
|
||||
|
||||
def windowsPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows.groovy')
|
||||
def windowsAppverifPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows-appverif.groovy')
|
||||
|
||||
def configurations = [
|
||||
'Debug',
|
||||
|
|
@ -16,4 +17,5 @@ configurations.each { configuration ->
|
|||
windowsPipeline.triggerPipelineOnEveryGithubPR("Windows ${configuration} x64 Build", params)
|
||||
windowsPipeline.triggerPipelineOnGithubPush(params)
|
||||
|
||||
windowsAppverifPipeline.triggerPipelineOnEveryGithubPR("Windows AppVerifier ${configuration} x64 Build", params)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
@Library('dotnet-ci') _
|
||||
|
||||
// 'node' indicates to Jenkins that the enclosed block runs on a node that matches
|
||||
// the label 'windows-with-vs'
|
||||
simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') {
|
||||
stage ('Checking out source') {
|
||||
checkout scm
|
||||
bat 'git submodule update --init --recursive'
|
||||
}
|
||||
stage ('Build') {
|
||||
def logFolder = getLogFolder()
|
||||
def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'"
|
||||
bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 Setup;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";"
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,6 @@ simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') {
|
|||
stage ('Build') {
|
||||
def logFolder = getLogFolder()
|
||||
def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'"
|
||||
bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;${environment};&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\""
|
||||
bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 SetupDumps;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,40 +102,46 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
|
|||
[RequiresIIS(IISCapability.ShutdownToken)]
|
||||
public async Task AppOfflineDroppedWhileSiteStarting_SiteShutsDown_InProcess()
|
||||
{
|
||||
var deploymentResult = await DeployApp(HostingModel.InProcess);
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
// This test often hits a race between debug logging and stdout redirection closing the handle
|
||||
// we are fine having this race
|
||||
using (AppVerifier.Disable(DeployerSelector.ServerType, 0x300))
|
||||
{
|
||||
// send first request and add app_offline while app is starting
|
||||
var runningTask = AssertAppOffline(deploymentResult);
|
||||
var deploymentResult = await DeployApp(HostingModel.InProcess);
|
||||
|
||||
// This test tries to hit a race where we drop app_offline file while
|
||||
// in process application is starting, application start takes at least 400ms
|
||||
// so we back off for 100ms to allow request to reach request handler
|
||||
// Test itself is racy and can result in two scenarios
|
||||
// 1. ANCM detects app_offline before it starts the request - if AssertAppOffline succeeds we've hit it
|
||||
// 2. Intended scenario where app starts and then shuts down
|
||||
// In first case we remove app_offline and try again
|
||||
await Task.Delay(RetryDelay);
|
||||
|
||||
AddAppOffline(deploymentResult.ContentRoot);
|
||||
|
||||
try
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
await runningTask.DefaultTimeout();
|
||||
// send first request and add app_offline while app is starting
|
||||
var runningTask = AssertAppOffline(deploymentResult);
|
||||
|
||||
// if AssertAppOffline succeeded ANCM have picked up app_offline before starting the app
|
||||
// try again
|
||||
RemoveAppOffline(deploymentResult.ContentRoot);
|
||||
}
|
||||
catch
|
||||
{
|
||||
deploymentResult.AssertWorkerProcessStop();
|
||||
return;
|
||||
// This test tries to hit a race where we drop app_offline file while
|
||||
// in process application is starting, application start takes at least 400ms
|
||||
// so we back off for 100ms to allow request to reach request handler
|
||||
// Test itself is racy and can result in two scenarios
|
||||
// 1. ANCM detects app_offline before it starts the request - if AssertAppOffline succeeds we've hit it
|
||||
// 2. Intended scenario where app starts and then shuts down
|
||||
// In first case we remove app_offline and try again
|
||||
await Task.Delay(RetryDelay);
|
||||
|
||||
AddAppOffline(deploymentResult.ContentRoot);
|
||||
|
||||
try
|
||||
{
|
||||
await runningTask.DefaultTimeout();
|
||||
|
||||
// if AssertAppOffline succeeded ANCM have picked up app_offline before starting the app
|
||||
// try again
|
||||
RemoveAppOffline(deploymentResult.ContentRoot);
|
||||
}
|
||||
catch
|
||||
{
|
||||
deploymentResult.AssertWorkerProcessStop();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(false);
|
||||
|
||||
}
|
||||
|
||||
Assert.True(false);
|
||||
}
|
||||
|
||||
[ConditionalFact]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
// Copyright (c) .NET Foundation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
|
||||
namespace Microsoft.AspNetCore.Server.IntegrationTesting
|
||||
{
|
||||
public class AppVerifier
|
||||
{
|
||||
private static readonly TimeSpan AppVerifierCommandTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
public static IDisposable Disable(ServerType serverType, int code)
|
||||
{
|
||||
// Set in SetupTestEnvironment.ps1
|
||||
var enabledCodes = (Environment.GetEnvironmentVariable("APPVERIFIER_ENABLED_CODES") ?? "").Split(' ');
|
||||
string processName;
|
||||
switch (serverType)
|
||||
{
|
||||
case ServerType.IISExpress:
|
||||
processName = "iisexpress.exe";
|
||||
break;
|
||||
case ServerType.IIS:
|
||||
processName = "w3wp.exe";
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(serverType), serverType, null);
|
||||
}
|
||||
|
||||
if (!enabledCodes.Contains(code.ToString()))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
RunProcessAndWaitForExit("appverif.exe", $"-configure {code} -for {processName} -with ErrorReport=0", AppVerifierCommandTimeout);
|
||||
return new AppVerifierToken(processName, code.ToString());
|
||||
}
|
||||
|
||||
private static void RunProcessAndWaitForExit(string fileName, string arguments, TimeSpan timeout)
|
||||
{
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = fileName,
|
||||
Arguments = arguments,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
|
||||
var process = Process.Start(startInfo);
|
||||
|
||||
if (!process.WaitForExit((int)timeout.TotalMilliseconds))
|
||||
{
|
||||
process.Kill();
|
||||
}
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Exit code {process.ExitCode} when running {fileName} {arguments}. Stdout: {process.StandardOutput.ReadToEnd()}");
|
||||
}
|
||||
}
|
||||
|
||||
public class AppVerifierToken : IDisposable
|
||||
{
|
||||
private readonly string _processName;
|
||||
|
||||
private readonly string _codes;
|
||||
|
||||
public AppVerifierToken(string processName, string codes)
|
||||
{
|
||||
_processName = processName;
|
||||
_codes = codes;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
//
|
||||
RunProcessAndWaitForExit("appverif.exe", $"-configure {_codes} -for {_processName} -with ErrorReport={Environment.GetEnvironmentVariable("APPVERIFIER_LEVEL")}", AppVerifierCommandTimeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -32,15 +32,20 @@ namespace IIS.FunctionalTests
|
|||
[InlineData(HostingModel.OutOfProcess)]
|
||||
public async Task ApplicationPreloadStartsApp(HostingModel hostingModel)
|
||||
{
|
||||
var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
|
||||
baseDeploymentParameters.TransformArguments((args, contentRoot)=> $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
|
||||
EnablePreload(baseDeploymentParameters);
|
||||
// This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
|
||||
using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
|
||||
{
|
||||
var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
|
||||
baseDeploymentParameters.TransformArguments(
|
||||
(args, contentRoot) => $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
|
||||
EnablePreload(baseDeploymentParameters);
|
||||
|
||||
var result = await DeployAsync(baseDeploymentParameters);
|
||||
var result = await DeployAsync(baseDeploymentParameters);
|
||||
|
||||
await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
|
||||
StopServer();
|
||||
EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
|
||||
await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
|
||||
StopServer();
|
||||
EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
|
||||
}
|
||||
}
|
||||
|
||||
[ConditionalTheory]
|
||||
|
|
@ -49,22 +54,26 @@ namespace IIS.FunctionalTests
|
|||
[InlineData(HostingModel.OutOfProcess)]
|
||||
public async Task ApplicationInitializationPageIsRequested(HostingModel hostingModel)
|
||||
{
|
||||
var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
|
||||
EnablePreload(baseDeploymentParameters);
|
||||
// This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
|
||||
using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
|
||||
{
|
||||
var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
|
||||
EnablePreload(baseDeploymentParameters);
|
||||
|
||||
baseDeploymentParameters.ServerConfigActionList.Add(
|
||||
(config, _) => {
|
||||
config
|
||||
.RequiredElement("system.webServer")
|
||||
.GetOrAdd("applicationInitialization")
|
||||
.GetOrAdd("add", "initializationPage", "/CreateFile");
|
||||
});
|
||||
baseDeploymentParameters.ServerConfigActionList.Add(
|
||||
(config, _) => {
|
||||
config
|
||||
.RequiredElement("system.webServer")
|
||||
.GetOrAdd("applicationInitialization")
|
||||
.GetOrAdd("add", "initializationPage", "/CreateFile");
|
||||
});
|
||||
|
||||
var result = await DeployAsync(baseDeploymentParameters);
|
||||
var result = await DeployAsync(baseDeploymentParameters);
|
||||
|
||||
await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
|
||||
StopServer();
|
||||
EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
|
||||
await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
|
||||
StopServer();
|
||||
EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
|
||||
}
|
||||
}
|
||||
|
||||
private static void EnablePreload(IISDeploymentParameters baseDeploymentParameters)
|
||||
|
|
|
|||
|
|
@ -1,39 +1,10 @@
|
|||
param($Mode)
|
||||
|
||||
function Setup-appverif($application)
|
||||
$DumpFolder = "$env:ASPNETCORE_TEST_LOG_DIR\dumps"
|
||||
if (!($DumpFolder))
|
||||
{
|
||||
appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application
|
||||
$onlyLog = 0x1E1;
|
||||
$codes = @(
|
||||
# Exceptions
|
||||
0x650,
|
||||
# Handles
|
||||
0x300, 0x301, 0x302, 0x303, 0x304, 0x305,
|
||||
# Heaps
|
||||
0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014,
|
||||
# Leak
|
||||
0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906,
|
||||
# Locks
|
||||
0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215,
|
||||
# Memory
|
||||
0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E,
|
||||
# SRWLock
|
||||
0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257,
|
||||
# TSL
|
||||
0x350, 0x351, 0x352,
|
||||
# ThreadPool
|
||||
0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D
|
||||
);
|
||||
appverif.exe -configure $codes -for $application -with ErrorReport=$onlyLog
|
||||
$DumpFolder = "$PSScriptRoot\..\artifacts\dumps"
|
||||
}
|
||||
|
||||
function Shutdown-appverif($application)
|
||||
{
|
||||
appverif.exe -export log -for $application -with To=$LogsFolder\$application.xml Log=0
|
||||
appverif.exe -disable * -for $application
|
||||
}
|
||||
|
||||
$DumpFolder = "$PSScriptRoot\..\artifacts\dumps"
|
||||
if (!(Test-Path $DumpFolder))
|
||||
{
|
||||
New-Item $DumpFolder -ItemType Directory;
|
||||
|
|
@ -51,27 +22,56 @@ $werHive = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting";
|
|||
$ldHive = "$werHive\LocalDumps";
|
||||
|
||||
|
||||
$cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
|
||||
if (!(Test-Path $cdb))
|
||||
function Setup-appverif($application)
|
||||
{
|
||||
$downloadedFile = [System.IO.Path]::GetTempFileName();
|
||||
$downloadedFile = "$downloadedFile.exe";
|
||||
Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
|
||||
& $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
|
||||
appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application
|
||||
$level = 0x1E1;
|
||||
$codes = @(
|
||||
# Exceptions
|
||||
0x650,
|
||||
# Handles
|
||||
0x300, 0x301, 0x302, 0x303, 0x304, # 0x305,
|
||||
# Heaps
|
||||
0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014,
|
||||
# Leak
|
||||
0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906,
|
||||
# Locks
|
||||
0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215,
|
||||
# Memory
|
||||
0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E,
|
||||
# SRWLock
|
||||
0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257,
|
||||
# TSL
|
||||
0x350, 0x351, 0x352,
|
||||
# ThreadPool
|
||||
0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D
|
||||
);
|
||||
|
||||
setx APPVERIFIER_ENABLED_CODES "$codes";
|
||||
setx APPVERIFIER_LEVEL $level;
|
||||
appverif.exe -configure $codes -for $application -with ErrorReport=$level
|
||||
|
||||
# 0x305, - disabled because coreclr.dll!SetThreadName(void *) ofthen passes invalid handle (0xffffff)
|
||||
appverif.exe -configure 0x305 -for $application -with ErrorReport=0
|
||||
}
|
||||
|
||||
if ($Mode -eq "Setup")
|
||||
function Shutdown-appverif($application)
|
||||
{
|
||||
Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
|
||||
setx APPVERIFIER_ENABLED_CODES "`"`"";
|
||||
setx APPVERIFIER_LEVEL "`"`"";
|
||||
|
||||
Setup-appverif w3wp.exe
|
||||
Setup-appverif iisexpress.exe
|
||||
appverif.exe -disable * -for $application
|
||||
}
|
||||
|
||||
function Setup-Dumps()
|
||||
{
|
||||
if (!(Test-Path $ldHive ))
|
||||
{
|
||||
New-Item -Path $werHive -Name LocalDumps
|
||||
}
|
||||
|
||||
Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
|
||||
|
||||
New-ItemProperty $werHive -Name "DontShowUI" -Value 1 -PropertyType "DWORD" -Force;
|
||||
|
||||
New-ItemProperty $ldHive -Name "DumpFolder" -Value $DumpFolder -PropertyType "ExpandString" -Force;
|
||||
|
|
@ -81,7 +81,7 @@ if ($Mode -eq "Setup")
|
|||
Restart-Service WerSvc
|
||||
}
|
||||
|
||||
if ($Mode -eq "Shutdown")
|
||||
function Shutdown-Dumps()
|
||||
{
|
||||
Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe;
|
||||
|
||||
|
|
@ -89,8 +89,14 @@ if ($Mode -eq "Shutdown")
|
|||
|
||||
New-ItemProperty $werHive -Name "DontShowUI" -Value 0 -PropertyType "DWORD" -Force;
|
||||
|
||||
Shutdown-appverif w3wp.exe
|
||||
Shutdown-appverif iisexpress.exe
|
||||
$cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
|
||||
if (!(Test-Path $cdb))
|
||||
{
|
||||
$downloadedFile = [System.IO.Path]::GetTempFileName();
|
||||
$downloadedFile = "$downloadedFile.exe";
|
||||
Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
|
||||
& $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
|
||||
}
|
||||
|
||||
foreach ($dump in (Get-ChildItem -Path $DumpFolder -Filter "*.dmp"))
|
||||
{
|
||||
|
|
@ -102,4 +108,28 @@ if ($Mode -eq "Shutdown")
|
|||
}
|
||||
}
|
||||
|
||||
if ($Mode -eq "Setup")
|
||||
{
|
||||
Setup-appverif w3wp.exe
|
||||
Setup-appverif iisexpress.exe
|
||||
|
||||
Setup-Dumps;
|
||||
}
|
||||
|
||||
if ($Mode -eq "SetupDumps")
|
||||
{
|
||||
Shutdown-appverif w3wp.exe
|
||||
Shutdown-appverif iisexpress.exe
|
||||
|
||||
Setup-Dumps;
|
||||
}
|
||||
|
||||
if ($Mode -eq "Shutdown")
|
||||
{
|
||||
Shutdown-appverif w3wp.exe
|
||||
Shutdown-appverif iisexpress.exe
|
||||
|
||||
Shutdown-Dumps;
|
||||
}
|
||||
|
||||
Exit 0;
|
||||
Loading…
Reference in New Issue