Capture dumps on hanging CI builds (#8727)

* Update the build definition on VSTS pipelines to use a custom build script.
* Create a build script that starts a powershell job in the background
  that sleeps for 20 minutes and captures dumps for dotnet.exe and testhost.*.exe
  when it awakes if the build is still running.

.vsts-pipelines/builds/ci-public.yml

@@ -2,14 +2,5 @@ trigger:
 - master
 - release/*
 
-# See https://github.com/aspnet/BuildTools
-resources:
-  repositories:
-  - repository: buildtools
-    type: github
-    endpoint: DotNet-Bot GitHub Connection
-    name: aspnet/BuildTools
-    ref: refs/heads/master
-
 phases:
-- template: .vsts-pipelines/templates/project-ci.yml@buildtools
+- template: ../templates/project-ci.yml

.vsts-pipelines/templates/phases/default-build.yml

@@ -0,0 +1,118 @@
+#
+# Obsolete! New projects should use .azure/templates/jobs/default-build.yml instead
+#
+# TODO: remove this once templated projects have referenced the new location.
+#
+# default-build.yml
+# Description: Defines a build phase for invoking build.sh/cmd
+# Parameters:
+#   phaseName: string
+#       The name of the phase. Defaults to the name of the OS.
+#   queueName: string
+#       The name of the VSTS agent queue to use.
+#   agentOs: string
+#       Used in templates to define variables which are OS specific. Typically from the set { Windows, Linux, macOS }
+#   buildArgs: string
+#       Additional arguments to pass to the build.sh/cmd script.
+#       Note: -ci is always passed
+#   beforeBuild: [steps]
+#       Additional steps to run before build.sh/cmd
+#   afterBuild: [steps]
+#       Additional steps to run after build.sh/cmd
+#   artifacts:
+#      publish: boolean
+#           Should artifacts be published
+#      path: string
+#           The file path to artifacts output
+#      name: string
+#           The name of the artifact container
+#   variables: { string: string }
+#     A map of custom variables
+#   matrix: { string: { string: string } }
+#     A map of matrix configurations and variables. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#matrix
+#   demands: string | [ string ]
+#     A list of agent demands. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#demands
+#   dependsOn: string | [ string ]
+#     For fan-out/fan-in. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#phase
+
+#
+# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details
+#
+
+parameters:
+  agentOs: 'Windows'
+  # phaseName: ''
+  queueName: ''
+  buildArgs: ''
+  configuration: 'Release'
+  demands: []
+  beforeBuild: []
+  afterBuild: []
+  variables: {}
+  dependsOn: ''
+  # matrix: {} - don't define an empty object default because there is no way in template expression yet to check "if isEmpty(parameters.matrix)"
+  artifacts:
+    publish: true
+    path: 'artifacts/'  # TODO: this is going to change when we converge with dotnet/arcade tooling
+
+phases:
+- phase: ${{ coalesce(parameters.phaseName, parameters.agentOs) }}
+  dependsOn: ${{ parameters.dependsOn }}
+  queue:
+    # If a matrix of builds has been configured, run the matrix in parallel
+    ${{ if ne(parameters.matrix, '') }}:
+      parallel: 4 # Pick 4 as the default because we usually don't have a matrix of more than 4 configs, and there is no way to say parallel: all
+      matrix: ${{ parameters.matrix }}
+    # Map friendly OS names to the right queue
+    ${{ if ne(parameters.queueName, '') }}:
+      name: ${{ parameters.queueName }}
+    ${{ if and(eq(parameters.queueName, ''), eq(parameters.agentOs, 'macOS')) }}:
+      name: Hosted macOS Preview
+    ${{ if and(eq(parameters.queueName, ''), eq(parameters.agentOs, 'Linux')) }}:
+      name: Hosted Linux Preview
+    ${{ if and(eq(parameters.queueName, ''), eq(parameters.agentOs, 'Windows')) }}:
+      name: Hosted VS2017
+    demands: ${{ parameters.demands }}
+  variables:
+    AgentOsName: ${{ parameters.agentOs }}
+    ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
+    DOTNET_HOME: $(Agent.WorkFolder)/.dotnet
+    BuildScriptArgs: ${{ parameters.buildArgs }}
+    BuildConfiguration: ${{ parameters.configuration }}
+    ${{ insert }}: ${{ parameters.variables }}
+  steps:
+  - script: echo "This build template is obsolete and will be removed in the future. Please update your schema to use the  .azure/templates/jobs/default-build.yml template"
+    displayName: THIS TEMPLATE IS OBSOLETE
+  - checkout: self
+    clean: true
+  - ${{ parameters.beforeBuild }}
+  - ${{ if eq(parameters.agentOs, 'Windows') }}:
+    - script: .\buildWithProcDump.cmd -ci /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs)
+      displayName: Run build.cmd
+  - ${{ if ne(parameters.agentOs, 'Windows') }}:
+    - script: ./build.sh -ci -p:Configuration=$(BuildConfiguration) $(BuildScriptArgs)
+      displayName: Run build.sh
+  - task: PublishTestResults@2
+    displayName: Publish test results
+    condition: always()
+    inputs:
+      testRunner: vstest
+      testResultsFiles: 'artifacts/logs/**/*.trx'
+  - task: PublishTestResults@2
+    displayName: Publish junit test results
+    condition: always()
+    inputs:
+      testRunner: junit
+      testResultsFiles: 'artifacts/logs/**/*.junit.xml'
+  - ${{ if eq(parameters.artifacts.publish, 'true') }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Upload artifacts
+      condition: eq(variables['system.pullrequest.isfork'], false)
+      inputs:
+        pathtoPublish: ${{ parameters.artifacts.path }}
+        ${{ if eq(parameters.artifacts.name, '') }}:
+          artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration)
+        ${{ if ne(parameters.artifacts.name, '') }}:
+          artifactName: ${{ parameters.artifacts.name }}
+        artifactType: Container
+  - ${{ parameters.afterBuild }}

.vsts-pipelines/templates/project-ci.yml

@@ -0,0 +1,56 @@
+#
+# Obsolete! New projects should use .azure/templates/project-ci.yml instead
+#
+# TODO: remove this once templated projects have referenced the new location.
+#
+#
+# Description: Runs build.cmd/sh on macOS, Linux, and Windows
+# Parameters:
+#   buildArgs: string
+#       Additional arguments to pass to the build.sh/cmd script.
+#       Note: -ci is always passed
+#   beforeBuild: [steps]
+#       Additional steps to run before build.sh/cmd
+#   afterBuild: [steps]
+#       Additional steps to run after build.sh/cmd
+#   variables: {}
+#       VSTS build and environment variables
+#   matrix: {}
+#       The matrix of configurations to run. By default, it runs a Debug and Release build on all platforms
+
+parameters:
+  buildArgs: ''
+  beforeBuild: []
+  afterBuild: []
+  variables: {}
+  matrix:
+    Release:
+      BuildConfiguration: Release
+    Debug:
+      BuildConfiguration: Debug
+
+phases:
+- template: phases/default-build.yml
+  parameters:
+    agentOs: Windows
+    matrix: ${{ parameters.matrix }}
+    buildArgs: ${{ parameters.buildArgs }}
+    beforeBuild: ${{ parameters.beforeBuild }}
+    afterBuild: ${{ parameters.afterBuild }}
+    variables: ${{ parameters.variables }}
+- template: phases/default-build.yml
+  parameters:
+    agentOs: macOS
+    matrix: ${{ parameters.matrix }}
+    buildArgs: ${{ parameters.buildArgs }}
+    beforeBuild: ${{ parameters.beforeBuild }}
+    afterBuild: ${{ parameters.afterBuild }}
+    variables: ${{ parameters.variables }}
+- template: phases/default-build.yml
+  parameters:
+    agentOs: Linux
+    matrix: ${{ parameters.matrix }}
+    buildArgs: ${{ parameters.buildArgs }}
+    beforeBuild: ${{ parameters.beforeBuild }}
+    afterBuild: ${{ parameters.afterBuild }}
+    variables: ${{ parameters.variables }}

buildWithProcDump.cmd

@@ -0,0 +1,2 @@
+@ECHO OFF
+PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0runWithProcDump.ps1' %*; exit $LASTEXITCODE"

runWithProcDump.ps1

@@ -0,0 +1,37 @@
+try
+{
+  $cwd = $PSScriptRoot;
+  $job = Start-Job {    
+    $dumpsFolder = "${using:cwd}/artifacts/dumps";
+    mkdir $dumpsFolder -Force;
+
+    $procDumpFolder = "${using:cwd}/obj";
+    mkdir $procDumpFolder -Force;
+
+    $procDumpFolder = Resolve-Path $procDumpFolder;
+    Invoke-WebRequest https://download.sysinternals.com/files/Procdump.zip -OutFile "$procDumpFolder/procdump.zip";
+    Expand-Archive "$procDumpFolder/procdump.zip" -DestinationPath "$procDumpFolder" -Force;
+  
+    $sleepTime = (1 * 20 * 60)
+    Start-Sleep -Seconds $sleepTime;
+    Write-Host "Producing dumps in $dumpsFolder";
+    Write-Host "Process dumps to capture:"
+    $processes = Get-Process dotnet*, testhost*;
+    $processes | Format-Table;
+    Write-Host "Using ProcDump from $procDumpFolder/procdump.exe";
+    
+    $processes |
+     Select-Object -ExpandProperty ID | 
+     ForEach-Object { &"$procDumpFolder/procdump.exe" -accepteula -ma $_ $dumpsFolder }
+  }
+  Write-Host "Process dump capture job started. Running run.ps1 next";
+  ./run.ps1 default-build @args
+  # Receive-Job $job
+  Stop-Job $job
+  Remove-Job $job
+}
+catch
+{
+  write-host $_
+  exit -1;
+}