Switch agent queues used during Windows builds

.azure/pipelines/jobs/default-build.yml

@@ -27,8 +27,6 @@
 #     A map of custom variables
 #   matrix: { string: { string: string } }
 #     A map of matrix configurations and variables. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#matrix
-#   demands: string | [ string ]
-#     A list of agent demands. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#demands
 #   dependsOn: string | [ string ]
 #     For fan-out/fan-in. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#phase
 #   codeSign: boolean
@@ -47,7 +45,6 @@ parameters:
   poolName: ''
   buildArgs: ''
   configuration: 'Release'
-  demands: []
   beforeBuild: []
   afterBuild: []
   codeSign: false
@@ -74,6 +71,7 @@ jobs:
       maxParallel: 8
       matrix: ${{ parameters.matrix }}
   # Map friendly OS names to the right queue
+  # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md
   pool:
     ${{ if ne(parameters.poolName, '') }}:
       name: ${{ parameters.poolName }}
@@ -84,25 +82,25 @@ jobs:
       name: Hosted Ubuntu 1604
       vmImage: ubuntu-16.04
     ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Windows')) }}:
-      ${{ if ne(parameters.codeSign, 'true') }}:
-        name: Hosted VS2017
-        vmImage: vs2017-win2016
-      ${{ if eq(parameters.codeSign, 'true') }}:
-        name: DotNetCore-Windows
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        name: dotnet-internal-temp
+      ${{ if ne(variables['System.TeamProject'], 'internal') }}:
+        name: dotnet-external-temp
   variables:
     AgentOsName: ${{ parameters.agentOs }}
     ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
-    DOTNET_HOME: $(Agent.WorkFolder)/.dotnet
+    DOTNET_HOME: $(Agent.BuildDirectory)/.dotnet
     BuildScript: ${{ parameters.buildScript }}
     BuildScriptArgs: ${{ parameters.buildArgs }}
     BuildConfiguration: ${{ parameters.configuration }}
     BuildDirectory: ${{ parameters.buildDirectory }}
-    VSTS_OVERWRITE_TEMP: false # Workaround for https://github.com/dotnet/core-eng/issues/2812
-    ${{ if eq(parameters.codeSign, 'true') }}:
+    ${{ if eq(parameters.agentOs, 'Windows') }}:
+      JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk
+    ${{ if or(ne(parameters.codeSign, 'true'), ne(variables['System.TeamProject'], 'internal')) }}:
+      _SignType:
+    ${{ if and(eq(parameters.codeSign, 'true'), eq(variables['System.TeamProject'], 'internal')) }}:
       TeamName: AspNetCore
       _SignType: real
-    ${{ if ne(parameters.codeSign, 'true') }}:
-      _SignType:
     ${{ insert }}: ${{ parameters.variables }}
   steps:
   - checkout: self
@@ -111,13 +109,17 @@ jobs:
     displayName: Install Node 10.x
     inputs:
       versionSpec: 10.x
-  - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
+  - ${{ if eq(parameters.agentOs, 'Windows') }}:
+    - powershell: ./eng/scripts/InstallJdk.ps1 '11.0.1'
+      displayName: Install JDK 11
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
     - task: MicroBuildSigningPlugin@1
       displayName: Install MicroBuild Signing plugin
       condition: and(succeeded(), in(variables['_SignType'], 'test', 'real'))
       inputs:
         signType: $(_SignType)
         zipSources: false
+        feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
   - ${{ parameters.beforeBuild }}
   - ${{ if eq(parameters.buildScript, '') }}:
     - ${{ if eq(parameters.agentOs, 'Windows') }}:
@@ -132,6 +134,7 @@ jobs:
   - task: PublishTestResults@2
     displayName: Publish test results
     condition: always()
+    continueOnError: true
     inputs:
       testRunTitle: $(AgentOsName)-$(BuildConfiguration)
       testRunner: vstest
@@ -141,6 +144,7 @@ jobs:
     - task: PublishBuildArtifacts@1
       displayName: Upload artifacts
       condition: eq(variables['system.pullrequest.isfork'], false)
+      continueOnError: true
       inputs:
         ${{ if eq(parameters.buildDirectory, '') }}:
           pathtoPublish: ${{ parameters.artifacts.path }}
@@ -153,7 +157,7 @@ jobs:
         artifactType: Container
         parallel: true
   - ${{ parameters.afterBuild }}
-  - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows')) }}:
     - task: MicroBuildCleanup@1
       displayName: Cleanup MicroBuild tasks
       condition: always()

build.ps1

@@ -175,6 +175,7 @@ function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
     while ($retries -gt 0) {
         $retries -= 1
         try {
+            $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
             Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
             return
         }

eng/scripts/InstallJdk.ps1

@@ -0,0 +1,27 @@
+
+param(
+    [Parameter(Mandatory = $true)]
+    $JdkVersion
+    )
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+
+Set-StrictMode -Version 1
+
+if (-not $env:JAVA_HOME) {
+    throw 'You must set the JAVA_HOME environment variable to the destination of the JDK.'
+}
+
+$repoRoot = Resolve-Path "$PSScriptRoot/../.."
+$tempDir = "$repoRoot/obj"
+mkdir $tempDir -ea Ignore | out-null
+Write-Host "Starting download of JDK ${JdkVersion}"
+Invoke-WebRequest -UseBasicParsing -Uri "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/java/jdk-${JdkVersion}_windows-x64_bin.zip" -Out "$tempDir/jdk.zip"
+Write-Host "Done downloading JDK ${JdkVersion}"
+Expand-Archive "$tempDir/jdk.zip" -d "$tempDir/jdk/"
+Write-Host "Expanded JDK to $tempDir"
+mkdir (split-path -parent $env:JAVA_HOME) -ea ignore | out-null
+Write-Host "Installing JDK to $env:JAVA_HOME"
+Move-Item "$tempDir/jdk/jdk-${jdkVersion}" $env:JAVA_HOME
+Write-Host "Done installing JDK to $env:JAVA_HOME"

eng/scripts/InstallVisualStudio.ps1

@@ -48,7 +48,8 @@ $intermedateDir = "$PSScriptRoot\obj"
 mkdir $intermedateDir -ErrorAction Ignore | Out-Null
 
 $bootstrapper = "$intermedateDir\vsinstaller.exe"
-Invoke-WebRequest -Uri "https://aka.ms/vs/15/release/vs_$($Edition.ToLowerInvariant()).exe" -OutFile $bootstrapper
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+Get-RemoteFile "https://aka.ms/vs/15/release/vs_$($Edition.ToLowerInvariant()).exe" -OutFile $bootstrapper
 
 if (-not $InstallPath) {
     $InstallPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\$Edition"
@@ -78,4 +79,4 @@ Write-Host "Installing Visual Studio 2017 $Edition" -f Magenta
 Write-Host ""
 Write-Host "Running '$bootstrapper $arguments'"
 
-& $bootstrapper @arguments
+& $bootstrapper @arguments

eng/scripts/common.psm1

@@ -62,6 +62,7 @@ function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
     while ($retries -gt 0) {
         $retries -= 1
         try {
+            $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
             Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
             return
         }

korebuild.json

@@ -14,7 +14,6 @@
       "includePrerelease": true,
       "versionRange": "[15.8, 16.0)",
       "requiredWorkloads": [
-        "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81",
         "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
         "Microsoft.VisualStudio.Component.VC.ATL",
         "Microsoft.VisualStudio.Component.Windows10SDK.17134"

src/Middleware/WebSockets/setup-wstest.ps1

@@ -14,6 +14,7 @@ $WsTest = Join-Path $ScriptsDir "wstest.exe"
 $VCPythonMsi = Join-Path $VendorDir "VCForPython27.msi"
 if(!(Test-Path $VCPythonMsi)) {
     Write-Host "Downloading VCForPython27.msi"
+    $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
     Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile "$VCPythonMsi"
 }
 else {
@@ -49,4 +50,4 @@ virtualenv $VirtualEnvDir
 # Install autobahn into the virtualenv
 & "$ScriptsDir\pip" install autobahntestsuite
 
-Write-Host "Using wstest from: '$WsTest'"
+Write-Host "Using wstest from: '$WsTest'"

src/Mvc/runWithProcDump.ps1

@@ -1,7 +1,7 @@
 try
 {
   $cwd = $PSScriptRoot;
-  $job = Start-Job {    
+  $job = Start-Job {
     $dumpsFolder = "${using:cwd}/artifacts/dumps";
     mkdir $dumpsFolder -Force;
 
@@ -9,9 +9,10 @@ try
     mkdir $procDumpFolder -Force;
 
     $procDumpFolder = Resolve-Path $procDumpFolder;
+    $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
     Invoke-WebRequest https://download.sysinternals.com/files/Procdump.zip -OutFile "$procDumpFolder/procdump.zip";
     Expand-Archive "$procDumpFolder/procdump.zip" -DestinationPath "$procDumpFolder" -Force;
-  
+
     $sleepTime = (1 * 20 * 60)
     Start-Sleep -Seconds $sleepTime;
     Write-Host "Producing dumps in $dumpsFolder";
@@ -19,9 +20,9 @@ try
     $processes = Get-Process dotnet*, testhost*;
     $processes | Format-Table;
     Write-Host "Using ProcDump from $procDumpFolder/procdump.exe";
-    
+
     $processes |
-     Select-Object -ExpandProperty ID | 
+     Select-Object -ExpandProperty ID |
      ForEach-Object { &"$procDumpFolder/procdump.exe" -accepteula -ma $_ $dumpsFolder }
   }
   Write-Host "Process dump capture job started. Running run.ps1 next";
@@ -34,4 +35,4 @@ catch
 {
   write-host $_
   exit -1;
-}
+}

src/Servers/IIS/tools/SetupTestEnvironment.ps1

@@ -99,6 +99,7 @@ function Shutdown-Dumps()
     {
         $downloadedFile = [System.IO.Path]::GetTempFileName();
         $downloadedFile = "$downloadedFile.exe";
+        $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
         Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
         & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
     }
@@ -137,4 +138,4 @@ if ($Mode -eq "Shutdown")
     Shutdown-Dumps;
 }
 
-Exit 0;
+Exit 0;