Move to BYOC queues (#8694)

* Move to BYOC queues - aspnet/AspNetCore-Internal#2033 - use conditions matching aspnet/Extensions and dotnet/Arcade - e.g. 05cb24592a/azure-pipelines.yml (L49-L54) - install necessary Build Tools for Visual Studio 2019 components and workloads on CI - revert part of 85ae18c723 because multiple editions of VS 2019 are now publicly available - that is, restore support for multiple editions in InstallVisualStudio.ps1 - update InstallVisualStudio.ps1 to - support BuildTools edition of VS 2019 nit: merge a couple of Windows build steps - support automatically updating latest version of a VS edition on the machine - include `-Quiet` option for a completely non-interactive installation * Install SQL Server 2017 Express LocalDB in Windows test job * Drop back to SQL Server 2016 Express LocalDB * Instead: Install SQL Server 2017 Express LocalDB and its cumulative update * Don't move Microsoft.AspNetCore.Blazor.Templates.dll into output directory - assembly is not of interest and should not be signed * Revert most of "Don't move Microsoft.AspNetCore.Blazor.Templates.dll into output directory" - didn't help odd attempt to sign this file and it was only a warning This reverts commit b55d69c370. * Back to using SQL Server 2016 Express LocalDB again - installing cumulative update was taking extra time and not reliably updating the LocalDB install - reverts part of 70d8d125f9, leaving direct download and some reordering * Do not assume vsjitdebuger.exe exists - component is not available in BuildTools edition of VS
2019-03-27 08:31:56 -07:00 · 2019-03-27 08:31:56 -07:00 · b068c9a62c
parent bc032624da f7440bed67
commit b068c9a62c
6 changed files with 137 additions and 44 deletions
--- a/.azure/pipelines/ci.yml
+++ b/.azure/pipelines/ci.yml
@ -295,6 +295,8 @@ jobs:
    buildScript: ./eng/scripts/cibuild.cmd
    buildArgs: -test "/p:SkipIISBackwardsCompatibilityTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISForwardsCompatibilityTests=true"
    beforeBuild:
+    - powershell: ./eng/scripts/InstallSqlServerLocalDB.ps1
+      displayName: Install SQL Server 2016 Express LocalDB
    - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
      displayName: Setup IISExpress test certificates and schema
    - powershell: "& ./.azure/pipelines/tools/SetupTestEnvironment.ps1 Setup signalrclienttests.exe"
@ -330,6 +332,7 @@ jobs:
    - name: MacOS_Test_Logs
      path: artifacts/logs/
      publishOnError: true
+
 - template: jobs/default-build.yml
  parameters:
    condition: ne(variables['SkipTests'], 'true')
--- a/.azure/pipelines/jobs/default-build.yml
+++ b/.azure/pipelines/jobs/default-build.yml
@ -6,7 +6,9 @@
 #   jobDisplayName: string
 #       The friendly job name to display in the UI. Defaults to the name of the OS.
 #   poolName: string
-#       The name of the VSTS agent queue to use.
+#       The name of the Azure DevOps agent pool to use.
+#   poolVmImage: string
+#       The name of a virtual machine image to use. Primarily of interest when using the Hosted pools.
 #   agentOs: string
 #       Used in templates to define variables which are OS specific. Typically from the set { Windows, Linux, macOS }
 #   buildArgs: string
@ -47,6 +49,7 @@
 parameters:
  agentOs: 'Windows'
  poolName: ''
+  poolVmImage: ''
  buildArgs: ''
  configuration: 'Release'
  beforeBuild: []
@ -91,10 +94,12 @@ jobs:
    ${{ if ne(parameters.poolName, '') }}:
      name: ${{ parameters.poolName }}
    ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Windows')) }}:
-      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: dotnet-internal-vs2019-preview
-      ${{ if ne(variables['System.TeamProject'], 'internal') }}:
-        name: dotnet-external-vs2019-preview
+      ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
+        name: NetCorePublic-Int-Pool
+        queue: BuildPool.Windows.10.Amd64.VS2019.BT.Open
+      ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        name: NetCoreInternal-Int-Pool
+        queue: BuildPool.Windows.10.Amd64.VS2019.BT
  variables:
    AgentOsName: ${{ parameters.agentOs }}
    ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
@ -115,20 +120,25 @@ jobs:
  steps:
  - checkout: self
    clean: true
+  - ${{ if eq(parameters.agentOs, 'Windows') }}:
+    - powershell: ./eng/scripts/InstallVisualStudio.ps1 -Edition BuildTools -Quiet
+      displayName: Install Build Tools for Visual Studio 2019
  - ${{ if eq(parameters.installNodeJs, 'true') }}:
    - task: NodeTool@0
      displayName: Install Node 10.x
      inputs:
        versionSpec: 10.x
  - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-    - powershell: ./eng/scripts/InstallJdk.ps1 '11.0.1'
+    - powershell: |
+        ./eng/scripts/InstallJdk.ps1 '11.0.1'
+        Write-Host "##vso[task.prependpath]$env:JAVA_HOME\bin"
+
      displayName: Install JDK 11
-    - powershell: Write-Host "##vso[task.prependpath]$env:JAVA_HOME\bin"
-      displayName: Prepend JAVA bin folder to the PATH.
-    - powershell: Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(BuildDirectory)\obj\selenium\"
-      displayName: Add Selenium process tracking folder environment variable
-    - powershell: ./eng/scripts/InstallGoogleChrome.ps1
-      displayName: Install chrome
+    - powershell: |
+        Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(BuildDirectory)\obj\selenium\"
+        ./eng/scripts/InstallGoogleChrome.ps1
+
+      displayName: Install Chrome
  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
    - task: MicroBuildSigningPlugin@1
      displayName: Install MicroBuild Signing plugin
--- a/.azure/pipelines/tools/SetupTestEnvironment.ps1
+++ b/.azure/pipelines/tools/SetupTestEnvironment.ps1
@ -77,7 +77,7 @@ function Setup-Dumps()

 function Shutdown-Dumps()
 {
-    Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe;
+    Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe -ErrorAction Ignore;

    Remove-Item $ldHive -Recurse -Force

--- a/eng/scripts/InstallSqlServerLocalDB.ps1
+++ b/eng/scripts/InstallSqlServerLocalDB.ps1
@ -0,0 +1,34 @@
+<#
+.SYNOPSIS
+    Installs SQL Server 2016 Express LocalDB on a machine.
+.DESCRIPTION
+    This script installs Microsoft SQL Server 2016 Express LocalDB on a machine.
+.LINK
+    https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/sql-server-2016-express-localdb?view=sql-server-2016
+    https://docs.microsoft.com/en-us/sql/database-engine/install-windows/install-sql-server-from-the-command-prompt?view=sql-server-2016
+#>
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+Set-StrictMode -Version 1
+
+$intermedateDir = "$PSScriptRoot\obj"
+mkdir $intermedateDir -ErrorAction Ignore | Out-Null
+
+Write-Host "Installing SQL Server 2016 Express LocalDB" -f Magenta
+
+# Download SqlLocalDB.msi.
+$installerFilename = "SqlLocalDB.msi"
+$installerPath = "$intermedateDir\$installerFilename"
+Write-Host ""
+Write-Host "Downloading '$installerFilename' to '$installerPath'."
+Invoke-WebRequest -OutFile $installerPath -Uri `
+  "https://download.microsoft.com/download/9/0/7/907AD35F-9F9C-43A5-9789-52470555DB90/ENU/SqlLocalDB.msi"
+
+# Install LocalDB.
+$arguments = '/package', "`"$installerPath`"", '/NoRestart', '/Passive', `
+  'IACCEPTSQLLOCALDBLICENSETERMS=YES', 'HIDEPROGRESSBAR=YES'
+Write-Host ""
+Write-Host "Running 'msiexec $arguments'."
+$process = Start-Process msiexec.exe -ArgumentList $arguments -NoNewWindow -PassThru -Verbose -Wait
+exit $process.ExitCode
--- a/eng/scripts/InstallVisualStudio.ps1
+++ b/eng/scripts/InstallVisualStudio.ps1
@ -3,46 +3,40 @@
    Installs or updates Visual Studio on a local developer machine.
 .DESCRIPTION
    This installs Visual Studio along with all the workloads required to contribute to this repository.
-
+.PARAMETER Edition
+    Selects which 'offering' of Visual Studio to install. Must be one of these values:
+        BuildTools
+        Community
+        Professional
+        Enterprise (the default)
 .PARAMETER InstallPath
-    The location of Visual Studio
+    The location on disk where Visual Studio should be installed or updated. Default path is location of latest
+    existing installation of the specified edition, if any. If that VS edition is not currently installed, default
+    path is '${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\`$Edition".
 .PARAMETER Passive
    Run the installer without requiring interaction.
+.PARAMETER Quiet
+    Run the installer without UI and wait for installation to complete.
 .LINK
    https://visualstudio.com
    https://github.com/aspnet/AspNetCore/blob/master/docs/BuildFromSource.md
 .EXAMPLE
-    To install VS 2019 Preview, run this command in PowerShell:
+    To install VS 2019 Enterprise, run this command in PowerShell:

        .\InstallVisualStudio.ps1
 #>
 [CmdletBinding(DefaultParameterSetName = 'Default')]
 param(
-    # TODO - once VS 2019 16.0 RTM is released, make this a parameter again
-    # .PARAMETER Edition
-    # Must be one of these values:
-
-    # Community
-    # Professional
-    # Enterprise
-
-    # Selects which 'offering' of Visual Studio to install.
-
-    # [ValidateSet('Community', 'Professional', 'Enterprise')]
-    # [string]$Edition,
+    [ValidateSet('BuildTools','Community', 'Professional', 'Enterprise')]
+    [string]$Edition = 'Enterprise',
    [string]$InstallPath,
-    [switch]$Passive
+    [switch]$Passive,
+    [switch]$Quiet
 )

-# VS previews are only available publicly as 'Enterprise' versions. They should be available to the community to use without a paid license.
-$Edition = 'Enterprise'
-
-if (-not $Edition) {
-    Write-Host "You must specify a value for the -Edition parameter which selects the kind of Visual Studio to install." -f Red
+if ($Passive -and $Quiet) {
+    Write-Host "The -Passive and -Quiet options cannot be used together." -f Red
    Write-Host "Run ``Get-Help $PSCommandPath`` for more details." -f Red
-    Write-Host ""
-    Write-Host "Example:  ./InstallVisualStudio -Edition Community" -f Red
-    Write-Host ""
    exit 1
 }

@ -54,35 +48,57 @@ mkdir $intermedateDir -ErrorAction Ignore | Out-Null

 $bootstrapper = "$intermedateDir\vsinstaller.exe"
 $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
-Invoke-WebRequest -Uri "https://aka.ms/vs/16/pre/vs_$($Edition.ToLowerInvariant()).exe" -OutFile $bootstrapper
+Invoke-WebRequest -Uri "https://aka.ms/vs/16/release/vs_$($Edition.ToLowerInvariant()).exe" -OutFile $bootstrapper
+
+$productId = "Microsoft.VisualStudio.Product.$Edition"
+if (-not $InstallPath) {
+    $vsWhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
+    if (Test-Path $vsWhere)
+    {
+        $InstallPath = &$vsWhere -version '[16,17)' -latest -prerelease -products $productId -property installationPath
+    }
+}

 if (-not $InstallPath) {
-    # $InstallPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\$Edition"
-    $InstallPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\Preview"
+    $InstallPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\$Edition"
 }

 # no backslashes - this breaks the installer
 $InstallPath = $InstallPath.TrimEnd('\')

 [string[]] $arguments = @()
-
 if (Test-path $InstallPath) {
    $arguments += 'modify'
 }

+$responseFile = "$PSScriptRoot\vs.json"
+if ("$Edition" -eq "BuildTools") {
+    $responseFile = "$PSScriptRoot\vs.buildtools.json"
+}
+
 $arguments += `
-    '--productId', "Microsoft.VisualStudio.Product.$Edition", `
+    '--productId', $productId, `
    '--installPath', "`"$InstallPath`"", `
-    '--in', "$PSScriptRoot\vs.json", `
+    '--in', "`"$responseFile`"", `
    '--norestart'

 if ($Passive) {
    $arguments += '--passive'
 }
+if ($Quiet) {
+    $arguments += '--quiet', '--wait'
+}

 Write-Host ""
 Write-Host "Installing Visual Studio 2019 $Edition" -f Magenta
 Write-Host ""
 Write-Host "Running '$bootstrapper $arguments'"

-& $bootstrapper @arguments
+$process = Start-Process -FilePath "$bootstrapper" -ArgumentList $arguments `
+    -PassThru -RedirectStandardError "$intermedateDir\errors.txt" -Verbose -Wait
+if ($process.ExitCode -ne 0) {
+    Get-Content "$intermedateDir\errors.txt" | Write-Error
+}
+
+Remove-Item "$intermedateDir\errors.txt" -errorAction SilentlyContinue
+exit $process.ExitCode
--- a/eng/scripts/vs.buildtools.json
+++ b/eng/scripts/vs.buildtools.json
@ -0,0 +1,30 @@
+{
+    "channelUri": "https://aka.ms/vs/16/pre/channel",
+    "channelId": "VisualStudio.16.Preview",
+    "includeRecommended": false,
+    "addProductLang": [
+        "en-US"
+    ],
+    "add": [
+        "Microsoft.Net.Component.4.6.1.TargetingPack",
+        "Microsoft.Net.Component.4.6.2.TargetingPack",
+        "Microsoft.Net.Component.4.7.1.TargetingPack",
+        "Microsoft.Net.Component.4.7.2.SDK",
+        "Microsoft.Net.Component.4.7.2.TargetingPack",
+        "Microsoft.Net.Component.4.7.TargetingPack",
+        "Microsoft.VisualStudio.Component.FSharp.MSBuild",
+        "Microsoft.VisualStudio.Component.NuGet",
+        "Microsoft.VisualStudio.Component.NuGet.BuildTools",
+        "Microsoft.VisualStudio.Component.VC.ATL",
+        "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+        "Microsoft.VisualStudio.Component.VC.v141.ATL",
+        "Microsoft.VisualStudio.Component.VC.v141.x86.x64",
+        "Microsoft.VisualStudio.Component.Windows10SDK.17134",
+        "Microsoft.VisualStudio.Workload.ManagedDesktopBuildTools",
+        "Microsoft.VisualStudio.Workload.MSBuildTools",
+        "Microsoft.VisualStudio.Workload.NetCoreBuildTools",
+        "Microsoft.VisualStudio.Workload.VCTools",
+        "Microsoft.VisualStudio.Workload.VisualStudioExtensionBuildTools",
+        "Microsoft.VisualStudio.Workload.WebBuildTools"
+    ]
+}