[release/3.1] Fix the badness and get this branch building (#14594)

.azure/pipelines/jobs/default-build.yml

@@ -55,6 +55,7 @@ parameters:
   artifacts:  []
   buildDirectory: ''
   buildScript: ''
+  installTar: true
   installNodeJs: true
   installJdk: true
   timeoutInMinutes: 180
@@ -151,6 +152,9 @@ jobs:
             Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(BuildDirectory)\artifacts\tmp\selenium\"
             ./eng/scripts/InstallGoogleChrome.ps1
           displayName: Install Chrome
+    - ${{ if and(eq(parameters.installTar, 'true'), eq(parameters.agentOs, 'Windows')) }}:
+      - powershell: ./eng/scripts/InstallTar.ps1
+        displayName: Find or install Tar
 
     - ${{ parameters.beforeBuild }}
 

build.ps1

@@ -307,6 +307,8 @@ if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJ
 # Initialize global variables need to be set before the import of Arcade is imported
 $restore = $RunRestore
 
+# Though VS Code may indicate $nodeReuse, $warnAsError and $msbuildEngine are unused, tools.ps1 uses them.
+
 # Disable node reuse - Workaround perpetual issues in node reuse and custom task assemblies
 $nodeReuse = $false
 $env:MSBUILDDISABLENODEREUSE=1
@@ -328,10 +330,10 @@ if ($CI) {
 }
 
 # tools.ps1 corrupts global state, so reset these values in case they carried over from a previous build
-rm variable:global:_BuildTool -ea Ignore
-rm variable:global:_DotNetInstallDir -ea Ignore
-rm variable:global:_ToolsetBuildProj -ea Ignore
-rm variable:global:_MSBuildExe -ea Ignore
+Remove-Item variable:global:_BuildTool -ea Ignore
+Remove-Item variable:global:_DotNetInstallDir -ea Ignore
+Remove-Item variable:global:_ToolsetBuildProj -ea Ignore
+Remove-Item variable:global:_MSBuildExe -ea Ignore
 
 # Import Arcade
 . "$PSScriptRoot/eng/common/tools.ps1"
@@ -391,10 +393,10 @@ finally {
     }
 
     # tools.ps1 corrupts global state, so reset these values so they don't carry between invocations of build.ps1
-    rm variable:global:_BuildTool -ea Ignore
-    rm variable:global:_DotNetInstallDir -ea Ignore
-    rm variable:global:_ToolsetBuildProj -ea Ignore
-    rm variable:global:_MSBuildExe -ea Ignore
+    Remove-Item variable:global:_BuildTool -ea Ignore
+    Remove-Item variable:global:_DotNetInstallDir -ea Ignore
+    Remove-Item variable:global:_ToolsetBuildProj -ea Ignore
+    Remove-Item variable:global:_MSBuildExe -ea Ignore
 
     if ($DumpProcesses -or $ci) {
         Stop-Job -Name DumpProcesses

eng/scripts/InstallTar.ps1

@@ -0,0 +1,74 @@
+<#
+.SYNOPSIS
+    Finds or installs the Tar command on this system.
+.DESCRIPTION
+    This script searches for Tar on this system. If not found, downloads and extracts Git to use its tar.exe. Prefers
+    global installation locations even if Git has been downloaded into this repo.
+.PARAMETER GitVersion
+    The version of the Git to install. If not set, the default value is read from global.json.
+.PARAMETER Force
+    Overwrite the existing installation if one exists in this repo and Tar isn't installed globally.
+#>
+param(
+    [string]$GitVersion,
+    [switch]$Force
+)
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+
+Set-StrictMode -Version 1
+
+# Find tar. If not found, install Git to get it.
+$repoRoot = (Join-Path $PSScriptRoot "..\.." -Resolve)
+$installDir = "$repoRoot\.tools\Git\win-x64"
+$tarCommand = "$installDir\usr\bin\tar.exe"
+$finalCommand = "$repoRoot\.tools\tar.exe"
+
+Write-Host "Windows version and other information..."
+cmd.exe /c ver
+systeminfo.exe
+Write-Host "Processor Architecture: $env:PROCESSOR_ARCHITECTURE"
+
+Write-Host "Checking $env:SystemRoot\System32\tar.exe"
+Get-ChildItem "$env:SystemRoot\System32\ta*.exe"
+if (Test-Path "$env:SystemRoot\System32\tar.exe") {
+    Write-Host "Found $env:SystemRoot\System32\tar.exe"
+    $tarCommand = "$env:SystemRoot\System32\tar.exe"
+}
+elseif (Test-Path "$env:ProgramFiles\Git\usr\bin\tar.exe") {
+    $tarCommand = "$env:ProgramFiles\Git\usr\bin\tar.exe"
+}
+elseif (Test-Path "${env:ProgramFiles(x86)}\Git\usr\bin\tar.exe") {
+    $tarCommand = "${env:ProgramFiles(x86)}\Git\usr\bin\tar.exe"
+}
+elseif (Test-Path "$env:AGENT_HOMEDIRECTORY\externals\git\usr\bin\tar.exe") {
+    $tarCommand = "$env:AGENT_HOMEDIRECTORY\externals\git\usr\bin\tar.exe"
+}
+elseif ((Test-Path $tarCommand) -And (-Not $Force)) {
+    Write-Verbose "Repo-local Git installation and $tarCommand already exist, skipping Git install."
+}
+else {
+    if (-not $GitVersion) {
+        $globalJson = Get-Content "$repoRoot\global.json" | ConvertFrom-Json
+        $GitVersion = $globalJson.tools.Git
+    }
+
+    $Uri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/git/Git-${GitVersion}-64-bit.zip"
+
+    Import-Module -Name (Join-Path $PSScriptRoot "..\common\native\CommonLibrary.psm1" -Resolve)
+    $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri -InstallDirectory "$installDir\" -Force:$Force -Verbose
+
+    if ($InstallStatus -Eq $False) {
+        Write-Error "Installation failed"
+        exit 1
+    }
+}
+
+New-Item "$repoRoot\.tools\" -ErrorAction SilentlyContinue -ItemType Directory
+Copy-Item "$tarCommand" "$finalCommand" -Verbose
+Write-Host "Tar now available at '$finalCommand'"
+
+if ($tarCommand -like '*\Git\*') {
+    $null >.\.tools\tar.fromGit
+}

eng/targets/ReferenceAssembly.targets

@@ -61,15 +61,21 @@
     </ItemGroup>
 
     <PropertyGroup>
+      <_GenApiFile>$([MSBuild]::NormalizePath('$(ArtifactsDir)', 'log', 'GenAPI.rsp'))</_GenApiFile>
       <_GenAPICommand Condition="'$(MSBuildRuntimeType)' == 'core'">"$(DotNetTool)" --roll-forward-on-no-candidate-fx 2 "$(_GenAPIPath)"</_GenAPICommand>
-      <_GenAPICmd>$(_GenAPICommand)</_GenAPICmd>
-      <_GenAPICmd>$(_GenAPICmd) "$(TargetPath)"</_GenAPICmd>
-      <_GenAPICmd>$(_GenAPICmd) --lib-path "@(_ReferencePathDirectories)"</_GenAPICmd>
-      <_GenAPICmd>$(_GenAPICmd) --out "$(_RefSourceFileOutputPath)"</_GenAPICmd>
-      <_GenAPICmd>$(_GenAPICmd) --header-file "$(RepoRoot)/eng/LicenseHeader.txt"</_GenAPICmd>
-      <_GenAPICmd>$(_GenAPICmd) --exclude-api-list "$(RepoRoot)/eng/GenAPI.exclusions.txt"</_GenAPICmd>
+      <_GenAPICmd>$(_GenAPICommand) @"$(_GenApiFile)"</_GenAPICmd>
+      <_GenApiArguments><![CDATA[
+"$(TargetPath)"
+--lib-path "@(_ReferencePathDirectories, '%3B')"
+--out "$(_RefSourceFileOutputPath)"
+--header-file "$(RepoRoot)/eng/LicenseHeader.txt"
+--exclude-api-list "$(RepoRoot)/eng/GenAPI.exclusions.txt"
+]]>
+      </_GenApiArguments>
     </PropertyGroup>
 
+    <WriteLinesToFile File="$(_GenApiFile)" Lines="$(_GenApiArguments)" Overwrite="true" />
+
     <MakeDir Directories="$(_RefSourceOutputPath)" />
     <Message Importance="High" Text="Generating $(_RefSourceFileOutputPath)" />
     <Exec Command="$(_GenAPICmd)" />
@@ -96,4 +102,4 @@
     </ItemGroup>
   </Target>
 
-</Project>
+</Project>

global.json

@@ -12,6 +12,7 @@
         "$(MicrosoftNETCoreAppRuntimeVersion)"
       ]
     },
+    "Git": "2.22.0",
     "jdk": "11.0.3",
     "vs": {
       "version": "16.0",

src/Framework/ref/Microsoft.AspNetCore.App.Ref.csproj

@@ -171,14 +171,26 @@ This package is an internal implementation of the .NET Core SDK and is not meant
           Inputs="@(RefPackContent)"
           Outputs="$(ZipArchiveOutputPath);$(TarArchiveOutputPath)"
           Condition="'$(IsPackable)' == 'true'">
+    <PropertyGroup>
+      <_TarCommand>tar</_TarCommand>
+      <_TarCommand Condition="Exists('$(RepoRoot).tools\tar.exe')">$(RepoRoot).tools\tar.exe</_TarCommand>
+
+      <!-- For the tar packed with git, transform e.g. "C:\root\AspNetCore\File.tar.gz" to "/C/root/AspNetCore/File.tar.gz". -->
+      <_TarArchiveOutputPath>$(TarArchiveOutputPath)</_TarArchiveOutputPath>
+      <_TarArchiveOutputPath
+          Condition="Exists('$(repoRoot)\.tools\tar.fromGit')">/$(TarArchiveOutputPath.Replace('\','/').Replace(':',''))</_TarArchiveOutputPath>
+    </PropertyGroup>
+
     <ZipDirectory
       SourceDirectory="$(TargetingPackLayoutRoot)"
       DestinationFile="$(ZipArchiveOutputPath)"
       Overwrite="true" />
+
     <!-- Requires Windows 10 version 1803 or newer -->
-    <Exec
-      Command="tar -czf $(TarArchiveOutputPath) ."
-      WorkingDirectory="$(TargetingPackLayoutRoot)" />
+    <Message Importance="High" Text="Executing: $(_TarCommand) -czf $(_TarArchiveOutputPath) ." />
+    <Exec Command="$(_TarCommand) -czf $(_TarArchiveOutputPath) ."
+        WorkingDirectory="$(TargetingPackLayoutRoot)" />
+
     <Message Importance="High" Text="$(MSBuildProjectName) -> $(TarArchiveOutputPath)" />
   </Target>