From 86c03c3025f5fb6af27720770c1dc3492f636f57 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2019 13:39:31 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20190830.3 (#13621) - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19430.3 - Microsoft.DotNet.GenAPI - 1.0.0-beta.19430.3 - Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19430.3 --- eng/Version.Details.xml | 12 +++++------ eng/Versions.props | 2 +- eng/common/darc-init.ps1 | 4 ++-- eng/common/darc-init.sh | 6 +++++- eng/common/native/CommonLibrary.psm1 | 2 ++ eng/common/performance/perfhelixpublish.proj | 8 ++++---- eng/common/post-build/symbols-validation.ps1 | 20 +++++++++---------- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 ++-- eng/common/sdl/init-sdl.ps1 | 3 +++ eng/common/sdl/packages.config | 4 ++-- eng/common/sdl/run-sdl.ps1 | 10 +++++----- eng/common/templates/job/execute-sdl.yml | 15 ++++++++------ .../templates/post-build/common-variables.yml | 4 ++-- .../templates/post-build/post-build.yml | 6 ++++-- eng/common/tools.ps1 | 2 ++ eng/common/tools.sh | 15 ++++++++++---- global.json | 4 ++-- 17 files changed, 72 insertions(+), 49 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7e7a084e0b..4d56f307ea 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -412,17 +412,17 @@ https://github.com/aspnet/Extensions 77525442b7d79d608935676b6ddbe653c7dc6d61 - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 316c80d0c373be63f991cc4d586db85273c1c553 - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 316c80d0c373be63f991cc4d586db85273c1c553 - + https://github.com/dotnet/arcade - 2de3acc671fc624191672a45564f9ef130af5cd4 + 316c80d0c373be63f991cc4d586db85273c1c553 https://github.com/aspnet/Extensions diff --git a/eng/Versions.props b/eng/Versions.props index c75ef3a72b..b438098cd3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -53,7 +53,7 @@ --> - 1.0.0-beta.19425.1 + 1.0.0-beta.19430.3 3.3.1-beta3-19430-03 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 8854d979f3..46d175fdfd 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,9 +1,9 @@ param ( $darcVersion = $null, - $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16", + $verbosity = "m" ) -$verbosity = "m" . $PSScriptRoot\tools.ps1 function InstallDarcCli ($darcVersion) { diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index abdd0bc05a..242429bca6 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -3,6 +3,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" +verbosity=m while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -15,6 +16,10 @@ while [[ $# > 0 ]]; do versionEndpoint=$2 shift ;; + --verbosity) + verbosity=$2 + shift + ;; *) echo "Invalid argument: $1" usage @@ -34,7 +39,6 @@ while [[ -h "$source" ]]; do [[ $source != /* ]] && source="$scriptroot/$source" done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -verbosity=m . "$scriptroot/tools.sh" diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 2a08d5246e..41416862d9 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -152,6 +152,8 @@ function Get-File { } else { Write-Verbose "Downloading $Uri" + # Don't display the console progress UI - it's a huge perf hit + $ProgressPreference = 'SilentlyContinue' while($Attempt -Lt $DownloadRetries) { try { diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index d07b1fa6e2..e5826b5323 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -83,8 +83,8 @@ $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(BaselineArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) 4:00 @@ -93,8 +93,8 @@ $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(ArtifactsDirectory)" - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) 4:00 diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index d5ec51b150..096ac321d1 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -37,10 +37,10 @@ function FirstMatchingSymbolDescriptionOrDefault { # DWARF file for a .dylib $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null if (Test-Path $PdbPath) { return "PDB" @@ -159,25 +159,25 @@ function CheckSymbolsAvailable { } } -function Installdotnetsymbol { - $dotnetsymbolPackageName = "dotnet-symbol" +function InstallDotnetSymbol { + $dotnetSymbolPackageName = "dotnet-symbol" $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" $toolList = & "$dotnet" tool list --global - if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." } else { - Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global + & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global } } try { - Installdotnetsymbol + InstallDotnetSymbol CheckSymbolsAvailable } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 6d9bdcf72b..fa239484f2 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -53,8 +53,8 @@ if ($ValidPath -eq $False) exit 1 } -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel -$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory (Split-Path $SourceDirectory -Parent) -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path (Split-Path $SourceDirectory -Parent) ".gdn" if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 26e01c0673..c737eb0e71 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -11,6 +11,9 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 $LASTEXITCODE = 0 +# Don't display the console progress UI - it's a huge perf hit +$ProgressPreference = 'SilentlyContinue' + # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3f97ac2f16..256ffbfb93 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + - + diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index b90c4399ba..9bc25314ae 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -32,16 +32,16 @@ foreach ($tool in $ToolsList) { Write-Host $tool # We have to manually configure tools that run on source to look at the source directory only if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } } if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -56,4 +56,4 @@ Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --bas if ($LASTEXITCODE -ne 0) { Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." exit $LASTEXITCODE -} \ No newline at end of file +} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 91621cf88f..a7f9964195 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,7 +1,10 @@ parameters: overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - continueOnError: false # optional: determines whether to continue the build if the step errors; + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; dependsOn: '' # Optional: dependencies of the job jobs: @@ -26,12 +29,12 @@ jobs: -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -45,12 +48,12 @@ jobs: - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b00d85d8ce..7b3fdb1361 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -49,6 +49,6 @@ variables: # Default locations for Installers and checksums - name: ChecksumsBlobFeedUrl - value: https://dotnetcli.blob.core.windows.net/dotnet/index.json - - name: InstallersBlobFeedUrl value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json + - name: InstallersBlobFeedUrl + value: https://dotnetcli.blob.core.windows.net/dotnet/index.json diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3f239fae2b..34667b6c09 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,11 +1,12 @@ parameters: - enableSourceLinkValidation: true + enableSourceLinkValidation: false enableSigningValidation: true - enableSymbolValidation: true + enableSymbolValidation: false enableNugetValidation: true publishInstallersAndChecksums: false SDLValidationParameters: enable: false + continueOnError: false params: '' # These parameters let the user customize the call to sdk-task.ps1 for publishing @@ -92,6 +93,7 @@ stages: - template: /eng/common/templates/job/execute-sdl.yml parameters: additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - template: \eng\common\templates\post-build\channels\netcore-dev-5.yml parameters: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9c12b1b4fd..bb5638930a 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,6 +163,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript } @@ -282,6 +283,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath Unzip $packagePath $packageDir } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3af9be6157..94a1edd7d0 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -208,12 +208,19 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" - else - wget -q -O "$install_script" "$install_script_url" + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + else + wget -q -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } fi fi - # return value _GetDotNetInstallScript="$install_script" } diff --git a/global.json b/global.json index 4ead16dbcf..4a0dc91d0b 100644 --- a/global.json +++ b/global.json @@ -24,7 +24,7 @@ }, "msbuild-sdks": { "Yarn.MSBuild": "1.15.2", - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19425.1", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19425.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19430.3", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19430.3" } }