Add additional signing configuration for external assemblies in the tools packages

.vsts-pipelines/builds/ci-internal.yml

@@ -0,0 +1,13 @@
+trigger:
+- master
+- release/*
+
+resources:
+  repositories:
+  - repository: buildtools
+    type: git
+    name: aspnet-BuildTools
+    ref: refs/heads/release/2.1
+
+phases:
+- template: .vsts-pipelines/templates/project-ci.yml@buildtools

.vsts-pipelines/builds/ci-public.yml

@@ -0,0 +1,15 @@
+trigger:
+- master
+- release/*
+
+# See https://github.com/aspnet/BuildTools
+resources:
+  repositories:
+  - repository: buildtools
+    type: github
+    endpoint: DotNet-Bot GitHub Connection
+    name: aspnet/BuildTools
+    ref: refs/heads/release/2.1
+
+phases:
+- template: .vsts-pipelines/templates/project-ci.yml@buildtools

Directory.Build.props

@@ -14,8 +14,6 @@
     <RepositoryRoot>$(MSBuildThisFileDirectory)</RepositoryRoot>
     <AssemblyOriginatorKeyFile>$(MSBuildThisFileDirectory)build\Key.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
-    <PublicSign Condition="'$(OS)' != 'Windows_NT'">true</PublicSign>
-    <AssemblySigningCertName>Microsoft</AssemblySigningCertName>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
 
     <!-- Binary compatiblity is not a goal for command-line tools. -->

build/VSIX.targets

@@ -41,6 +41,7 @@
       </ArtifactInfo>
 
       <FilesToSign Include="$(VSIXOutputPath)" Certificate="$(VsixSigningCertName)" IsContainer="true" />
+      <FilesToSign Include="$(RepositoryRoot)tooling/Microsoft.VisualStudio.SecretManager/bin/$(Configuration)/Microsoft.VisualStudio.SecretManager.dll" Certificate="$(AssemblySigningCertName)" />
       <FilesToExcludeFromSigning Include="$(VSIXManifestOutputPath)" />
       <FilesToExcludeFromSigning Include="$(VSIXSymbolsOutputPath)" />
     </ItemGroup>

run.ps1

@@ -26,12 +26,18 @@ The base url where build tools can be downloaded. Overrides the value from the c
 .PARAMETER Update
 Updates KoreBuild to the latest version even if a lock file is present.
 
+.PARAMETER Reinstall
+Re-installs KoreBuild
+
 .PARAMETER ConfigFile
 The path to the configuration file that stores values. Defaults to korebuild.json.
 
 .PARAMETER ToolsSourceSuffix
 The Suffix to append to the end of the ToolsSource. Useful for query strings in blob stores.
 
+.PARAMETER CI
+Sets up CI specific settings and variables.
+
 .PARAMETER Arguments
 Arguments to be passed to the command
 
@@ -46,8 +52,8 @@ in the file are overridden by command line parameters.
 Example config file:
 ```json
 {
-  "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/dev/tools/korebuild.schema.json",
+  "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/master/tools/korebuild.schema.json",
-  "channel": "dev",
+  "channel": "master",
   "toolsSource": "https://aspnetcore.blob.core.windows.net/buildtools"
 }
 ```
@@ -65,8 +71,10 @@ param(
     [string]$ToolsSource,
     [Alias('u')]
     [switch]$Update,
-    [string]$ConfigFile,
+    [switch]$Reinstall,
     [string]$ToolsSourceSuffix,
+    [string]$ConfigFile = $null,
+    [switch]$CI,
     [Parameter(ValueFromRemainingArguments = $true)]
     [string[]]$Arguments
 )
@@ -93,6 +101,10 @@ function Get-KoreBuild {
     $version = $version.TrimStart('version:').Trim()
     $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version)
 
+    if ($Reinstall -and (Test-Path $korebuildPath)) {
+        Remove-Item -Force -Recurse $korebuildPath
+    }
+
     if (!(Test-Path $korebuildPath)) {
         Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version"
         New-Item -ItemType Directory -Path $korebuildPath | Out-Null
@@ -101,9 +113,9 @@ function Get-KoreBuild {
         try {
             $tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip"
             Get-RemoteFile $remotePath $tmpfile $ToolsSourceSuffix
-            if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) {
+            if (Get-Command -Name 'Microsoft.PowerShell.Archive\Expand-Archive' -ErrorAction Ignore) {
                 # Use built-in commands where possible as they are cross-plat compatible
-                Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath
+                Microsoft.PowerShell.Archive\Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath
             }
             else {
                 # Fallback to old approach for old installations of PowerShell
@@ -167,19 +179,21 @@ if (Test-Path $ConfigFile) {
         }
     }
     catch {
-        Write-Warning "$ConfigFile could not be read. Its settings will be ignored."
+        Write-Host -ForegroundColor Red $Error[0]
-        Write-Warning $Error[0]
+        Write-Error "$ConfigFile contains invalid JSON."
+        exit 1
     }
 }
 
 if (!$DotNetHome) {
     $DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } `
+        elseif ($CI) { Join-Path $PSScriptRoot '.dotnet' } `
         elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} `
         elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}`
         else { Join-Path $PSScriptRoot '.dotnet'}
 }
 
-if (!$Channel) { $Channel = 'dev' }
+if (!$Channel) { $Channel = 'master' }
 if (!$ToolsSource) { $ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' }
 
 # Execute
@@ -188,7 +202,7 @@ $korebuildPath = Get-KoreBuild
 Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1')
 
 try {
-    Set-KoreBuildSettings -ToolsSource $ToolsSource -DotNetHome $DotNetHome -RepoPath $Path -ConfigFile $ConfigFile
+    Set-KoreBuildSettings -ToolsSource $ToolsSource -DotNetHome $DotNetHome -RepoPath $Path -ConfigFile $ConfigFile -CI:$CI
     Invoke-KoreBuildCommand $Command @Arguments
 }
 finally {

run.sh

@@ -11,13 +11,14 @@ RED="\033[0;31m"
 YELLOW="\033[0;33m"
 MAGENTA="\033[0;95m"
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-[ -z "${DOTNET_HOME:-}" ] && DOTNET_HOME="$HOME/.dotnet"
 verbose=false
 update=false
+reinstall=false
 repo_path="$DIR"
 channel=''
 tools_source=''
 tools_source_suffix=''
+ci=false
 
 #
 # Functions
@@ -38,6 +39,8 @@ __usage() {
     echo "    -s|--tools-source|-ToolsSource <URL>                  The base url where build tools can be downloaded. Overrides the value from the config file."
     echo "    --tools-source-suffix|-ToolsSourceSuffix <SUFFIX>     The suffix to append to tools-source. Useful for query strings."
     echo "    -u|--update                                           Update to the latest KoreBuild even if the lock file is present."
+    echo "    --reinstall                                           Reinstall KoreBuild."
+    echo "    --ci                                                  Apply CI specific settings and environment variables."
     echo ""
     echo "Description:"
     echo "    This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be."
@@ -62,6 +65,10 @@ get_korebuild() {
     version="$(echo "${version#version:}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
     local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version"
 
+    if [ "$reinstall" = true ] && [ -d "$korebuild_path" ]; then
+        rm -rf "$korebuild_path"
+    fi
+
     {
         if [ ! -d "$korebuild_path" ]; then
             mkdir -p "$korebuild_path"
@@ -175,6 +182,15 @@ while [[ $# -gt 0 ]]; do
         -u|--update|-Update)
             update=true
             ;;
+        --reinstall|-[Rr]einstall)
+            reinstall=true
+            ;;
+        --ci|-[Cc][Ii])
+            ci=true
+            if [[ -z "${DOTNET_HOME:-}" ]]; then
+                DOTNET_HOME="$DIR/.dotnet"
+            fi
+            ;;
         --verbose|-Verbose)
             verbose=true
             ;;
@@ -206,26 +222,38 @@ if [ -f "$config_file" ]; then
             config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")"
             config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")"
         else
-            __warn "$config_file is invalid JSON. Its settings will be ignored."
+            __error "$config_file contains invalid JSON."
+            exit 1
         fi
     elif __machine_has python ; then
         if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then
             config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")"
             config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")"
         else
-            __warn "$config_file is invalid JSON. Its settings will be ignored."
+            __error "$config_file contains invalid JSON."
+            exit 1
+        fi
+    elif __machine_has python3 ; then
+        if python3 -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then
+            config_channel="$(python3 -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")"
+            config_tools_source="$(python3 -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")"
+        else
+            __error "$config_file contains invalid JSON."
+            exit 1
         fi
     else
-        __warn 'Missing required command: jq or pyton. Could not parse the JSON file. Its settings will be ignored.'
+        __error 'Missing required command: jq or python. Could not parse the JSON file.'
+        exit 1
     fi
 
     [ ! -z "${config_channel:-}" ] && channel="$config_channel"
     [ ! -z "${config_tools_source:-}" ] && tools_source="$config_tools_source"
 fi
 
-[ -z "$channel" ] && channel='dev'
+[ -z "${DOTNET_HOME:-}" ] && DOTNET_HOME="$HOME/.dotnet"
+[ -z "$channel" ] && channel='master'
 [ -z "$tools_source" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools'
 
 get_korebuild
-set_korebuildsettings "$tools_source" "$DOTNET_HOME" "$repo_path" "$config_file"
+set_korebuildsettings "$tools_source" "$DOTNET_HOME" "$repo_path" "$config_file" "$ci"
 invoke_korebuild_command "$command" "$@"

src/dotnet-sql-cache/dotnet-sql-cache.csproj

@@ -19,4 +19,17 @@
     <PackageReference Include="System.Data.SqlClient" Version="$(SystemDataSqlClientPackageVersion)" />
   </ItemGroup>
 
+  <ItemGroup>
+    <!-- These files should be signed by corefx -->
+    <ExcludePackageFileFromSigning Include="$(PublishDir)runtimes/win-arm64/native/sni.dll" PackagePath="tools/$(TargetFramework)/any/runtimes/win-arm64/native/sni.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)runtimes/win-x64/native/sni.dll" PackagePath="tools/$(TargetFramework)/any/runtimes/win-x64/native/sni.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)runtimes/win-x86/native/sni.dll" PackagePath="tools/$(TargetFramework)/any/runtimes/win-x86/native/sni.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)System.Data.SqlClient.dll" PackagePath="tools/$(TargetFramework)/any/System.Data.SqlClient.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)runtimes/unix/lib/netcoreapp2.1/System.Data.SqlClient.dll" PackagePath="tools/$(TargetFramework)/any/runtimes/unix/lib/netcoreapp2.1/System.Data.SqlClient.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)runtimes/win/lib/netcoreapp2.1/System.Data.SqlClient.dll" PackagePath="tools/$(TargetFramework)/any/runtimes/win/lib/netcoreapp2.1/System.Data.SqlClient.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)System.Runtime.CompilerServices.Unsafe.dll" PackagePath="tools/$(TargetFramework)/any/System.Runtime.CompilerServices.Unsafe.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)System.Text.Encoding.CodePages.dll" PackagePath="tools/$(TargetFramework)/any/System.Text.Encoding.CodePages.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)runtimes/win/lib/netcoreapp2.0/System.Text.Encoding.CodePages.dll" PackagePath="tools/$(TargetFramework)/any/runtimes/win/lib/netcoreapp2.0/System.Text.Encoding.CodePages.dll" />
+  </ItemGroup>
+
 </Project>

src/dotnet-user-secrets/dotnet-user-secrets.csproj

@@ -22,4 +22,21 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="$(MicrosoftExtensionsConfigurationUserSecretsPackageVersion)" />
   </ItemGroup>
 
+  <ItemGroup>
+    <!-- 3rd party binary -->
+    <SignedPackageFile Include="$(PublishDir)Newtonsoft.Json.dll" PackagePath="tools/$(TargetFramework)/any/Newtonsoft.Json.dll" Certificate="$(AssemblySigning3rdPartyCertName)" />
+
+    <!-- Exclude files that should already be signed -->
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.Configuration.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.Configuration.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.Configuration.Abstractions.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.Configuration.Abstractions.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.Configuration.FileExtensions.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.Configuration.FileExtensions.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.Configuration.Json.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.Configuration.Json.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.Configuration.UserSecrets.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.Configuration.UserSecrets.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.FileProviders.Abstractions.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.FileProviders.Abstractions.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.FileProviders.Physical.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.FileProviders.Physical.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.FileSystemGlobbing.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.FileSystemGlobbing.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)Microsoft.Extensions.Primitives.dll" PackagePath="tools/$(TargetFramework)/any/Microsoft.Extensions.Primitives.dll" />
+    <ExcludePackageFileFromSigning Include="$(PublishDir)System.Runtime.CompilerServices.Unsafe.dll" PackagePath="tools/$(TargetFramework)/any/System.Runtime.CompilerServices.Unsafe.dll" />
+  </ItemGroup>
+
 </Project>