From 0dde7b1a270beda4feaa4200fb00c86c438516f0 Mon Sep 17 00:00:00 2001 From: John Luo Date: Thu, 31 Aug 2017 15:48:19 -0700 Subject: [PATCH 001/115] Patch 2.0.1 --- build.ps1 | 2 +- build.sh | 2 +- build/Repositories.props | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build.ps1 b/build.ps1 index b2d4a38062..4772c19efa 100644 --- a/build.ps1 +++ b/build.ps1 @@ -32,7 +32,7 @@ cd $PSScriptRoot $repoFolder = $PSScriptRoot $env:REPO_FOLDER = $repoFolder -$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.0.zip" +$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.1.zip" if ($env:KOREBUILD_ZIP) { $koreBuildZip=$env:KOREBUILD_ZIP diff --git a/build.sh b/build.sh index 5e27ed8efb..487eeb81f6 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $repoFolder -koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.0.zip" +koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.1.zip" if [ ! -z $KOREBUILD_ZIP ]; then koreBuildZip=$KOREBUILD_ZIP fi diff --git a/build/Repositories.props b/build/Repositories.props index 977704223c..abe38bd873 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -1,7 +1,7 @@ - rel/2.0.0 + patch/2.0.1 @@ -32,7 +32,9 @@ - + + rel/2.0.1 + From ac2fe035c2b376c89baa458835a18fb4f024c812 Mon Sep 17 00:00:00 2001 From: John Luo Date: Thu, 31 Aug 2017 16:33:16 -0700 Subject: [PATCH 002/115] EF rename --- build/Repositories.props | 47 +--------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index abe38bd873..014e28146d 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -6,51 +6,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - rel/2.0.1 - - - - - - - - - - - - - - - - - - + From 31df733119e89d95e68fa9e7986e2a23558916e2 Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 1 Sep 2017 13:47:17 -0700 Subject: [PATCH 003/115] Revert WIP commit --- build/Repositories.props | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/build/Repositories.props b/build/Repositories.props index 014e28146d..04c3abb98c 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -6,6 +6,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + rel/2.0.1 + + + + + + + + + + + + + + + + + + From fda1730a7e85dde68571cce92e7f340d46cf0dd8 Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 1 Sep 2017 14:16:19 -0700 Subject: [PATCH 004/115] Revert --- build/Repositories.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Repositories.props b/build/Repositories.props index 04c3abb98c..abe38bd873 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -18,7 +18,7 @@ - + From fdf8ace3534b5b6e60156325a139740c4d45a4dc Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 1 Sep 2017 17:12:06 -0700 Subject: [PATCH 005/115] Temporarily remove SignalR from build --- build/Repositories.props | 1 - 1 file changed, 1 deletion(-) diff --git a/build/Repositories.props b/build/Repositories.props index abe38bd873..bcf74984f4 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -48,7 +48,6 @@ - From 9895b712270957db89d402099a77d595e285da58 Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 5 Sep 2017 10:30:20 -0700 Subject: [PATCH 006/115] Remove vcs overrides --- build/repo.targets | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index df2b9e68b7..8b0782c1b0 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -180,16 +180,6 @@ - - - <_RepositoryName>$([System.String]::new('%(Repository.Identity)').Replace('.', '_')) - <_CloneUrl>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_URL_$(_RepositoryName)")) - <_CommitHash>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_NUMBER_$(_RepositoryName)")) - From 1fef45db671d3f64670913c6a9d865410adbd497 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 14 Sep 2017 14:10:41 -0700 Subject: [PATCH 007/115] Start preparing list of repositories that need patching (#556) --- build/Repositories.props | 102 +++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index bcf74984f4..6ec2904fe3 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -1,55 +1,55 @@ - - - patch/2.0.1 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - rel/2.0.1 - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 9c0eea3788729e6df2ff0d9821583d2c96680627 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 14 Sep 2017 14:40:03 -0700 Subject: [PATCH 008/115] Update the bootstrapper to use compiled KoreBuild --- .gitignore | 3 +- build.cmd | 2 +- build.ps1 | 217 ++++++++++++++++++++++++++++++++----------- build.sh | 223 +++++++++++++++++++++++++++++++++++++-------- korebuild-lock.txt | 2 + 5 files changed, 355 insertions(+), 92 deletions(-) create mode 100644 korebuild-lock.txt diff --git a/.gitignore b/.gitignore index 0a7eb1e4f8..f834920dd1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ node_modules .nuget .r .deps -global.json \ No newline at end of file +global.json +korebuild-lock.txt diff --git a/build.cmd b/build.cmd index 7d4894cb4a..b6c8d24864 100644 --- a/build.cmd +++ b/build.cmd @@ -1,2 +1,2 @@ @ECHO OFF -PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' %*; exit $LASTEXITCODE" \ No newline at end of file +PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' %*; exit $LASTEXITCODE" diff --git a/build.ps1 b/build.ps1 index 4772c19efa..aef29dac05 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,66 +1,177 @@ -$ErrorActionPreference = "Stop" +#!/usr/bin/env powershell +#requires -version 4 -function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries) -{ - while($true) - { - try - { - Invoke-WebRequest $url -OutFile $downloadLocation - break - } - catch - { - $exceptionMessage = $_.Exception.Message - Write-Host "Failed to download '$url': $exceptionMessage" - if ($retries -gt 0) { - $retries-- - Write-Host "Waiting 10 seconds before retrying. Retries left: $retries" - Start-Sleep -Seconds 10 +<# +.SYNOPSIS +Build this repository +.DESCRIPTION +Downloads korebuild if required. Then builds the repository. + +.PARAMETER Path +The folder to build. Defaults to the folder containing this script. + +.PARAMETER Channel +The channel of KoreBuild to download. Overrides the value from the config file. + +.PARAMETER DotNetHome +The directory where .NET Core tools will be stored. + +.PARAMETER ToolsSource +The base url where build tools can be downloaded. Overrides the value from the config file. + +.PARAMETER Update +Updates KoreBuild to the latest version even if a lock file is present. + +.PARAMETER ConfigFile +The path to the configuration file that stores values. Defaults to version.props. + +.PARAMETER MSBuildArgs +Arguments to be passed to MSBuild + +.NOTES +This function will create a file $PSScriptRoot/korebuild-lock.txt. This lock file can be committed to source, but does not have to be. +When the lockfile is not present, KoreBuild will create one using latest available version from $Channel. + +The $ConfigFile is expected to be an XML file. It is optional, and the configuration values in it are optional as well. + +.EXAMPLE +Example config file: +```xml + + + + dev + https://aspnetcore.blob.core.windows.net/buildtools + + +``` +#> +[CmdletBinding(PositionalBinding = $false)] +param( + [string]$Path = $PSScriptRoot, + [Alias('c')] + [string]$Channel, + [Alias('d')] + [string]$DotNetHome, + [Alias('s')] + [string]$ToolsSource, + [Alias('u')] + [switch]$Update, + [string]$ConfigFile = (Join-Path $PSScriptRoot 'version.props'), + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$MSBuildArgs +) + +Set-StrictMode -Version 2 +$ErrorActionPreference = 'Stop' + +# +# Functions +# + +function Get-KoreBuild { + + $lockFile = Join-Path $Path 'korebuild-lock.txt' + + if (!(Test-Path $lockFile) -or $Update) { + Get-RemoteFile "$ToolsSource/korebuild/channels/$Channel/latest.txt" $lockFile + } + + $version = Get-Content $lockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1 + if (!$version) { + Write-Error "Failed to parse version from $lockFile. Expected a line that begins with 'version:'" + } + $version = $version.TrimStart('version:').Trim() + $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version) + + if (!(Test-Path $korebuildPath)) { + Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version" + New-Item -ItemType Directory -Path $korebuildPath | Out-Null + $remotePath = "$ToolsSource/korebuild/artifacts/$version/korebuild.$version.zip" + + try { + $tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip" + Get-RemoteFile $remotePath $tmpfile + if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) { + # Use built-in commands where possible as they are cross-plat compatible + Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath } - else - { - $exception = $_.Exception - throw $exception + else { + # Fallback to old approach for old installations of PowerShell + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($tmpfile, $korebuildPath) } } + catch { + remove-item -Recurse -Force $korebuildPath -ErrorAction Ignore + throw + } + finally { + remove-item $tmpfile -ErrorAction Ignore + } } + + return $korebuildPath } -cd $PSScriptRoot - -$repoFolder = $PSScriptRoot -$env:REPO_FOLDER = $repoFolder -$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.1.zip" -if ($env:KOREBUILD_ZIP) -{ - $koreBuildZip=$env:KOREBUILD_ZIP +function Join-Paths([string]$path, [string[]]$childPaths) { + $childPaths | ForEach-Object { $path = Join-Path $path $_ } + return $path } -$buildFolder = ".build" -$buildFile="$buildFolder\KoreBuild.ps1" - -if (!(Test-Path $buildFolder)) { - Write-Host "Downloading KoreBuild from $koreBuildZip" - - $tempFolder=$env:TEMP + "\KoreBuild-" + [guid]::NewGuid() - New-Item -Path "$tempFolder" -Type directory | Out-Null - - $localZipFile="$tempFolder\korebuild.zip" - - DownloadWithRetry -url $koreBuildZip -downloadLocation $localZipFile -retries 6 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($localZipFile, $tempFolder) - - New-Item -Path "$buildFolder" -Type directory | Out-Null - copy-item "$tempFolder\**\build\*" $buildFolder -Recurse - - # Cleanup - if (Test-Path $tempFolder) { - Remove-Item -Recurse -Force $tempFolder +function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) { + if ($RemotePath -notlike 'http*') { + Copy-Item $RemotePath $LocalPath + return } + + $retries = 10 + while ($retries -gt 0) { + $retries -= 1 + try { + Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath + return + } + catch { + Write-Verbose "Request failed. $retries retries remaining" + } + } + + Write-Error "Download failed: '$RemotePath'." } -&"$buildFile" @args +# +# Main +# + +# Load configuration or set defaults + +if (Test-Path $ConfigFile) { + [xml] $config = Get-Content $ConfigFile + if (!($Channel)) { [string] $Channel = Select-Xml -Xml $config -XPath '/Project/PropertyGroup/KoreBuildChannel' } + if (!($ToolsSource)) { [string] $ToolsSource = Select-Xml -Xml $config -XPath '/Project/PropertyGroup/KoreBuildToolsSource' } +} + +if (!$DotNetHome) { + $DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } ` + elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} ` + elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}` + else { Join-Path $PSScriptRoot '.dotnet'} +} + +if (!$Channel) { $Channel = 'dev' } +if (!$ToolsSource) { $ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' } + +# Execute + +$korebuildPath = Get-KoreBuild +Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1') + +try { + Install-Tools $ToolsSource $DotNetHome + Invoke-RepositoryBuild $Path @MSBuildArgs +} +finally { + Remove-Module 'KoreBuild' -ErrorAction Ignore +} diff --git a/build.sh b/build.sh index 487eeb81f6..dc1cb5f14b 100755 --- a/build.sh +++ b/build.sh @@ -1,46 +1,195 @@ #!/usr/bin/env bash -repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $repoFolder -koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.1.zip" -if [ ! -z $KOREBUILD_ZIP ]; then - koreBuildZip=$KOREBUILD_ZIP -fi +set -euo pipefail -buildFolder=".build" -buildFile="$buildFolder/KoreBuild.sh" +# +# variables +# -if test ! -d $buildFolder; then - echo "Downloading KoreBuild from $koreBuildZip" +RESET="\033[0m" +RED="\033[0;31m" +MAGENTA="\033[0;95m" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +[ -z "${DOTNET_HOME:-}"] && DOTNET_HOME="$HOME/.dotnet" +config_file="$DIR/version.props" +verbose=false +update=false +repo_path="$DIR" +channel='' +tools_source='' - tempFolder="/tmp/KoreBuild-$(uuidgen)" - mkdir $tempFolder +# +# Functions +# +__usage() { + echo "Usage: $(basename ${BASH_SOURCE[0]}) [options] [[--] ...]" + echo "" + echo "Arguments:" + echo " ... Arguments passed to MSBuild. Variable number of arguments allowed." + echo "" + echo "Options:" + echo " --verbose Show verbose output." + echo " -c|--channel The channel of KoreBuild to download. Overrides the value from the config file.." + echo " --config-file TThe path to the configuration file that stores values. Defaults to version.props." + echo " -d|--dotnet-home The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet." + echo " --path The directory to build. Defaults to the directory containing the script." + echo " -s|--tools-source The base url where build tools can be downloaded. Overrides the value from the config file." + echo " -u|--update Update to the latest KoreBuild even if the lock file is present." + echo "" + echo "Description:" + echo " This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be." + echo " When the lockfile is not present, KoreBuild will create one using latest available version from \$channel." - localZipFile="$tempFolder/korebuild.zip" - - retries=6 - until (wget -O $localZipFile $koreBuildZip 2>/dev/null || curl -o $localZipFile --location $koreBuildZip 2>/dev/null) - do - echo "Failed to download '$koreBuildZip'" - if [ "$retries" -le 0 ]; then - exit 1 - fi - retries=$((retries - 1)) - echo "Waiting 10 seconds before retrying. Retries left: $retries" - sleep 10s - done - - unzip -q -d $tempFolder $localZipFile - - mkdir $buildFolder - cp -r $tempFolder/**/build/** $buildFolder - - chmod +x $buildFile - - # Cleanup - if test -d $tempFolder; then - rm -rf $tempFolder + if [[ "${1:-}" != '--no-exit' ]]; then + exit 2 fi +} + +get_korebuild() { + local lock_file="$repo_path/korebuild-lock.txt" + if [ ! -f $lock_file ] || [ "$update" = true ]; then + __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" $lock_file + fi + local version="$(grep 'version:*' -m 1 $lock_file)" + if [[ "$version" == '' ]]; then + __error "Failed to parse version from $lock_file. Expected a line that begins with 'version:'" + return 1 + fi + version="$(echo ${version#version:} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version" + + { + if [ ! -d "$korebuild_path" ]; then + mkdir -p "$korebuild_path" + local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip" + tmpfile="$(mktemp)" + echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}" + if __get_remote_file $remote_path $tmpfile; then + unzip -q -d "$korebuild_path" $tmpfile + fi + rm $tmpfile || true + fi + + source "$korebuild_path/KoreBuild.sh" + } || { + if [ -d "$korebuild_path" ]; then + echo "Cleaning up after failed installation" + rm -rf "$korebuild_path" || true + fi + return 1 + } +} + +__error() { + echo -e "${RED}$@${RESET}" 1>&2 +} + +__machine_has() { + hash "$1" > /dev/null 2>&1 + return $? +} + +__get_remote_file() { + local remote_path=$1 + local local_path=$2 + + if [[ "$remote_path" != 'http'* ]]; then + cp $remote_path $local_path + return 0 + fi + + failed=false + if __machine_has curl ; then + curl --retry 10 -sSL -f --create-dirs -o $local_path $remote_path || failed=true + elif __machine_has wget; then + wget --tries 10 -O $local_path $remote_path || failed=true + else + failed=true + fi + + if [ "$failed" = true ]; then + __error "Download failed: $remote_path" 1>&2 + return 1 + fi +} + +__read_dom () { local IFS=\> ; read -d \< ENTITY CONTENT ;} + +# +# main +# + +while [[ $# > 0 ]]; do + case $1 in + -\?|-h|--help) + __usage --no-exit + exit 0 + ;; + -c|--channel|-Channel) + shift + channel=${1:-} + [ -z "$channel" ] && __usage + ;; + --config-file|-ConfigFile) + shift + config_file="${1:-}" + [ -z "$config_file" ] && __usage + ;; + -d|--dotnet-home|-DotNetHome) + shift + DOTNET_HOME=${1:-} + [ -z "$DOTNET_HOME" ] && __usage + ;; + --path|-Path) + shift + repo_path="${1:-}" + [ -z "$repo_path" ] && __usage + ;; + -s|--tools-source|-ToolsSource) + shift + tools_source="${1:-}" + [ -z "$tools_source" ] && __usage + ;; + -u|--update|-Update) + update=true + ;; + --verbose|-Verbose) + verbose=true + ;; + --) + shift + break + ;; + *) + break + ;; + esac + shift +done + +if ! __machine_has unzip; then + __error 'Missing required command: unzip' + exit 1 fi -$buildFile -r $repoFolder "$@" +if ! __machine_has curl && ! __machine_has wget; then + __error 'Missing required command. Either wget or curl is required.' + exit 1 +fi + +if [ -f $config_file ]; then + comment=false + while __read_dom; do + if [ "$comment" = true ]; then [[ $CONTENT == *'-->'* ]] && comment=false ; continue; fi + if [[ $ENTITY == '!--'* ]]; then comment=true; continue; fi + if [ -z "$channel" ] && [[ $ENTITY == "KoreBuildChannel" ]]; then channel=$CONTENT; fi + if [ -z "$tools_source" ] && [[ $ENTITY == "KoreBuildToolsSource" ]]; then tools_source=$CONTENT; fi + done < $config_file +fi + +[ -z "$channel" ] && channel='dev' +[ -z "$tools_source" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools' + +get_korebuild +install_tools "$tools_source" "$DOTNET_HOME" +invoke_repository_build "$repo_path" $@ diff --git a/korebuild-lock.txt b/korebuild-lock.txt new file mode 100644 index 0000000000..7b57925f8f --- /dev/null +++ b/korebuild-lock.txt @@ -0,0 +1,2 @@ +version:2.0.0-rtm-15492 +commithash:9ea72bcf88063cee9afbe53835681702e2efd720 From 9eb27fa53fc0722ea4b6d1fe940077e0a5d6790e Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 14 Sep 2017 14:48:11 -0700 Subject: [PATCH 009/115] Stub out a task to validate cascading version effects (#557) --- build/repo.targets | 21 ++++++++++++++----- build/tasks/RepoTasks.tasks | 3 ++- build/tasks/VerifyBuildGraph.cs | 37 +++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 build/tasks/VerifyBuildGraph.cs diff --git a/build/repo.targets b/build/repo.targets index 8b0782c1b0..5d67fafd0d 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -57,12 +57,13 @@ + <_CloneRepositories Include="@(Repository);@(VerifyRepositories)" /> <_CloneRepository Include="$(MSBuildProjectFullPath)"> - CloneRepository=%(Repository.Identity); - CloneUrl=%(Repository.CloneUrl); - CloneBranch=%(Repository.Branch); - CloneRepositoryCommit=%(Repository.Commit); + CloneRepository=%(_CloneRepositories.Identity); + CloneUrl=%(_CloneRepositories.CloneUrl); + CloneBranch=%(_CloneRepositories.Branch); + CloneRepositoryCommit=%(_CloneRepositories.Commit); UseGateBranch=$(UseGateBranch) @@ -110,12 +111,22 @@ Condition="'$(CloneRepositoryCommit)'!=''" /> + + + + + + + + - + %(Repository.Identity) diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index b858cdc4aa..dec196d939 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -4,4 +4,5 @@ - \ No newline at end of file + + diff --git a/build/tasks/VerifyBuildGraph.cs b/build/tasks/VerifyBuildGraph.cs new file mode 100644 index 0000000000..5e69965aac --- /dev/null +++ b/build/tasks/VerifyBuildGraph.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + public class VerifyBuildGraph : Task + { + /// + /// Repositories that we are building new versions of. + /// + [Required] + public ITaskItem[] BuildRepositories { get; set; } + + /// + /// Repos that have already been build and released. We don't compile and build them, + /// but we still want to be sure their packages are accounted for in our graph calculations. + /// + public ITaskItem[] NoBuildRepositories { get; set; } + + /// + /// New packages we are compiling. Used in the pin tool. + /// + [Output] + public ITaskItem[] PackagesToBeProduced { get; set; } + + public override bool Execute() + { + return false; + } + } +} From 52757943acb412e8bbafb5f678c128aa7581691a Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 14 Sep 2017 15:23:19 -0700 Subject: [PATCH 010/115] Backport improvements from dev branch to repo tasks - CalculateBuildGraph - PinVersion tool - Ensuring sub-repositories build with the same version of KoreBuild executing in Universe --- Universe.sln | 27 ------ build/Repositories.props | 8 +- build/RepositoryBuild.targets | 43 ++++------ build/repo.props | 6 ++ build/repo.targets | 13 +-- .../BuildGraph/DependencyGraphSpecProvider.cs | 9 ++ build/tasks/BuildGraph/GraphBuilder.cs | 19 ++++- build/tasks/BuildGraph/GraphNode.cs | 5 +- build/tasks/BuildGraph/Project.cs | 7 +- build/tasks/BuildGraph/Repository.cs | 12 ++- build/tasks/BuildGraph/TopologicalSort.cs | 5 +- .../{BuildGraph => }/CalculateBuildGraph.cs | 24 ++++-- build/tasks/Logger/FlowLogger.cs | 61 ++++++++++++++ build/tasks/PinVersions.cs | 46 +++++++++++ build/tasks/RepoTasks.csproj | 2 +- build/tasks/RepoTasks.tasks | 1 + .../DependencyGraphSpecProvider.cs | 15 ++-- .../VersionPinning}/PinVersionUtility.cs | 82 +++++++++++++------ tools/PinVersions/PinVersions.csproj | 17 ---- tools/PinVersions/Program.cs | 53 ------------ 20 files changed, 269 insertions(+), 186 deletions(-) delete mode 100644 Universe.sln create mode 100644 build/repo.props rename build/tasks/{BuildGraph => }/CalculateBuildGraph.cs (88%) create mode 100644 build/tasks/Logger/FlowLogger.cs create mode 100644 build/tasks/PinVersions.cs rename {tools/shared => build/tasks/VersionPinning}/DependencyGraphSpecProvider.cs (86%) rename {tools/PinVersions => build/tasks/VersionPinning}/PinVersionUtility.cs (69%) delete mode 100644 tools/PinVersions/PinVersions.csproj delete mode 100644 tools/PinVersions/Program.cs diff --git a/Universe.sln b/Universe.sln deleted file mode 100644 index d1dd9a12a5..0000000000 --- a/Universe.sln +++ /dev/null @@ -1,27 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26228.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PinVersions", "tools\PinVersions\PinVersions.csproj", "{DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{085280EC-7055-426A-BF9C-1B692B9599AB}" - ProjectSection(SolutionItems) = preProject - tools\shared\DependencyGraphSpecProvider.cs = tools\shared\DependencyGraphSpecProvider.cs - EndProjectSection -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/build/Repositories.props b/build/Repositories.props index 6ec2904fe3..1bdfb82212 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -1,10 +1,7 @@ - - - - + @@ -13,6 +10,9 @@ Build tools will *verify* that these repos will be unaffected by the patch update and do not need updating. --> + + + diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 52fe532c38..eb6e75b617 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -41,20 +41,14 @@ - - - <_KorebuildItems Include="$(RepositoryRoot).build\**\*.*" /> - - - - + + + + WorkingDirectory="$(RepositoryRoot)" /> @@ -71,24 +65,19 @@ SourceFiles="@(RepositoryMSBuildArtifacts)" DestinationFolder="$(ArtifactsDir)msbuild\$(RepositoryToBuild)\%(RecursiveDir)" /> - - - - - + + + + + - - $(RepositoryRoot)tools\PinVersions\bin\$(Configuration)\netcoreapp1.1\PinVersions.dll - $(DotNetPath) $(PinToolBinary) --graph-specs-root "$(_RestoreGraphSpecsDirectory) " -s "$(BuildDir) " "$(BuildRepositoryRoot) " - $(PinVersionArgs) -s "$(_DependencyPackagesDirectory) " - - - + - \ No newline at end of file + + diff --git a/build/repo.props b/build/repo.props new file mode 100644 index 0000000000..844540c47a --- /dev/null +++ b/build/repo.props @@ -0,0 +1,6 @@ + + + + true + + diff --git a/build/repo.targets b/build/repo.targets index 5d67fafd0d..9dc8fdd308 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -23,10 +23,6 @@ $(BuildDependsOn);CloneRepositories;BuildRepositories - - - - @@ -57,7 +53,7 @@ - <_CloneRepositories Include="@(Repository);@(VerifyRepositories)" /> + <_CloneRepositories Include="@(Repository);@(VerifyRepository)" /> <_CloneRepository Include="$(MSBuildProjectFullPath)"> CloneRepository=%(_CloneRepositories.Identity); @@ -122,7 +118,7 @@ + DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;_GenerateBuildGraph;_BuildRepositories" /> @@ -213,9 +209,4 @@ - - - - - diff --git a/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs b/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs index 805033c1c2..d16600578a 100644 --- a/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs +++ b/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs @@ -1,3 +1,6 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System.IO; using NuGet.ProjectModel; @@ -15,6 +18,12 @@ namespace RepoTools.BuildGraph public DependencyGraphSpec GetDependencyGraphSpec(string repositoryName, string solutionPath) { var outputFile = Path.Combine(_packageSpecDirectory, repositoryName, Path.GetFileName(solutionPath) + ".json"); + + if (!File.Exists(outputFile)) + { + return null; + } + return DependencyGraphSpec.Load(outputFile); } } diff --git a/build/tasks/BuildGraph/GraphBuilder.cs b/build/tasks/BuildGraph/GraphBuilder.cs index 8199335433..d70b1ba45d 100644 --- a/build/tasks/BuildGraph/GraphBuilder.cs +++ b/build/tasks/BuildGraph/GraphBuilder.cs @@ -1,12 +1,17 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using Microsoft.Build.Utilities; namespace RepoTools.BuildGraph { public static class GraphBuilder { - public static IList Generate(IList repositories, string root) + public static IList Generate(IList repositories, string root, TaskLoggingHelper log) { // Build global list of primary projects var primaryProjects = repositories.SelectMany(c => c.Projects) @@ -31,7 +36,15 @@ namespace RepoTools.BuildGraph var dependencyRepository = dependencyProject.Repository; var dependencyNode = graphNodes[dependencyRepository]; - thisProjectRepositoryNode.Incoming.Add(dependencyNode); + if (ReferenceEquals(thisProjectRepositoryNode, dependencyNode)) + { + log.LogWarning("{0} has a package reference to a package produced in the same repo. {1} -> {2}", project.Repository.Name, Path.GetFileName(project.Path), packageDependency); + } + else + { + thisProjectRepositoryNode.Incoming.Add(dependencyNode); + } + dependencyNode.Outgoing.Add(thisProjectRepositoryNode); } } diff --git a/build/tasks/BuildGraph/GraphNode.cs b/build/tasks/BuildGraph/GraphNode.cs index 8c2a023ab2..b7197e8b38 100644 --- a/build/tasks/BuildGraph/GraphNode.cs +++ b/build/tasks/BuildGraph/GraphNode.cs @@ -1,4 +1,7 @@ -using System.Collections.Generic; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; using System.Diagnostics; namespace RepoTools.BuildGraph diff --git a/build/tasks/BuildGraph/Project.cs b/build/tasks/BuildGraph/Project.cs index 10a821a336..72b5eee254 100644 --- a/build/tasks/BuildGraph/Project.cs +++ b/build/tasks/BuildGraph/Project.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using System.Diagnostics; @@ -14,6 +17,8 @@ namespace RepoTools.BuildGraph public string Name { get; } + public string Version { get; set; } + public string Path { get; set; } public Repository Repository { get; set; } diff --git a/build/tasks/BuildGraph/Repository.cs b/build/tasks/BuildGraph/Repository.cs index c71ade8555..c1e3e5c619 100644 --- a/build/tasks/BuildGraph/Repository.cs +++ b/build/tasks/BuildGraph/Repository.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -49,6 +52,7 @@ namespace RepoTools.BuildGraph var repository = new Repository(name); ReadSharedSourceProjects(Path.Combine(repositoryPath, "shared"), repository, repository.Projects); + var srcDirectory = Path.GetFullPath(Path.Combine(repositoryPath, "src")) .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); @@ -56,6 +60,11 @@ namespace RepoTools.BuildGraph foreach (var file in solutionFiles) { var spec = provider.GetDependencyGraphSpec(name, file); + if (spec == null) + { + continue; + } + var projects = spec.Projects.OrderBy(p => p.RestoreMetadata.ProjectStyle == ProjectStyle.PackageReference ? 0 : 1); foreach (var specProject in projects) { @@ -73,6 +82,7 @@ namespace RepoTools.BuildGraph { Repository = repository, Path = specProject.FilePath, + Version = specProject.Version?.ToString(), }; projectGroup.Add(project); diff --git a/build/tasks/BuildGraph/TopologicalSort.cs b/build/tasks/BuildGraph/TopologicalSort.cs index 4d362be3e4..161a9913d7 100644 --- a/build/tasks/BuildGraph/TopologicalSort.cs +++ b/build/tasks/BuildGraph/TopologicalSort.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using System.Linq; diff --git a/build/tasks/BuildGraph/CalculateBuildGraph.cs b/build/tasks/CalculateBuildGraph.cs similarity index 88% rename from build/tasks/BuildGraph/CalculateBuildGraph.cs rename to build/tasks/CalculateBuildGraph.cs index 6e1dc6de2b..9475bd99ec 100644 --- a/build/tasks/BuildGraph/CalculateBuildGraph.cs +++ b/build/tasks/CalculateBuildGraph.cs @@ -15,19 +15,19 @@ namespace RepoTasks [Required] public ITaskItem[] Repositories { get; set; } - [Output] - public ITaskItem[] RepositoriesToBuildInOrder { get; set; } + /// + /// Directory that contains the package spec files. + /// + [Required] + public string PackageSpecsDirectory { get; set; } /// /// The repository at which to root the graph at /// public string StartGraphAt { get; set; } - /// - /// Directory that contains the package spec files. - /// - [Required] - public string PackageSpecsDirectory { get; set; } + [Output] + public ITaskItem[] RepositoriesToBuildInOrder { get; set; } public override bool Execute() { @@ -36,12 +36,18 @@ namespace RepoTasks var repositoryPaths = Repositories.Select(r => r.GetMetadata("RepositoryPath")).ToList(); var repositories = Repository.ReadAllRepositories(repositoryPaths, graphSpecProvider); - var graph = GraphBuilder.Generate(repositories, StartGraphAt); + var graph = GraphBuilder.Generate(repositories, StartGraphAt, Log); var repositoriesWithOrder = new List<(ITaskItem repository, int order)>(); foreach (var repositoryTaskItem in Repositories) { var repositoryName = repositoryTaskItem.ItemSpec; - var graphNodeRepository = graph.First(g => g.Repository.Name == repositoryName); + var graphNodeRepository = graph.FirstOrDefault(g => g.Repository.Name == repositoryName); + if (graphNodeRepository == null) + { + // StartGraphAt was specified so the graph is incomplete. + continue; + } + var order = TopologicalSort.GetOrder(graphNodeRepository); repositoryTaskItem.SetMetadata("Order", order.ToString()); repositoriesWithOrder.Add((repositoryTaskItem, order)); diff --git a/build/tasks/Logger/FlowLogger.cs b/build/tasks/Logger/FlowLogger.cs new file mode 100644 index 0000000000..e6a214fdff --- /dev/null +++ b/build/tasks/Logger/FlowLogger.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Logging; + +namespace RepoTasks +{ + public class FlowLogger : ConsoleLogger + { + private volatile bool _initialized; + + public FlowLogger() + { + } + + public override void Initialize(IEventSource eventSource, int nodeCount) + { + PreInit(eventSource); + base.Initialize(eventSource, nodeCount); + } + + public override void Initialize(IEventSource eventSource) + { + PreInit(eventSource); + base.Initialize(eventSource); + } + + private void PreInit(IEventSource eventSource) + { + if (_initialized) return; + _initialized = true; + + var flowId = GetFlowId(); + var prefix = $"{flowId,-22}| "; + var write = WriteHandler; + WriteHandler = msg => write(prefix + msg); + + eventSource.BuildStarted += (o, e) => + { + WriteHandler(e.Message + Environment.NewLine); + }; + } + + private string GetFlowId() + { + var parameters = Parameters?.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries); + if (parameters == null || parameters.Length == 0) + { + return null; + } + + const string flowIdParamName = "FlowId="; + return parameters + .FirstOrDefault(p => p.StartsWith(flowIdParamName, StringComparison.Ordinal)) + ?.Substring(flowIdParamName.Length); + } + } +} diff --git a/build/tasks/PinVersions.cs b/build/tasks/PinVersions.cs new file mode 100644 index 0000000000..b98c878173 --- /dev/null +++ b/build/tasks/PinVersions.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Microsoft.Build.Framework; +using RepoTasks.VersionPinning; + +namespace RepoTasks +{ + public class PinVersions : Microsoft.Build.Utilities.Task + { + [Required] + public string BuildRepositoryRoot { get; set; } + + [Required] + public ITaskItem[] PackageSources { get; set; } + + public string GraphSpecsRoot { get; set; } + + public override bool Execute() + { + if (PackageSources?.Length == 0) + { + Log.LogError($"Missing PackageSources. At least one item source must be specified."); + return false; + } + + var graphSpecProvider = !string.IsNullOrEmpty(GraphSpecsRoot) + ? new DependencyGraphSpecProvider(GraphSpecsRoot) + : DependencyGraphSpecProvider.Default; + + using (graphSpecProvider) + { + var pinVersionUtility = new PinVersionUtility( + BuildRepositoryRoot, + PackageSources.Select(i => i.ItemSpec).ToList(), + graphSpecProvider, + Log); + pinVersionUtility.Execute(); + } + + return true; + } + } +} diff --git a/build/tasks/RepoTasks.csproj b/build/tasks/RepoTasks.csproj index d0f925d5d1..084b189145 100644 --- a/build/tasks/RepoTasks.csproj +++ b/build/tasks/RepoTasks.csproj @@ -6,7 +6,7 @@ - + diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index dec196d939..0c188ffb1e 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -5,4 +5,5 @@ + diff --git a/tools/shared/DependencyGraphSpecProvider.cs b/build/tasks/VersionPinning/DependencyGraphSpecProvider.cs similarity index 86% rename from tools/shared/DependencyGraphSpecProvider.cs rename to build/tasks/VersionPinning/DependencyGraphSpecProvider.cs index 4eb96e1234..1a5e2af362 100644 --- a/tools/shared/DependencyGraphSpecProvider.cs +++ b/build/tasks/VersionPinning/DependencyGraphSpecProvider.cs @@ -1,17 +1,19 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using Microsoft.DotNet.Cli.Utils; using NuGet.ProjectModel; -namespace UniverseTools +namespace RepoTasks.VersionPinning { public class DependencyGraphSpecProvider : IDisposable { private readonly string _packageSpecDirectory; private readonly bool _deleteSpecDirectoryOnDispose; - private readonly string _muxerPath; + private readonly string _dotnetPath; public DependencyGraphSpecProvider(string packageSpecDirectory) : this(packageSpecDirectory, deleteSpecDirectoryOnDispose: false) @@ -22,7 +24,7 @@ namespace UniverseTools { _packageSpecDirectory = packageSpecDirectory; _deleteSpecDirectoryOnDispose = deleteSpecDirectoryOnDispose; - _muxerPath = new Muxer().MuxerPath; + _dotnetPath = Process.GetCurrentProcess().MainModule.FileName; } public static DependencyGraphSpecProvider Default { get; } = @@ -42,7 +44,7 @@ namespace UniverseTools private void RunMSBuild(string solutionPath, string outputFile) { - var psi = new ProcessStartInfo(_muxerPath); + var psi = new ProcessStartInfo(_dotnetPath); var arguments = new List { @@ -53,6 +55,7 @@ namespace UniverseTools "/v:q", "/p:BuildProjectReferences=false", $"/p:RestoreGraphOutputPath=\"{outputFile}\"", + "/p:KoreBuildRestoreTargetsImported=true", }; psi.Arguments = string.Join(" ", arguments); diff --git a/tools/PinVersions/PinVersionUtility.cs b/build/tasks/VersionPinning/PinVersionUtility.cs similarity index 69% rename from tools/PinVersions/PinVersionUtility.cs rename to build/tasks/VersionPinning/PinVersionUtility.cs index 42d1e90176..c94e19fc62 100644 --- a/tools/PinVersions/PinVersionUtility.cs +++ b/build/tasks/VersionPinning/PinVersionUtility.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; @@ -6,6 +9,8 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; using NuGet.Common; using NuGet.Frameworks; using NuGet.LibraryModel; @@ -13,19 +18,23 @@ using NuGet.ProjectModel; using NuGet.Protocol; using NuGet.Protocol.Core.Types; using NuGet.Versioning; -using UniverseTools; -namespace PinVersions +namespace RepoTasks.VersionPinning { - class PinVersionUtility + internal class PinVersionUtility { private readonly string _repositoryRoot; private readonly FindPackageByIdResource[] _findPackageResources; private readonly ConcurrentDictionary> _exactMatches = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); private readonly DependencyGraphSpecProvider _provider; private readonly SourceCacheContext _sourceCacheContext; + private readonly TaskLoggingHelper _logger; - public PinVersionUtility(string repositoryRoot, List pinSources, DependencyGraphSpecProvider provider) + public PinVersionUtility( + string repositoryRoot, + List pinSources, + DependencyGraphSpecProvider provider, + TaskLoggingHelper logger) { _repositoryRoot = repositoryRoot; _findPackageResources = new FindPackageByIdResource[pinSources.Count]; @@ -36,19 +45,27 @@ namespace PinVersions } _provider = provider; _sourceCacheContext = new SourceCacheContext(); + _logger = logger; } public void Execute() { - var solutionPinMetadata = GetPinVersionMetadata(); - foreach (var item in solutionPinMetadata) + _logger.LogMessage(MessageImportance.High, $"Pinning package references for projects in {_repositoryRoot}"); + + var solutionPinMetadata = GetProjectPinVersionMetadata(); + foreach (var cliToolReference in solutionPinMetadata.CLIToolReferences) + { + _logger.LogMessage(MessageImportance.Normal, $"Pinning CLI Tool {cliToolReference.Item1.Name}({cliToolReference.Item1.VersionRange} to {cliToolReference.Item2} for all projects in {_repositoryRoot}."); + } + + foreach (var item in solutionPinMetadata.PinVersionLookup) { var projectPinMetadata = item.Value; var specProject = projectPinMetadata.PackageSpec; - if (!(projectPinMetadata.Packages.Any() || projectPinMetadata.CLIToolReferences.Any())) + if (!(projectPinMetadata.Packages.Any() || solutionPinMetadata.CLIToolReferences.Any())) { - Console.WriteLine($"No package or tool references to pin for {specProject.FilePath}."); + _logger.LogMessage(MessageImportance.Normal, $"No package or tool references to pin for {specProject.FilePath}."); continue; } @@ -59,12 +76,16 @@ namespace PinVersions Directory.CreateDirectory(Path.GetDirectoryName(pinnedReferencesFile)); - Console.WriteLine($"Pinning package versions for {specProject.FilePath}."); - var pinnedReferences = new XElement("ItemGroup"); + if (projectPinMetadata.Packages.Any()) + { + _logger.LogMessage(MessageImportance.Normal, $"Pinning package versions for {specProject.FilePath}."); + } + + var pinnedReferences = new XElement("ItemGroup", new XAttribute("Condition", "'$(PolicyDesignTimeBuild)' != 'true' AND !Exists('$(MSBuildThisFileDirectory)$(MSBuildProjectFile).nugetpolicy.g.targets')")); foreach (var packageReference in projectPinMetadata.Packages) { (var tfm, var libraryRange, var exactVersion) = packageReference; - Console.WriteLine($"Pinning reference {libraryRange.Name}({libraryRange.VersionRange} to {exactVersion}."); + _logger.LogMessage(MessageImportance.Normal, $"Pinning reference {libraryRange.Name}({libraryRange.VersionRange} to {exactVersion}."); var metadata = new List { new XAttribute("Update", libraryRange.Name), @@ -79,10 +100,10 @@ namespace PinVersions pinnedReferences.Add(new XElement("PackageReference", metadata)); } - foreach (var toolReference in projectPinMetadata.CLIToolReferences) + // CLI Tool references are specified at solution level. + foreach (var toolReference in solutionPinMetadata.CLIToolReferences) { (var libraryRange, var exactVersion) = toolReference; - Console.WriteLine($"Pinning CLI Tool {libraryRange.Name}({libraryRange.VersionRange} to {exactVersion}."); var metadata = new List { new XAttribute("Update", libraryRange.Name), @@ -97,10 +118,11 @@ namespace PinVersions } } - private IDictionary GetPinVersionMetadata() + private SolutionPinVersionMetadata GetProjectPinVersionMetadata() { var repositoryDirectoryInfo = new DirectoryInfo(_repositoryRoot); - var projects = new Dictionary(StringComparer.OrdinalIgnoreCase); + var projects = new Dictionary(StringComparer.OrdinalIgnoreCase); + var cliToolReferences = new List<(LibraryRange, NuGetVersion)>(); foreach (var slnFile in repositoryDirectoryInfo.EnumerateFiles("*.sln")) { @@ -109,7 +131,7 @@ namespace PinVersions { if (!projects.TryGetValue(specProject.FilePath, out var pinMetadata)) { - pinMetadata = new PinVersionMetadata(specProject); + pinMetadata = new ProjectPinVersionMetadata(specProject); projects[specProject.FilePath] = pinMetadata; } @@ -139,7 +161,7 @@ namespace PinVersions } else if (projectStyle == ProjectStyle.DotnetCliTool) { - pinMetadata.CLIToolReferences.Add((reference.LibraryRange, exactVersion)); + cliToolReferences.Add((reference.LibraryRange, exactVersion)); } else { @@ -149,7 +171,7 @@ namespace PinVersions } } - return projects; + return new SolutionPinVersionMetadata(projects, cliToolReferences); } private NuGetVersion GetExactVersion(string name, VersionRange range) @@ -188,20 +210,32 @@ namespace PinVersions return null; } - private struct PinVersionMetadata + private struct SolutionPinVersionMetadata { - public PinVersionMetadata(PackageSpec packageSpec) + public SolutionPinVersionMetadata( + IDictionary pinVersionLookup, + List<(LibraryRange, NuGetVersion)> cliToolReferences) + { + PinVersionLookup = pinVersionLookup; + CLIToolReferences = cliToolReferences; + } + + public IDictionary PinVersionLookup { get; } + + public List<(LibraryRange, NuGetVersion)> CLIToolReferences { get; } + } + + private struct ProjectPinVersionMetadata + { + public ProjectPinVersionMetadata(PackageSpec packageSpec) { PackageSpec = packageSpec; Packages = new List<(NuGetFramework, LibraryRange, NuGetVersion)>(); - CLIToolReferences = new List<(LibraryRange, NuGetVersion)>(); } public PackageSpec PackageSpec { get; } public List<(NuGetFramework, LibraryRange, NuGetVersion)> Packages { get; } - - public List<(LibraryRange, NuGetVersion)> CLIToolReferences { get; } } } } diff --git a/tools/PinVersions/PinVersions.csproj b/tools/PinVersions/PinVersions.csproj deleted file mode 100644 index f52efb6b1c..0000000000 --- a/tools/PinVersions/PinVersions.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - Exe - netcoreapp1.1 - - - - - - - - - - - - \ No newline at end of file diff --git a/tools/PinVersions/Program.cs b/tools/PinVersions/Program.cs deleted file mode 100644 index f197fecf5e..0000000000 --- a/tools/PinVersions/Program.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System; -using Microsoft.Extensions.CommandLineUtils; -using UniverseTools; - -namespace PinVersions -{ - class Program - { - static int Main(string[] args) - { - var app = new CommandLineApplication(); - - var pinSourceOption = app.Option("-s|--source", - "Feed containing packages to pin.", - CommandOptionType.MultipleValue); - - var packageSpecsDirectoryOption = app.Option("--graph-specs-root", - "Directory containing package specs. (Optional)", - CommandOptionType.SingleValue); - - var repositoryArgument = app.Argument("Repository", "Repository directory"); - - app.OnExecute(() => - { - if (!pinSourceOption.HasValue()) - { - Console.Error.WriteLine($"Option {pinSourceOption.Template} must have a value."); - return 1; - } - - if (string.IsNullOrEmpty(repositoryArgument.Value)) - { - Console.Error.WriteLine($"Repository argument must be specified."); - return 1; - } - - var graphSpecProvider = packageSpecsDirectoryOption.HasValue() ? - new DependencyGraphSpecProvider(packageSpecsDirectoryOption.Value().Trim()) : - DependencyGraphSpecProvider.Default; - - using (graphSpecProvider) - { - var pinVersionUtility = new PinVersionUtility(repositoryArgument.Value.Trim(), pinSourceOption.Values, graphSpecProvider); - pinVersionUtility.Execute(); - } - - return 0; - }); - - return app.Execute(args); - } - } -} \ No newline at end of file From 8f25a559a54603b764a998b9cea6c3c0bf2c929c Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 14 Sep 2017 17:13:57 -0700 Subject: [PATCH 011/115] Begin implementation of build graph analysis --- build/RepositoryBuild.targets | 10 +- build/repo.targets | 69 ++++----- build/tasks/AnalyzeBuildGraph.cs | 74 ++++++++++ .../ProjectModel/DotNetCliReferenceInfo.cs | 24 ++++ .../ProjectModel/PackageReferenceInfo.cs | 29 ++++ .../ProjectModel/ProjectFrameworkInfo.cs | 22 +++ build/tasks/ProjectModel/ProjectInfo.cs | 48 +++++++ .../tasks/ProjectModel/ProjectInfoFactory.cs | 134 ++++++++++++++++++ build/tasks/ProjectModel/SolutionInfo.cs | 34 +++++ .../tasks/ProjectModel/SolutionInfoFactory.cs | 134 ++++++++++++++++++ build/tasks/RepoTasks.tasks | 2 +- build/tasks/Utilities/KoreBuildErrors.cs | 29 ++++ build/tasks/Utilities/LoggingExtensions.cs | 26 ++++ build/tasks/Utilities/MSBuildListSplitter.cs | 45 ++++++ build/tasks/VerifyBuildGraph.cs | 37 ----- 15 files changed, 636 insertions(+), 81 deletions(-) create mode 100644 build/tasks/AnalyzeBuildGraph.cs create mode 100644 build/tasks/ProjectModel/DotNetCliReferenceInfo.cs create mode 100644 build/tasks/ProjectModel/PackageReferenceInfo.cs create mode 100644 build/tasks/ProjectModel/ProjectFrameworkInfo.cs create mode 100644 build/tasks/ProjectModel/ProjectInfo.cs create mode 100644 build/tasks/ProjectModel/ProjectInfoFactory.cs create mode 100644 build/tasks/ProjectModel/SolutionInfo.cs create mode 100644 build/tasks/ProjectModel/SolutionInfoFactory.cs create mode 100644 build/tasks/Utilities/KoreBuildErrors.cs create mode 100644 build/tasks/Utilities/LoggingExtensions.cs create mode 100644 build/tasks/Utilities/MSBuildListSplitter.cs delete mode 100644 build/tasks/VerifyBuildGraph.cs diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index eb6e75b617..7a329f6dca 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -2,12 +2,12 @@ - %(RepositoryToBuildInOrder.Order) - %(RepositoryToBuildInOrder.Identity) + %(RepositoryBuildOrder.Order) + %(RepositoryBuildOrder.Identity) - RepositoryToBuild=%(RepositoryToBuildInOrder.Identity); - BuildRepositoryRoot=%(RepositoryToBuildInOrder.RepositoryPath)\; - CommitHash=%(RepositoryToBuildInOrder.Commit) + RepositoryToBuild=%(RepositoryBuildOrder.Identity); + BuildRepositoryRoot=%(RepositoryBuildOrder.RepositoryPath)\; + CommitHash=%(RepositoryBuildOrder.Commit) diff --git a/build/repo.targets b/build/repo.targets index 9dc8fdd308..0d387361d4 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -7,7 +7,6 @@ <_CloneRepositoryRoot>$(RepositoryRoot).r\ <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ <_DependencyPackagesDirectory>$(_DependencyBuildDirectory) - <_RestoreGraphSpecsDirectory>$(RepositoryRoot)obj\package-specs\ <_RepositoryListFileName>Repositories.props <_DefaultRepositoryList>$(MSBuildThisFileDirectory)$(_RepositoryListFileName) @@ -20,14 +19,13 @@ $(PrepareDependsOn);CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts - $(BuildDependsOn);CloneRepositories;BuildRepositories + $(BuildDependsOn);BuildRepositories - @@ -107,48 +105,43 @@ Condition="'$(CloneRepositoryCommit)'!=''" /> - - - - - - - - + DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;Graph;_BuildRepositories" /> + + + + + + + + + - - - %(Repository.Identity) - - - - RestoreGraphOutputPath=$(_RestoreGraphSpecsDirectory)%(Solution.Repository)\%(Solution.FileName)%(Solution.Extension).json - - - - + + <_NoBuildSolution Update="@(_NoBuildSolution)" Build="false" /> + + + - - - + + + + + - - - - + + diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs new file mode 100644 index 0000000000..a135e66568 --- /dev/null +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -0,0 +1,74 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using RepoTasks.ProjectModel; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class AnalyzeBuildGraph : Task, ICancelableTask + { + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + + /// + /// Repositories that we are building new versions of. + /// + [Required] + public ITaskItem[] Solutions { get; set; } + + [Required] + public string Properties { get; set; } + + /// + /// New packages we are compiling. Used in the pin tool. + /// + [Output] + public ITaskItem[] PackagesProduced { get; set; } + + /// + /// The order in which to build repositories + /// + [Output] + public ITaskItem[] RepositoryBuildOrder { get; set; } + + public void Cancel() + { + _cts.Cancel(); + } + + public override bool Execute() + { + var factory = new SolutionInfoFactory(Log, BuildEngine5); + var props = MSBuildListSplitter.GetNamedProperties(Properties); + + Log.LogMessage(MessageImportance.High, $"Beginning cross-repo analysis on {Solutions.Length} solutions. Hang tight..."); + + var solutions = factory.Create(Solutions, props, _cts.Token); + Log.LogMessage($"Found {solutions.Count} and {solutions.Sum(p => p.Projects.Count)} projects"); + + if (_cts.IsCancellationRequested) + { + return false; + } + + PackagesProduced = solutions + .Where(s => s.ShouldBuild) + .SelectMany(p => p.Projects) + .Where(p => p.IsPackable) + .Select(p => new TaskItem(p.PackageId, new Hashtable + { + ["Version"] = p.PackageVersion + })) + .ToArray(); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/build/tasks/ProjectModel/DotNetCliReferenceInfo.cs b/build/tasks/ProjectModel/DotNetCliReferenceInfo.cs new file mode 100644 index 0000000000..c490dab999 --- /dev/null +++ b/build/tasks/ProjectModel/DotNetCliReferenceInfo.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace RepoTasks.ProjectModel +{ + internal class DotNetCliReferenceInfo + { + public DotNetCliReferenceInfo(string id, string version) + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(nameof(id)); + } + + Id = id; + Version = version; + } + + public string Id { get; } + public string Version { get; } + } +} diff --git a/build/tasks/ProjectModel/PackageReferenceInfo.cs b/build/tasks/ProjectModel/PackageReferenceInfo.cs new file mode 100644 index 0000000000..d01089d594 --- /dev/null +++ b/build/tasks/ProjectModel/PackageReferenceInfo.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace RepoTasks.ProjectModel +{ + internal class PackageReferenceInfo + { + public PackageReferenceInfo(string id, string version, bool isImplicitlyDefined, IReadOnlyList noWarn) + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(nameof(id)); + } + + Id = id; + Version = version; + IsImplicitlyDefined = isImplicitlyDefined; + NoWarn = noWarn; + } + + public string Id { get; } + public string Version { get; } + public bool IsImplicitlyDefined { get; } + public IReadOnlyList NoWarn { get; } + } +} diff --git a/build/tasks/ProjectModel/ProjectFrameworkInfo.cs b/build/tasks/ProjectModel/ProjectFrameworkInfo.cs new file mode 100644 index 0000000000..35a212f9a1 --- /dev/null +++ b/build/tasks/ProjectModel/ProjectFrameworkInfo.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Frameworks; + +namespace RepoTasks.ProjectModel +{ + internal class ProjectFrameworkInfo + { + public ProjectFrameworkInfo(NuGetFramework targetFramework, IReadOnlyDictionary dependencies) + { + TargetFramework = targetFramework ?? throw new ArgumentNullException(nameof(targetFramework)); + Dependencies = dependencies ?? throw new ArgumentNullException(nameof(dependencies)); + } + + public NuGetFramework TargetFramework { get; } + public IReadOnlyDictionary Dependencies { get; } + } +} diff --git a/build/tasks/ProjectModel/ProjectInfo.cs b/build/tasks/ProjectModel/ProjectInfo.cs new file mode 100644 index 0000000000..1dd4339185 --- /dev/null +++ b/build/tasks/ProjectModel/ProjectInfo.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; + +namespace RepoTasks.ProjectModel +{ + internal class ProjectInfo + { + public ProjectInfo(string fullPath, + string projectExtensionsPath, + IReadOnlyList frameworks, + IReadOnlyList tools, + bool isPackable, + string packageId, + string packageVersion) + { + if (!Path.IsPathRooted(fullPath)) + { + throw new ArgumentException("Path must be absolute", nameof(fullPath)); + } + + Frameworks = frameworks ?? throw new ArgumentNullException(nameof(frameworks)); + Tools = tools ?? throw new ArgumentNullException(nameof(tools)); + + FullPath = fullPath; + FileName = Path.GetFileName(fullPath); + Directory = Path.GetDirectoryName(FullPath); + ProjectExtensionsPath = projectExtensionsPath ?? Path.Combine(Directory, "obj"); + IsPackable = isPackable; + PackageId = packageId; + PackageVersion = packageVersion; + } + + public string FullPath { get; } + public string FileName { get; } + public string ProjectExtensionsPath { get; } + public string Directory { get; } + public string PackageId { get; } + public string PackageVersion { get; } + public bool IsPackable { get; } + + public IReadOnlyList Frameworks { get; } + public IReadOnlyList Tools { get; } + } +} diff --git a/build/tasks/ProjectModel/ProjectInfoFactory.cs b/build/tasks/ProjectModel/ProjectInfoFactory.cs new file mode 100644 index 0000000000..5c739f1784 --- /dev/null +++ b/build/tasks/ProjectModel/ProjectInfoFactory.cs @@ -0,0 +1,134 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Build.Construction; +using Microsoft.Build.Evaluation; +using Microsoft.Build.Execution; +using NuGet.Frameworks; +using RepoTasks.Utilities; +using Microsoft.Build.Utilities; + +namespace RepoTasks.ProjectModel +{ + internal class ProjectInfoFactory + { + private readonly TaskLoggingHelper _logger; + + public ProjectInfoFactory(TaskLoggingHelper logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public ProjectInfo Create(string path, ProjectCollection projectCollection) + { + var project = GetProject(path, projectCollection); + var instance = project.CreateProjectInstance(ProjectInstanceSettings.ImmutableWithFastItemLookup); + var projExtPath = instance.GetPropertyValue("MSBuildProjectExtensionsPath"); + + var targetFrameworks = instance.GetPropertyValue("TargetFrameworks"); + var targetFramework = instance.GetPropertyValue("TargetFramework"); + + var frameworks = new List(); + if (!string.IsNullOrEmpty(targetFrameworks) && string.IsNullOrEmpty(targetFramework)) + { + // multi targeting + foreach (var tfm in targetFrameworks.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + project.SetGlobalProperty("TargetFramework", tfm); + var innerBuild = project.CreateProjectInstance(ProjectInstanceSettings.ImmutableWithFastItemLookup); + + var tfmInfo = new ProjectFrameworkInfo(NuGetFramework.Parse(tfm), GetDependencies(innerBuild)); + + frameworks.Add(tfmInfo); + } + + project.RemoveGlobalProperty("TargetFramework"); + } + else if (!string.IsNullOrEmpty(targetFramework)) + { + var tfmInfo = new ProjectFrameworkInfo(NuGetFramework.Parse(targetFramework), GetDependencies(instance)); + + frameworks.Add(tfmInfo); + } + + var projectDir = Path.GetDirectoryName(path); + + var tools = GetTools(instance).ToArray(); + bool.TryParse(instance.GetPropertyValue("IsPackable"), out var isPackable); + var packageId = instance.GetPropertyValue("PackageId"); + var packageVersion = instance.GetPropertyValue("PackageVersion"); + + return new ProjectInfo(path, + projExtPath, + frameworks, + tools, + isPackable, + packageId, + packageVersion); + } + + private static object _projLock = new object(); + + private static Project GetProject(string path, ProjectCollection projectCollection) + { + var projects = projectCollection.GetLoadedProjects(path); + foreach(var proj in projects) + { + if (proj.GetPropertyValue("DesignTimeBuild") == "true") + { + return proj; + } + } + + var xml = ProjectRootElement.Open(path, projectCollection); + var globalProps = new Dictionary() + { + ["DesignTimeBuild"] = "true", + }; + + var project = new Project(xml, + globalProps, + toolsVersion: "15.0", + projectCollection: projectCollection) + { + IsBuildEnabled = false + }; + + return project; + } + + private IReadOnlyDictionary GetDependencies(ProjectInstance project) + { + var references = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var item in project.GetItems("PackageReference")) + { + bool.TryParse(item.GetMetadataValue("IsImplicitlyDefined"), out var isImplicit); + var noWarn = item.GetMetadataValue("NoWarn"); + IReadOnlyList noWarnItems = string.IsNullOrEmpty(noWarn) + ? Array.Empty() + : MSBuildListSplitter.SplitItemList(noWarn).ToArray(); + + var info = new PackageReferenceInfo(item.EvaluatedInclude, item.GetMetadataValue("Version"), isImplicit, noWarnItems); + + if (references.ContainsKey(info.Id)) + { + _logger.LogKoreBuildWarning(project.ProjectFileLocation.File, KoreBuildErrors.DuplicatePackageReference, $"Found a duplicate PackageReference for {info.Id}. Restore results may be unpredictable."); + } + + references[info.Id] = info; + } + + return references; + } + + private static IEnumerable GetTools(ProjectInstance project) + { + return project.GetItems("DotNetCliToolReference").Select(item => + new DotNetCliReferenceInfo(item.EvaluatedInclude, item.GetMetadataValue("Version"))); + } + } +} diff --git a/build/tasks/ProjectModel/SolutionInfo.cs b/build/tasks/ProjectModel/SolutionInfo.cs new file mode 100644 index 0000000000..5fe51132db --- /dev/null +++ b/build/tasks/ProjectModel/SolutionInfo.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace RepoTasks.ProjectModel +{ + internal class SolutionInfo + { + public SolutionInfo(string fullPath, string configName, IReadOnlyList projects, bool shouldBuild) + { + if (string.IsNullOrEmpty(fullPath)) + { + throw new ArgumentException(nameof(fullPath)); + } + + if (string.IsNullOrEmpty(configName)) + { + throw new ArgumentException(nameof(configName)); + } + + FullPath = fullPath; + ConfigName = configName; + Projects = projects ?? throw new ArgumentNullException(nameof(projects)); + ShouldBuild = shouldBuild; + } + + public string FullPath { get; } + public string ConfigName { get; } + public IReadOnlyList Projects { get; } + public bool ShouldBuild { get; } + } +} diff --git a/build/tasks/ProjectModel/SolutionInfoFactory.cs b/build/tasks/ProjectModel/SolutionInfoFactory.cs new file mode 100644 index 0000000000..72601be2dc --- /dev/null +++ b/build/tasks/ProjectModel/SolutionInfoFactory.cs @@ -0,0 +1,134 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Construction; +using Microsoft.Build.Evaluation; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using RepoTasks.Utilities; + +namespace RepoTasks.ProjectModel +{ + internal class SolutionInfoFactory + { + private readonly TaskLoggingHelper _logger; + private readonly IBuildEngine4 _buildEngine; + + public SolutionInfoFactory(TaskLoggingHelper logger, IBuildEngine4 buildEngine) + { + _logger = logger; + _buildEngine = buildEngine; + } + + public IReadOnlyList Create(IEnumerable solutionItems, IDictionary properties, CancellationToken ct) + { + var timer = Stopwatch.StartNew(); + + var solutions = new ConcurrentBag(); + + Parallel.ForEach(solutionItems, solution => + { + if (ct.IsCancellationRequested) + { + return; + } + + var solutionFile = solution.ItemSpec.Replace('\\', '/'); + var solutionProps = new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + foreach (var prop in MSBuildListSplitter.GetNamedProperties(solution.GetMetadata("AdditionalProperties"))) + { + solutionProps[prop.Key] = prop.Value; + } + + if (solutionProps.TryGetValue("Configuration", out var configName)) + { + solutionProps["Configuration"] = configName = "Debug"; + } + + var key = $"SlnInfo:{solutionFile}:{configName}"; + var obj = _buildEngine.GetRegisteredTaskObject(key, RegisteredTaskObjectLifetime.Build); + + if (obj is SolutionInfo cachedSlnInfo) + { + solutions.Add(cachedSlnInfo); + return; + } + + _logger.LogMessage($"Analyzing {solutionFile} ({configName})"); + var projects = new ConcurrentBag(); + var projectFiles = GetProjectsForSolutionConfig(solutionFile, configName); + using (var projCollection = new ProjectCollection(solutionProps) { IsBuildEnabled = false }) + { + Parallel.ForEach(projectFiles, projectFile => + { + if (ct.IsCancellationRequested) + { + return; + } + + try + { + projects.Add(new ProjectInfoFactory(_logger).Create(projectFile, projCollection)); + } + catch (Exception ex) + { + _logger.LogErrorFromException(ex); + } + }); + } + + bool.TryParse(solution.GetMetadata("Build"), out var shouldBuild); + + var solutionInfo = new SolutionInfo( + solutionFile, + configName, + projects.ToArray(), + shouldBuild); + + _buildEngine.RegisterTaskObject(key, solutionInfo, RegisteredTaskObjectLifetime.Build, allowEarlyCollection: true); + + solutions.Add(solutionInfo); + }); + + timer.Stop(); + _logger.LogMessage(MessageImportance.Normal, $"Finished design-time build in {timer.ElapsedMilliseconds}ms"); + return solutions.ToArray(); + } + + private IList GetProjectsForSolutionConfig(string filePath, string configName) + { + var sln = SolutionFile.Parse(filePath); + + if (string.IsNullOrEmpty(configName)) + { + configName = sln.GetDefaultConfigurationName(); + } + + var projects = new List(); + + var config = sln.SolutionConfigurations.FirstOrDefault(c => c.ConfigurationName == configName); + if (config == null) + { + throw new InvalidOperationException($"A solution configuration by the name of '{configName}' was not found in '{filePath}'"); + } + + foreach (var project in sln.ProjectsInOrder + .Where(p => + p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat // skips solution folders + && p.ProjectConfigurations.TryGetValue(config.FullName, out var projectConfig) + && projectConfig.IncludeInBuild)) + { + projects.Add(project.AbsolutePath.Replace('\\', '/')); + } + + return projects; + } + } +} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 0c188ffb1e..4026e9810b 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -4,6 +4,6 @@ - + diff --git a/build/tasks/Utilities/KoreBuildErrors.cs b/build/tasks/Utilities/KoreBuildErrors.cs new file mode 100644 index 0000000000..7751985c6f --- /dev/null +++ b/build/tasks/Utilities/KoreBuildErrors.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace RepoTasks.Utilities +{ + public static class KoreBuildErrors + { + public const string Prefix = "KRB"; + + // Typically used in repos in Directory.Build.targets + public const int PackagesHaveNotYetBeenPinned = 1001; + + // Warnings + public const int DotNetAssetVersionIsFloating = 2000; + public const int RepoVersionDoesNotMatchProjectVersion = 2001; + public const int RepoPackageVersionDoesNotMatchProjectPackageVersion = 2002; + public const int DuplicatePackageReference = 2003; + + // NuGet errors + public const int InvalidNuspecFile = 4001; + public const int PackageReferenceHasVersion = 4002; + public const int DotNetCliReferenceReferenceHasVersion = 4003; + public const int PackageVersionNotFoundInLineup = 4004; + + // Other unknown errors + public const int PolicyFailedToApply = 5000; + public const int UnknownPolicyType = 5001; + } +} diff --git a/build/tasks/Utilities/LoggingExtensions.cs b/build/tasks/Utilities/LoggingExtensions.cs new file mode 100644 index 0000000000..833edb1aa4 --- /dev/null +++ b/build/tasks/Utilities/LoggingExtensions.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Build.Utilities; + +namespace RepoTasks.Utilities +{ + public static class LoggingExtensions + { + public static void LogKoreBuildError(this TaskLoggingHelper logger, int code, string message, params object[] messageArgs) + => LogKoreBuildError(logger, null, code, message, messageArgs: messageArgs); + + public static void LogKoreBuildError(this TaskLoggingHelper logger, string filename, int code, string message, params object[] messageArgs) + { + logger.LogError(null, KoreBuildErrors.Prefix + code, null, filename, 0, 0, 0, 0, message, messageArgs: messageArgs); + } + + public static void LogKoreBuildWarning(this TaskLoggingHelper logger, int code, string message, params object[] messageArgs) + => LogKoreBuildWarning(logger, null, code, message, messageArgs: messageArgs); + + public static void LogKoreBuildWarning(this TaskLoggingHelper logger, string filename, int code, string message, params object[] messageArgs) + { + logger.LogWarning(null, KoreBuildErrors.Prefix + code, null, filename, 0, 0, 0, 0, message, messageArgs: messageArgs); + } + } +} diff --git a/build/tasks/Utilities/MSBuildListSplitter.cs b/build/tasks/Utilities/MSBuildListSplitter.cs new file mode 100644 index 0000000000..de4ff8724b --- /dev/null +++ b/build/tasks/Utilities/MSBuildListSplitter.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace RepoTasks.Utilities +{ + internal static class MSBuildListSplitter + { + private static readonly char[] SemiColon = { ';' }; + + public static IEnumerable SplitItemList(string value) + { + return string.IsNullOrEmpty(value) + ? Enumerable.Empty() + : value.Split(SemiColon, StringSplitOptions.RemoveEmptyEntries); + } + + public static Dictionary GetNamedProperties(string input) + { + var values = new Dictionary(); + if (string.IsNullOrEmpty(input)) + { + return values; + } + + foreach (var item in input.Split(SemiColon, StringSplitOptions.RemoveEmptyEntries)) + { + var splitIdx = item.IndexOf('='); + if (splitIdx <= 0) + { + continue; + } + + var key = item.Substring(0, splitIdx).Trim(); + var value = item.Substring(splitIdx + 1); + values[key] = value; + } + + return values; + } + } +} diff --git a/build/tasks/VerifyBuildGraph.cs b/build/tasks/VerifyBuildGraph.cs deleted file mode 100644 index 5e69965aac..0000000000 --- a/build/tasks/VerifyBuildGraph.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace RepoTasks -{ - public class VerifyBuildGraph : Task - { - /// - /// Repositories that we are building new versions of. - /// - [Required] - public ITaskItem[] BuildRepositories { get; set; } - - /// - /// Repos that have already been build and released. We don't compile and build them, - /// but we still want to be sure their packages are accounted for in our graph calculations. - /// - public ITaskItem[] NoBuildRepositories { get; set; } - - /// - /// New packages we are compiling. Used in the pin tool. - /// - [Output] - public ITaskItem[] PackagesToBeProduced { get; set; } - - public override bool Execute() - { - return false; - } - } -} From 8276bd163e0c416425fe109ba922e3c3a14a22da Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 11:12:02 -0700 Subject: [PATCH 012/115] Analyze repository build order from AnalyzeBuildGraph --- build/tasks/AnalyzeBuildGraph.cs | 77 +++++++++++++++++++++++++++- build/tasks/BuildGraph/Project.cs | 2 +- build/tasks/BuildGraph/Repository.cs | 4 +- 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index a135e66568..1c94093ac0 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -5,9 +5,11 @@ using System; using System.Collections; using System.Collections.Generic; using System.Linq; +using System.IO; using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; +using RepoTools.BuildGraph; using RepoTasks.ProjectModel; using RepoTasks.Utilities; @@ -26,6 +28,8 @@ namespace RepoTasks [Required] public string Properties { get; set; } + public string StartGraphAt { get; set; } + /// /// New packages we are compiling. Used in the pin tool. /// @@ -58,7 +62,21 @@ namespace RepoTasks return false; } - PackagesProduced = solutions + EnsureConsistentGraph(solutions); + PackagesProduced = GetPackagesProduced(solutions); + RepositoryBuildOrder = GetRepositoryBuildOrder(solutions.Where(s => s.ShouldBuild)); + + return !Log.HasLoggedErrors; + } + + private void EnsureConsistentGraph(IEnumerable solutions) + { + // TODO + } + + private ITaskItem[] GetPackagesProduced(IEnumerable solutions) + { + return solutions .Where(s => s.ShouldBuild) .SelectMany(p => p.Projects) .Where(p => p.IsPackable) @@ -67,8 +85,63 @@ namespace RepoTasks ["Version"] = p.PackageVersion })) .ToArray(); + } - return !Log.HasLoggedErrors; + private ITaskItem[] GetRepositoryBuildOrder(IEnumerable solutions) + { + var repositories = solutions.Select(s => + { + var repoName = Path.GetFileName(Path.GetDirectoryName(s.FullPath)); + var repo = new Repository(repoName); + repo.Projects = s.Projects + .Where(p => p.IsPackable) + .Select(p => + new Project(p.PackageId) + { + Repository = repo, + PackageReferences = new HashSet(p.Frameworks.SelectMany(f => f.Dependencies.Keys), StringComparer.OrdinalIgnoreCase), + }) + .ToList(); + repo.SupportProjects = s.Projects + .Where(p => !p.IsPackable) + .Select(p => + new Project(p.PackageId) + { + Repository = repo, + PackageReferences = new HashSet(p.Frameworks.SelectMany(f => f.Dependencies.Keys), StringComparer.OrdinalIgnoreCase), + }) + .ToList(); + return repo; + }).ToList(); + + var graph = GraphBuilder.Generate(repositories, StartGraphAt, Log); + var repositoriesWithOrder = new List<(ITaskItem repository, int order)>(); + foreach (var repository in repositories) + { + var graphNodeRepository = graph.FirstOrDefault(g => g.Repository.Name == repository.Name); + if (graphNodeRepository == null) + { + // StartGraphAt was specified so the graph is incomplete. + continue; + } + + var order = TopologicalSort.GetOrder(graphNodeRepository); + var repositoryTaskItem = new TaskItem(repository.Name); + repositoryTaskItem.SetMetadata("Order", order.ToString()); + repositoriesWithOrder.Add((repositoryTaskItem, order)); + } + + Log.LogMessage(MessageImportance.High, "Repository build order:"); + foreach (var buildGroup in repositoriesWithOrder.GroupBy(r => r.order).OrderBy(g => g.Key)) + { + var buildGroupRepos = buildGroup.Select(b => b.repository.ItemSpec); + Log.LogMessage(MessageImportance.High, $"{buildGroup.Key.ToString().PadLeft(2, ' ')}: {string.Join(", ", buildGroupRepos)}"); + } + + return repositoriesWithOrder + .OrderBy(r => r.order) + .Select(r => r.repository) + .ToArray(); } } } diff --git a/build/tasks/BuildGraph/Project.cs b/build/tasks/BuildGraph/Project.cs index 72b5eee254..ad03468a61 100644 --- a/build/tasks/BuildGraph/Project.cs +++ b/build/tasks/BuildGraph/Project.cs @@ -23,6 +23,6 @@ namespace RepoTools.BuildGraph public Repository Repository { get; set; } - public ISet PackageReferences { get; } = new HashSet(StringComparer.OrdinalIgnoreCase); + public ISet PackageReferences { get; set; } = new HashSet(StringComparer.OrdinalIgnoreCase); } } diff --git a/build/tasks/BuildGraph/Repository.cs b/build/tasks/BuildGraph/Repository.cs index c1e3e5c619..ed7135491c 100644 --- a/build/tasks/BuildGraph/Repository.cs +++ b/build/tasks/BuildGraph/Repository.cs @@ -22,9 +22,9 @@ namespace RepoTools.BuildGraph public string Name { get; private set; } - public IList Projects { get; } = new List(); + public IList Projects { get; set; } = new List(); - public IList SupportProjects { get; } = new List(); + public IList SupportProjects { get; set; } = new List(); public IEnumerable AllProjects => Projects.Concat(SupportProjects); From a43fb2271e88fe19034fb9b922c6637d85b8fb6a Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 13:17:49 -0700 Subject: [PATCH 013/115] Use artifact info to correctly analyze the packages produced from a repository --- build/repo.targets | 15 +++- build/tasks/AnalyzeBuildGraph.cs | 75 ++++++++++--------- build/tasks/BuildGraph/Repository.cs | 4 +- build/tasks/ProjectModel/PackageInfo.cs | 45 +++++++++++ .../tasks/ProjectModel/SolutionInfoFactory.cs | 2 +- 5 files changed, 98 insertions(+), 43 deletions(-) create mode 100644 build/tasks/ProjectModel/PackageInfo.cs diff --git a/build/repo.targets b/build/repo.targets index 0d387361d4..eca755d4ba 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -108,7 +108,14 @@ - + + + + + - + - - + diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 1c94093ac0..401b2c21b0 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -9,6 +9,7 @@ using System.IO; using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; +using NuGet.Frameworks; using RepoTools.BuildGraph; using RepoTasks.ProjectModel; using RepoTasks.Utilities; @@ -25,17 +26,14 @@ namespace RepoTasks [Required] public ITaskItem[] Solutions { get; set; } + [Required] + public ITaskItem[] Artifacts { get; set; } + [Required] public string Properties { get; set; } public string StartGraphAt { get; set; } - /// - /// New packages we are compiling. Used in the pin tool. - /// - [Output] - public ITaskItem[] PackagesProduced { get; set; } - /// /// The order in which to build repositories /// @@ -49,6 +47,8 @@ namespace RepoTasks public override bool Execute() { + var packageArtifacts = GetPackageInfo(Artifacts); + var factory = new SolutionInfoFactory(Log, BuildEngine5); var props = MSBuildListSplitter.GetNamedProperties(Properties); @@ -63,8 +63,7 @@ namespace RepoTasks } EnsureConsistentGraph(solutions); - PackagesProduced = GetPackagesProduced(solutions); - RepositoryBuildOrder = GetRepositoryBuildOrder(solutions.Where(s => s.ShouldBuild)); + RepositoryBuildOrder = GetRepositoryBuildOrder(packageArtifacts, solutions.Where(s => s.ShouldBuild)); return !Log.HasLoggedErrors; } @@ -74,43 +73,47 @@ namespace RepoTasks // TODO } - private ITaskItem[] GetPackagesProduced(IEnumerable solutions) + private IEnumerable<(PackageInfo, string repoName)> GetPackageInfo(IEnumerable items) { - return solutions - .Where(s => s.ShouldBuild) - .SelectMany(p => p.Projects) - .Where(p => p.IsPackable) - .Select(p => new TaskItem(p.PackageId, new Hashtable + return items + .Where(a => "NuGetPackage".Equals(a.GetMetadata("ArtifactType"), StringComparison.OrdinalIgnoreCase)) + .Select(a => { - ["Version"] = p.PackageVersion - })) - .ToArray(); + var info = new PackageInfo( + a.GetMetadata("PackageId"), + a.GetMetadata("Version"), + string.IsNullOrEmpty(a.GetMetadata("TargetFramework")) + ? MSBuildListSplitter.SplitItemList(a.GetMetadata("TargetFramework")).Select(s => NuGetFramework.Parse(s)).ToArray() + : new [] { NuGetFramework.Parse(a.GetMetadata("TargetFramework")) }, + Path.GetDirectoryName(a.ItemSpec), + a.GetMetadata("PackageType")); + + var repoName = Path.GetFileName(a.GetMetadata("RepositoryRoot").TrimEnd(new [] { '\\', '/' })); + return (info, repoName); + }); } - private ITaskItem[] GetRepositoryBuildOrder(IEnumerable solutions) + private ITaskItem[] GetRepositoryBuildOrder(IEnumerable<(PackageInfo pkg, string repoName)> artifacts, IEnumerable solutions) { var repositories = solutions.Select(s => { var repoName = Path.GetFileName(Path.GetDirectoryName(s.FullPath)); var repo = new Repository(repoName); - repo.Projects = s.Projects - .Where(p => p.IsPackable) - .Select(p => - new Project(p.PackageId) - { - Repository = repo, - PackageReferences = new HashSet(p.Frameworks.SelectMany(f => f.Dependencies.Keys), StringComparer.OrdinalIgnoreCase), - }) - .ToList(); - repo.SupportProjects = s.Projects - .Where(p => !p.IsPackable) - .Select(p => - new Project(p.PackageId) - { - Repository = repo, - PackageReferences = new HashSet(p.Frameworks.SelectMany(f => f.Dependencies.Keys), StringComparer.OrdinalIgnoreCase), - }) - .ToList(); + var packages = artifacts.Where(a => a.repoName.Equals(repoName, StringComparison.OrdinalIgnoreCase)).ToList(); + + foreach (var proj in s.Projects) + { + var projectGroup = packages.Any(p => p.pkg.Id == proj.PackageId) + ? repo.Projects + : repo.SupportProjects; + + projectGroup.Add(new Project(proj.PackageId) + { + Repository = repo, + PackageReferences = new HashSet(proj.Frameworks.SelectMany(f => f.Dependencies.Keys), StringComparer.OrdinalIgnoreCase), + }); + } + return repo; }).ToList(); diff --git a/build/tasks/BuildGraph/Repository.cs b/build/tasks/BuildGraph/Repository.cs index ed7135491c..c1e3e5c619 100644 --- a/build/tasks/BuildGraph/Repository.cs +++ b/build/tasks/BuildGraph/Repository.cs @@ -22,9 +22,9 @@ namespace RepoTools.BuildGraph public string Name { get; private set; } - public IList Projects { get; set; } = new List(); + public IList Projects { get; } = new List(); - public IList SupportProjects { get; set; } = new List(); + public IList SupportProjects { get; } = new List(); public IEnumerable AllProjects => Projects.Concat(SupportProjects); diff --git a/build/tasks/ProjectModel/PackageInfo.cs b/build/tasks/ProjectModel/PackageInfo.cs new file mode 100644 index 0000000000..7293546c61 --- /dev/null +++ b/build/tasks/ProjectModel/PackageInfo.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using NuGet.Frameworks; + +namespace RepoTasks.ProjectModel +{ + internal class PackageInfo + { + public PackageInfo(string id, + string version, + IReadOnlyList frameworks, + string source, + string packageType = "Dependency") + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(nameof(id)); + } + + if (string.IsNullOrEmpty(version)) + { + throw new ArgumentException(nameof(version)); + } + + Id = id; + Version = version; + Frameworks = frameworks; + PackageType = packageType; + Source = source; + } + + public string Id { get; } + public string Version { get; } + public string PackageType { get; } + /// + /// Can be a https feed or a file path. May be null. + /// + public string Source { get; } + public IReadOnlyList Frameworks { get; } + } +} diff --git a/build/tasks/ProjectModel/SolutionInfoFactory.cs b/build/tasks/ProjectModel/SolutionInfoFactory.cs index 72601be2dc..9ea368cc0a 100644 --- a/build/tasks/ProjectModel/SolutionInfoFactory.cs +++ b/build/tasks/ProjectModel/SolutionInfoFactory.cs @@ -98,7 +98,7 @@ namespace RepoTasks.ProjectModel }); timer.Stop(); - _logger.LogMessage(MessageImportance.Normal, $"Finished design-time build in {timer.ElapsedMilliseconds}ms"); + _logger.LogMessage(MessageImportance.High, $"Finished design-time build in {timer.ElapsedMilliseconds}ms"); return solutions.ToArray(); } From 4220c516bc0f252cd891217a70064bc9bfd2bc3a Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 14:52:47 -0700 Subject: [PATCH 014/115] Ensure PackageReference versions are consistent with artifacts that will be produced --- build/tasks/AnalyzeBuildGraph.cs | 66 +++++++++++++++++++++++-- build/tasks/ProjectModel/PackageInfo.cs | 5 +- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 401b2c21b0..0d56e6a62d 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -6,10 +6,12 @@ using System.Collections; using System.Collections.Generic; using System.Linq; using System.IO; +using System.Text; using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using NuGet.Frameworks; +using NuGet.Versioning; using RepoTools.BuildGraph; using RepoTasks.ProjectModel; using RepoTasks.Utilities; @@ -62,15 +64,73 @@ namespace RepoTasks return false; } - EnsureConsistentGraph(solutions); + EnsureConsistentGraph(packageArtifacts.Select(p => p.Item1), solutions); RepositoryBuildOrder = GetRepositoryBuildOrder(packageArtifacts, solutions.Where(s => s.ShouldBuild)); return !Log.HasLoggedErrors; } - private void EnsureConsistentGraph(IEnumerable solutions) + private struct VersionMismatch { - // TODO + public SolutionInfo Solution; + public ProjectInfo Project; + public string PackageId; + public string ActualVersion; + public NuGetVersion ExpectedVersion; + } + + private void EnsureConsistentGraph(IEnumerable packages, IEnumerable solutions) + { + // ensure versions cascade + var lookup = packages.ToDictionary(p => p.Id, p => p, StringComparer.OrdinalIgnoreCase); + + var inconsistentVersions = new List(); + + // holy crap, o^4 + foreach (var sln in solutions) + foreach (var proj in sln.Projects) + foreach (var tfm in proj.Frameworks) + foreach (var dependency in tfm.Dependencies) + { + if (!lookup.TryGetValue(dependency.Key, out var package)) continue; + + var refVersion = VersionRange.Parse(dependency.Value.Version); + if (refVersion.IsFloating && refVersion.Float.Satisfies(package.Version)) + continue; + else if (package.Version.Equals(refVersion)) + continue; + + inconsistentVersions.Add(new VersionMismatch + { + Solution = sln, + Project = proj, + PackageId = dependency.Key, + ActualVersion = dependency.Value.Version, + ExpectedVersion = package.Version, + }); + } + + if (inconsistentVersions.Count != 0) + { + var sb = new StringBuilder(); + sb.AppendLine(); + sb.AppendLine($"Repos are inconsistent. The following projects have PackageReferences that should be updated"); + foreach (var sln in inconsistentVersions.GroupBy(p => p.Solution.FullPath)) + { + sb.AppendLine($" - {Path.GetFileName(sln.Key)}"); + foreach (var proj in sln.GroupBy(p => p.Project.FullPath)) + { + sb.AppendLine($" - {Path.GetFileName(proj.Key)}"); + foreach (var m in proj) + { + sb.AppendLine($" - {m.PackageId}/{{{m.ActualVersion} => {m.ExpectedVersion}}}"); + } + } + } + sb.AppendLine(); + Log.LogMessage(MessageImportance.High, sb.ToString()); + Log.LogError("Package versions are inconsistent. See build log for details."); + } } private IEnumerable<(PackageInfo, string repoName)> GetPackageInfo(IEnumerable items) diff --git a/build/tasks/ProjectModel/PackageInfo.cs b/build/tasks/ProjectModel/PackageInfo.cs index 7293546c61..af1976686e 100644 --- a/build/tasks/ProjectModel/PackageInfo.cs +++ b/build/tasks/ProjectModel/PackageInfo.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; using NuGet.Frameworks; +using NuGet.Versioning; namespace RepoTasks.ProjectModel { @@ -27,14 +28,14 @@ namespace RepoTasks.ProjectModel } Id = id; - Version = version; + Version = NuGetVersion.Parse(version); Frameworks = frameworks; PackageType = packageType; Source = source; } public string Id { get; } - public string Version { get; } + public NuGetVersion Version { get; } public string PackageType { get; } /// /// Can be a https feed or a file path. May be null. From a0547c7b9fc2532045578fa85b47e48ede81a7f9 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 16:10:24 -0700 Subject: [PATCH 015/115] Generate a file that contains all of the dependencies and their versions --- build/repo.targets | 8 +++ build/tasks/AnalyzeBuildGraph.cs | 42 ++++--------- build/tasks/GenerateLineup.cs | 85 +++++++++++++++++++++++++++ build/tasks/RepoTasks.tasks | 1 + build/tasks/Utilities/ArtifactInfo.cs | 61 +++++++++++++++++++ korebuild-lock.txt | 4 +- 6 files changed, 169 insertions(+), 32 deletions(-) create mode 100644 build/tasks/GenerateLineup.cs create mode 100644 build/tasks/Utilities/ArtifactInfo.cs diff --git a/build/repo.targets b/build/repo.targets index eca755d4ba..a07cbf6e65 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -146,9 +146,17 @@ Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber)"> + + + + diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 0d56e6a62d..8116088414 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -49,7 +49,9 @@ namespace RepoTasks public override bool Execute() { - var packageArtifacts = GetPackageInfo(Artifacts); + var packageArtifacts = Artifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact); var factory = new SolutionInfoFactory(Log, BuildEngine5); var props = MSBuildListSplitter.GetNamedProperties(Properties); @@ -64,7 +66,7 @@ namespace RepoTasks return false; } - EnsureConsistentGraph(packageArtifacts.Select(p => p.Item1), solutions); + EnsureConsistentGraph(packageArtifacts, solutions); RepositoryBuildOrder = GetRepositoryBuildOrder(packageArtifacts, solutions.Where(s => s.ShouldBuild)); return !Log.HasLoggedErrors; @@ -79,10 +81,10 @@ namespace RepoTasks public NuGetVersion ExpectedVersion; } - private void EnsureConsistentGraph(IEnumerable packages, IEnumerable solutions) + private void EnsureConsistentGraph(IEnumerable packages, IEnumerable solutions) { // ensure versions cascade - var lookup = packages.ToDictionary(p => p.Id, p => p, StringComparer.OrdinalIgnoreCase); + var lookup = packages.ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); var inconsistentVersions = new List(); @@ -95,9 +97,9 @@ namespace RepoTasks if (!lookup.TryGetValue(dependency.Key, out var package)) continue; var refVersion = VersionRange.Parse(dependency.Value.Version); - if (refVersion.IsFloating && refVersion.Float.Satisfies(package.Version)) + if (refVersion.IsFloating && refVersion.Float.Satisfies(package.PackageInfo.Version)) continue; - else if (package.Version.Equals(refVersion)) + else if (package.PackageInfo.Version.Equals(refVersion)) continue; inconsistentVersions.Add(new VersionMismatch @@ -106,7 +108,7 @@ namespace RepoTasks Project = proj, PackageId = dependency.Key, ActualVersion = dependency.Value.Version, - ExpectedVersion = package.Version, + ExpectedVersion = package.PackageInfo.Version, }); } @@ -133,37 +135,17 @@ namespace RepoTasks } } - private IEnumerable<(PackageInfo, string repoName)> GetPackageInfo(IEnumerable items) - { - return items - .Where(a => "NuGetPackage".Equals(a.GetMetadata("ArtifactType"), StringComparison.OrdinalIgnoreCase)) - .Select(a => - { - var info = new PackageInfo( - a.GetMetadata("PackageId"), - a.GetMetadata("Version"), - string.IsNullOrEmpty(a.GetMetadata("TargetFramework")) - ? MSBuildListSplitter.SplitItemList(a.GetMetadata("TargetFramework")).Select(s => NuGetFramework.Parse(s)).ToArray() - : new [] { NuGetFramework.Parse(a.GetMetadata("TargetFramework")) }, - Path.GetDirectoryName(a.ItemSpec), - a.GetMetadata("PackageType")); - - var repoName = Path.GetFileName(a.GetMetadata("RepositoryRoot").TrimEnd(new [] { '\\', '/' })); - return (info, repoName); - }); - } - - private ITaskItem[] GetRepositoryBuildOrder(IEnumerable<(PackageInfo pkg, string repoName)> artifacts, IEnumerable solutions) + private ITaskItem[] GetRepositoryBuildOrder(IEnumerable artifacts, IEnumerable solutions) { var repositories = solutions.Select(s => { var repoName = Path.GetFileName(Path.GetDirectoryName(s.FullPath)); var repo = new Repository(repoName); - var packages = artifacts.Where(a => a.repoName.Equals(repoName, StringComparison.OrdinalIgnoreCase)).ToList(); + var packages = artifacts.Where(a => a.RepoName.Equals(repoName, StringComparison.OrdinalIgnoreCase)).ToList(); foreach (var proj in s.Projects) { - var projectGroup = packages.Any(p => p.pkg.Id == proj.PackageId) + var projectGroup = packages.Any(p => p.PackageInfo.Id == proj.PackageId) ? repo.Projects : repo.SupportProjects; diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs new file mode 100644 index 0000000000..92c2de9eb9 --- /dev/null +++ b/build/tasks/GenerateLineup.cs @@ -0,0 +1,85 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Xml; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using RepoTasks.ProjectModel; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class GenerateLineup : Task + { + [Required] + public ITaskItem[] Artifacts { get; set; } + + [Required] + public string OutputPath { get; set; } + + public bool UseFloatingVersions { get; set; } + + public string BuildNumber { get; set; } + + public override bool Execute() + { + OutputPath = OutputPath.Replace('\\', '/'); + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + if (UseFloatingVersions && string.IsNullOrEmpty(BuildNumber)) + { + Log.LogWarning("Cannot compute floating versions when BuildNumber is not specified"); + } + + var items = new XElement("ItemGroup"); + var root = new XElement("Project", items); + var doc = new XDocument(root); + + var packages = new List(); + + foreach (var item in Artifacts) + { + var info = ArtifactInfo.Parse(item); + switch (info) + { + case ArtifactInfo.Package pkg when (!pkg.IsSymbolsArtifact): + packages.Add(pkg.PackageInfo); + break; + } + } + + foreach (var pkg in packages.OrderBy(i => i.Id)) + { + var version = pkg.Version.ToString(); + if (UseFloatingVersions && version.EndsWith(BuildNumber)) + { + version = version.Substring(0, version.Length - BuildNumber.Length) + "*"; + } + + var refType = "DotNetCliTool".Equals(pkg.Id, StringComparison.OrdinalIgnoreCase) + ? "DotNetCliToolReference" + : "PackageReference"; + + items.Add(new XElement(refType, + new XAttribute("Update", pkg.Id), + new XAttribute("Version", version))); + } + + var settings = new XmlWriterSettings + { + OmitXmlDeclaration = true, + Indent = true, + }; + using (var writer = XmlWriter.Create(OutputPath, settings)) + { + doc.Save(writer); + } + return true; + } + } +} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 4026e9810b..d6c6ae3488 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -5,5 +5,6 @@ + diff --git a/build/tasks/Utilities/ArtifactInfo.cs b/build/tasks/Utilities/ArtifactInfo.cs new file mode 100644 index 0000000000..f3bfa91ced --- /dev/null +++ b/build/tasks/Utilities/ArtifactInfo.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using NuGet.Frameworks; +using Microsoft.Build.Framework; +using RepoTasks.ProjectModel; + +namespace RepoTasks.Utilities +{ + internal abstract class ArtifactInfo + { + public static ArtifactInfo Parse(ITaskItem item) + { + ArtifactInfo info; + switch (item.GetMetadata("ArtifactType").ToLowerInvariant()) + { + case "nugetpackage": + info = new Package { PackageInfo = GetPackageInfo(item) }; + break; + case "nugetsymbolspackage": + info = new Package { PackageInfo = GetPackageInfo(item), IsSymbolsArtifact = true }; + break; + default: + throw new InvalidDataException($"Unrecognized artifact type: {item.GetMetadata("ArtifactType")} for artifact {item.ItemSpec}"); + } + + info.RepositoryRoot = item.GetMetadata("RepositoryRoot")?.TrimEnd(new [] { '\\', '/' }); + + if (!string.IsNullOrEmpty(info.RepositoryRoot)) + { + info.RepoName = Path.GetFileName(info.RepositoryRoot); + } + + return info; + } + + public string RepositoryRoot { get; private set; } + public string RepoName { get; private set; } + + public class Package : ArtifactInfo + { + public PackageInfo PackageInfo { get; set; } + public bool IsSymbolsArtifact { get; set; } + } + + private static PackageInfo GetPackageInfo(ITaskItem item) + { + return new PackageInfo( + item.GetMetadata("PackageId"), + item.GetMetadata("Version"), + string.IsNullOrEmpty(item.GetMetadata("TargetFramework")) + ? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => NuGetFramework.Parse(s)).ToArray() + : new [] { NuGetFramework.Parse(item.GetMetadata("TargetFramework")) }, + Path.GetDirectoryName(item.ItemSpec), + item.GetMetadata("PackageType")); + } + } +} diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 7b57925f8f..73a677fef7 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.0-rtm-15492 -commithash:9ea72bcf88063cee9afbe53835681702e2efd720 +version:2.0.2-alpha-15495 +commithash:60338bc298ac6f70287e89256470cccf24c9bd80 From 479e37d7ccb86322305252d909b3f46ad8669648 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 16:14:25 -0700 Subject: [PATCH 016/115] Include Identity in the list of repos that are being patched --- build/Repositories.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Repositories.props b/build/Repositories.props index 1bdfb82212..81b149aa3d 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -2,6 +2,7 @@ + @@ -11,7 +12,6 @@ by the patch update and do not need updating. --> - From 7568441acc36194c10bba2bd02d2789104a182d3 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 17:45:55 -0700 Subject: [PATCH 017/115] Update analysis to account for non-shipping package references (samples, tests) --- build/repo.targets | 15 ++++++++++-- build/tasks/AnalyzeBuildGraph.cs | 39 ++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index a07cbf6e65..61fb75cef5 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -51,7 +51,7 @@ - <_CloneRepositories Include="@(Repository);@(VerifyRepository)" /> + <_CloneRepositories Include="@(Repository);@(ShippedRepository)" /> <_CloneRepository Include="$(MSBuildProjectFullPath)"> CloneRepository=%(_CloneRepositories.Identity); @@ -123,9 +123,19 @@ + + + + @@ -143,6 +153,7 @@ diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 8116088414..b9c2982f5e 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -31,6 +31,10 @@ namespace RepoTasks [Required] public ITaskItem[] Artifacts { get; set; } + // Artifacts that already shipped from repos + [Required] + public ITaskItem[] ShippedArtifacts { get; set; } + [Required] public string Properties { get; set; } @@ -83,10 +87,17 @@ namespace RepoTasks private void EnsureConsistentGraph(IEnumerable packages, IEnumerable solutions) { + var shippedPackageMap = ShippedArtifacts + .Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact) + .ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); + // ensure versions cascade - var lookup = packages.ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); + var buildPackageMap = packages.ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); var inconsistentVersions = new List(); + var reposThatShouldPatch = new HashSet(); // holy crap, o^4 foreach (var sln in solutions) @@ -94,7 +105,7 @@ namespace RepoTasks foreach (var tfm in proj.Frameworks) foreach (var dependency in tfm.Dependencies) { - if (!lookup.TryGetValue(dependency.Key, out var package)) continue; + if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) continue; var refVersion = VersionRange.Parse(dependency.Value.Version); if (refVersion.IsFloating && refVersion.Float.Satisfies(package.PackageInfo.Version)) @@ -102,6 +113,19 @@ namespace RepoTasks else if (package.PackageInfo.Version.Equals(refVersion)) continue; + if (!sln.ShouldBuild) + { + if (!shippedPackageMap.TryGetValue(proj.PackageId, out _)) + { + Log.LogMessage(MessageImportance.Normal, $"Detected inconsistent in a sample or test project {proj.FullPath}"); + continue; + } + else + { + reposThatShouldPatch.Add(Path.GetFileName(Path.GetDirectoryName(sln.FullPath))); + } + } + inconsistentVersions.Add(new VersionMismatch { Solution = sln, @@ -119,13 +143,13 @@ namespace RepoTasks sb.AppendLine($"Repos are inconsistent. The following projects have PackageReferences that should be updated"); foreach (var sln in inconsistentVersions.GroupBy(p => p.Solution.FullPath)) { - sb.AppendLine($" - {Path.GetFileName(sln.Key)}"); + sb.Append(" - ").AppendLine(Path.GetFileName(sln.Key)); foreach (var proj in sln.GroupBy(p => p.Project.FullPath)) { - sb.AppendLine($" - {Path.GetFileName(proj.Key)}"); + sb.Append(" - ").AppendLine(Path.GetFileName(proj.Key)); foreach (var m in proj) { - sb.AppendLine($" - {m.PackageId}/{{{m.ActualVersion} => {m.ExpectedVersion}}}"); + sb.AppendLine($" + {m.PackageId}/{{{m.ActualVersion} => {m.ExpectedVersion}}}"); } } } @@ -133,6 +157,11 @@ namespace RepoTasks Log.LogMessage(MessageImportance.High, sb.ToString()); Log.LogError("Package versions are inconsistent. See build log for details."); } + + foreach (var repo in reposThatShouldPatch) + { + Log.LogError($"{repo} should not be a 'ShippedRepository'. Version changes in other repositories mean it should be patched to perserve cascading version upgrades."); + } } private ITaskItem[] GetRepositoryBuildOrder(IEnumerable artifacts, IEnumerable solutions) From 997be52987c99e704f1f7e7bf79a70cea8ff014d Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 15 Sep 2017 18:03:29 -0700 Subject: [PATCH 018/115] Add repos that should be patched due to cascading versions --- build/Repositories.props | 81 ++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index 81b149aa3d..b95b7374c6 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -1,8 +1,15 @@ + + + + + + + @@ -11,45 +18,39 @@ Build tools will *verify* that these repos will be unaffected by the patch update and do not need updating. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From dc50526a6d60a7ec4bb38779173a4f26f065eb14 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 09:37:12 -0700 Subject: [PATCH 019/115] Rename target, remove unused parameter, and add comments to the AnalyzeBuildGraph task --- build/repo.targets | 4 ++-- build/tasks/AnalyzeBuildGraph.cs | 8 +++++--- build/tasks/GenerateLineup.cs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index 61fb75cef5..ad0354952e 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -106,7 +106,7 @@ + DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;ComputeGraph;_BuildRepositories" /> - + /// The order in which to build repositories /// @@ -105,7 +103,11 @@ namespace RepoTasks foreach (var tfm in proj.Frameworks) foreach (var dependency in tfm.Dependencies) { - if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) continue; + if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) + { + // this dependency is not a PackageReference to something that we build in Universe + continue; + } var refVersion = VersionRange.Parse(dependency.Value.Version); if (refVersion.IsFloating && refVersion.Float.Satisfies(package.PackageInfo.Version)) diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs index 92c2de9eb9..e2baf809de 100644 --- a/build/tasks/GenerateLineup.cs +++ b/build/tasks/GenerateLineup.cs @@ -61,7 +61,7 @@ namespace RepoTasks version = version.Substring(0, version.Length - BuildNumber.Length) + "*"; } - var refType = "DotNetCliTool".Equals(pkg.Id, StringComparison.OrdinalIgnoreCase) + var refType = "DotNetCliTool".Equals(pkg.PackageType, StringComparison.OrdinalIgnoreCase) ? "DotNetCliToolReference" : "PackageReference"; From a10ba83751cab0af7dc1d2a032959f5ae84824f1 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 10:33:48 -0700 Subject: [PATCH 020/115] Add /t:UpdateRepoLineups which will automatically update the build/dependencies.targets file with appropriate versions --- build/repo.targets | 12 +++++++++++- build/tasks/AnalyzeBuildGraph.cs | 2 ++ build/tasks/GenerateLineup.cs | 11 ++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index ad0354952e..7d105c3107 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -154,11 +154,21 @@ Solutions="@(Solution)" Artifacts="@(ArtifactInfo)" ShippedArtifacts="@(ShippedArtifactInfo)" + StartGraphAt="$(BuildGraphOf)" Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber)"> + + + + @@ -166,7 +176,7 @@ diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 49d8eed87d..ebfe3ea108 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -38,6 +38,8 @@ namespace RepoTasks [Required] public string Properties { get; set; } + public string StartGraphAt { get; set; } + /// /// The order in which to build repositories /// diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs index e2baf809de..544284e409 100644 --- a/build/tasks/GenerateLineup.cs +++ b/build/tasks/GenerateLineup.cs @@ -22,6 +22,9 @@ namespace RepoTasks [Required] public string OutputPath { get; set; } + // Can be set to filter the lists of packages when produce a list for a specific repository + public string Repository { get; set; } + public bool UseFloatingVersions { get; set; } public string BuildNumber { get; set; } @@ -48,7 +51,12 @@ namespace RepoTasks switch (info) { case ArtifactInfo.Package pkg when (!pkg.IsSymbolsArtifact): - packages.Add(pkg.PackageInfo); + // TODO filter this list based on topological sort info + if (string.IsNullOrEmpty(Repository) + || !Repository.Equals(pkg.RepoName, StringComparison.OrdinalIgnoreCase)) + { + packages.Add(pkg.PackageInfo); + } break; } } @@ -77,6 +85,7 @@ namespace RepoTasks }; using (var writer = XmlWriter.Create(OutputPath, settings)) { + Log.LogMessage(MessageImportance.High, $"Generate {OutputPath}"); doc.Save(writer); } return true; From 51dbdbeac2f03730ff48eac93de7c1a1c4b74bc1 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 11:15:45 -0700 Subject: [PATCH 021/115] Code cleanup --- build/tasks/AnalyzeBuildGraph.cs | 47 +++++++++++++++++----------- build/tasks/BuildGraph/Repository.cs | 2 ++ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index ebfe3ea108..c4f587cc75 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -99,10 +99,10 @@ namespace RepoTasks var inconsistentVersions = new List(); var reposThatShouldPatch = new HashSet(); - // holy crap, o^4 - foreach (var sln in solutions) - foreach (var proj in sln.Projects) - foreach (var tfm in proj.Frameworks) + // TODO cleanup the 4-deep nested loops + foreach (var solution in solutions) + foreach (var project in solution.Projects) + foreach (var tfm in project.Frameworks) foreach (var dependency in tfm.Dependencies) { if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) @@ -113,20 +113,24 @@ namespace RepoTasks var refVersion = VersionRange.Parse(dependency.Value.Version); if (refVersion.IsFloating && refVersion.Float.Satisfies(package.PackageInfo.Version)) - continue; - else if (package.PackageInfo.Version.Equals(refVersion)) - continue; - - if (!sln.ShouldBuild) { - if (!shippedPackageMap.TryGetValue(proj.PackageId, out _)) + continue; + } + else if (package.PackageInfo.Version.Equals(refVersion)) + { + continue; + } + + if (!solution.ShouldBuild) + { + if (!shippedPackageMap.TryGetValue(project.PackageId, out _)) { - Log.LogMessage(MessageImportance.Normal, $"Detected inconsistent in a sample or test project {proj.FullPath}"); + Log.LogMessage(MessageImportance.Normal, $"Detected inconsistent in a sample or test project {project.FullPath}"); continue; } else { - reposThatShouldPatch.Add(Path.GetFileName(Path.GetDirectoryName(sln.FullPath))); + reposThatShouldPatch.Add(Path.GetFileName(Path.GetDirectoryName(solution.FullPath))); } } @@ -145,15 +149,15 @@ namespace RepoTasks var sb = new StringBuilder(); sb.AppendLine(); sb.AppendLine($"Repos are inconsistent. The following projects have PackageReferences that should be updated"); - foreach (var sln in inconsistentVersions.GroupBy(p => p.Solution.FullPath)) + foreach (var solution in inconsistentVersions.GroupBy(p => p.Solution.FullPath)) { - sb.Append(" - ").AppendLine(Path.GetFileName(sln.Key)); - foreach (var proj in sln.GroupBy(p => p.Project.FullPath)) + sb.Append(" - ").AppendLine(Path.GetFileName(solution.Key)); + foreach (var project in solution.GroupBy(p => p.Project.FullPath)) { - sb.Append(" - ").AppendLine(Path.GetFileName(proj.Key)); - foreach (var m in proj) + sb.Append(" - ").AppendLine(Path.GetFileName(project.Key)); + foreach (var mismatchedReference in project) { - sb.AppendLine($" + {m.PackageId}/{{{m.ActualVersion} => {m.ExpectedVersion}}}"); + sb.AppendLine($" + {mismatchedReference.PackageId}/{{{mismatchedReference.ActualVersion} => {mismatchedReference.ExpectedVersion}}}"); } } } @@ -173,7 +177,11 @@ namespace RepoTasks var repositories = solutions.Select(s => { var repoName = Path.GetFileName(Path.GetDirectoryName(s.FullPath)); - var repo = new Repository(repoName); + var repo = new Repository(repoName) + { + RootDir = Path.GetDirectoryName(s.FullPath) + }; + var packages = artifacts.Where(a => a.RepoName.Equals(repoName, StringComparison.OrdinalIgnoreCase)).ToList(); foreach (var proj in s.Projects) @@ -206,6 +214,7 @@ namespace RepoTasks var order = TopologicalSort.GetOrder(graphNodeRepository); var repositoryTaskItem = new TaskItem(repository.Name); repositoryTaskItem.SetMetadata("Order", order.ToString()); + repositoryTaskItem.SetMetadata("RepositoryPath", repository.RootDir); repositoriesWithOrder.Add((repositoryTaskItem, order)); } diff --git a/build/tasks/BuildGraph/Repository.cs b/build/tasks/BuildGraph/Repository.cs index c1e3e5c619..5baaeedc92 100644 --- a/build/tasks/BuildGraph/Repository.cs +++ b/build/tasks/BuildGraph/Repository.cs @@ -22,6 +22,8 @@ namespace RepoTools.BuildGraph public string Name { get; private set; } + public string RootDir { get; set; } + public IList Projects { get; } = new List(); public IList SupportProjects { get; } = new List(); From 4924f79efc1153823d51a88488556f397bd4d80c Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 12:05:12 -0700 Subject: [PATCH 022/115] Update PackageReferences to non-floating versions when building multiple repos --- .gitignore | 2 + build/RepositoryBuild.targets | 30 ++- build/repo.targets | 17 +- build/tasks/AnalyzeBuildGraph.cs | 4 +- build/tasks/GenerateLineup.cs | 31 ++- build/tasks/PinVersions.cs | 46 ---- .../DependencyGraphSpecProvider.cs | 98 ------- .../tasks/VersionPinning/PinVersionUtility.cs | 241 ------------------ 8 files changed, 38 insertions(+), 431 deletions(-) delete mode 100644 build/tasks/PinVersions.cs delete mode 100644 build/tasks/VersionPinning/DependencyGraphSpecProvider.cs delete mode 100644 build/tasks/VersionPinning/PinVersionUtility.cs diff --git a/.gitignore b/.gitignore index f834920dd1..9aed881a4b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ _ReSharper.* *.pidb *.vspx *.psess +*.binlog +*.log packages target artifacts diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 7a329f6dca..bef1c6b508 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -17,15 +17,29 @@ false + + + + + + + + Properties="BuildGroup=%(BatchedRepository.BuildGroup);BuildNumber=$(BuildNumber);IsFinalBuild=$(IsFinalBuild);Configuration=$(Configuration)" /> - + $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash) @@ -68,16 +82,4 @@ - - - - - - - - - diff --git a/build/repo.targets b/build/repo.targets index 7d105c3107..ad9f7d7c7f 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -106,7 +106,7 @@ + DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;ComputeGraph;_BuildRepositories" /> - - - - - - - - $(BuildDir)$(_RepositoryListFileName) diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index c4f587cc75..2e9de07dd8 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -136,8 +136,8 @@ namespace RepoTasks inconsistentVersions.Add(new VersionMismatch { - Solution = sln, - Project = proj, + Solution = solution, + Project = project, PackageId = dependency.Key, ActualVersion = dependency.Value.Version, ExpectedVersion = package.PackageInfo.Version, diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs index 544284e409..1acbd7c577 100644 --- a/build/tasks/GenerateLineup.cs +++ b/build/tasks/GenerateLineup.cs @@ -25,6 +25,9 @@ namespace RepoTasks // Can be set to filter the lists of packages when produce a list for a specific repository public string Repository { get; set; } + // Items to add to the RestoreAdditionalProjectSources list in project + public ITaskItem[] RestoreAdditionalSources { get; set; } + public bool UseFloatingVersions { get; set; } public string BuildNumber { get; set; } @@ -40,25 +43,25 @@ namespace RepoTasks } var items = new XElement("ItemGroup"); - var root = new XElement("Project", items); + var props = new XElement("PropertyGroup"); + var root = new XElement("Project", props, items); var doc = new XDocument(root); + if (RestoreAdditionalSources.Length > 0) + { + var sources = RestoreAdditionalSources.Aggregate("$(RestoreAdditionalProjectSources)", (sum, piece) => sum + ";" + piece.ItemSpec); + props.Add(new XElement("RestoreAdditionalProjectSources", sources)); + } + var packages = new List(); - foreach (var item in Artifacts) + foreach (var pkg in Artifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact + && (string.IsNullOrEmpty(Repository) + || !Repository.Equals(p.RepoName, StringComparison.OrdinalIgnoreCase)))) { - var info = ArtifactInfo.Parse(item); - switch (info) - { - case ArtifactInfo.Package pkg when (!pkg.IsSymbolsArtifact): - // TODO filter this list based on topological sort info - if (string.IsNullOrEmpty(Repository) - || !Repository.Equals(pkg.RepoName, StringComparison.OrdinalIgnoreCase)) - { - packages.Add(pkg.PackageInfo); - } - break; - } + packages.Add(pkg.PackageInfo); } foreach (var pkg in packages.OrderBy(i => i.Id)) diff --git a/build/tasks/PinVersions.cs b/build/tasks/PinVersions.cs deleted file mode 100644 index b98c878173..0000000000 --- a/build/tasks/PinVersions.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Linq; -using Microsoft.Build.Framework; -using RepoTasks.VersionPinning; - -namespace RepoTasks -{ - public class PinVersions : Microsoft.Build.Utilities.Task - { - [Required] - public string BuildRepositoryRoot { get; set; } - - [Required] - public ITaskItem[] PackageSources { get; set; } - - public string GraphSpecsRoot { get; set; } - - public override bool Execute() - { - if (PackageSources?.Length == 0) - { - Log.LogError($"Missing PackageSources. At least one item source must be specified."); - return false; - } - - var graphSpecProvider = !string.IsNullOrEmpty(GraphSpecsRoot) - ? new DependencyGraphSpecProvider(GraphSpecsRoot) - : DependencyGraphSpecProvider.Default; - - using (graphSpecProvider) - { - var pinVersionUtility = new PinVersionUtility( - BuildRepositoryRoot, - PackageSources.Select(i => i.ItemSpec).ToList(), - graphSpecProvider, - Log); - pinVersionUtility.Execute(); - } - - return true; - } - } -} diff --git a/build/tasks/VersionPinning/DependencyGraphSpecProvider.cs b/build/tasks/VersionPinning/DependencyGraphSpecProvider.cs deleted file mode 100644 index 1a5e2af362..0000000000 --- a/build/tasks/VersionPinning/DependencyGraphSpecProvider.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using NuGet.ProjectModel; - -namespace RepoTasks.VersionPinning -{ - public class DependencyGraphSpecProvider : IDisposable - { - private readonly string _packageSpecDirectory; - private readonly bool _deleteSpecDirectoryOnDispose; - private readonly string _dotnetPath; - - public DependencyGraphSpecProvider(string packageSpecDirectory) - : this(packageSpecDirectory, deleteSpecDirectoryOnDispose: false) - { - } - - private DependencyGraphSpecProvider(string packageSpecDirectory, bool deleteSpecDirectoryOnDispose) - { - _packageSpecDirectory = packageSpecDirectory; - _deleteSpecDirectoryOnDispose = deleteSpecDirectoryOnDispose; - _dotnetPath = Process.GetCurrentProcess().MainModule.FileName; - } - - public static DependencyGraphSpecProvider Default { get; } = - new DependencyGraphSpecProvider(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), deleteSpecDirectoryOnDispose: true); - - public DependencyGraphSpec GetDependencyGraphSpec(string repositoryName, string solutionPath) - { - var outputFile = Path.Combine(_packageSpecDirectory, repositoryName, Path.GetFileName(solutionPath) + ".json"); - - if (!File.Exists(outputFile)) - { - RunMSBuild(solutionPath, outputFile); - } - - return DependencyGraphSpec.Load(outputFile); - } - - private void RunMSBuild(string solutionPath, string outputFile) - { - var psi = new ProcessStartInfo(_dotnetPath); - - var arguments = new List - { - "msbuild", - $"\"{solutionPath}\"", - "/t:GenerateRestoreGraphFile", - "/nologo", - "/v:q", - "/p:BuildProjectReferences=false", - $"/p:RestoreGraphOutputPath=\"{outputFile}\"", - "/p:KoreBuildRestoreTargetsImported=true", - }; - - psi.Arguments = string.Join(" ", arguments); - psi.RedirectStandardOutput = true; - - var process = new Process - { - StartInfo = psi, - EnableRaisingEvents = true, - }; - process.OutputDataReceived += (sender, args) => - { - if (!string.IsNullOrEmpty(args.Data)) - { - Console.WriteLine(args.Data); - } - }; - - using (process) - { - process.Start(); - process.BeginOutputReadLine(); - - process.WaitForExit(60 * 5000); - if (process.ExitCode != 0) - { - throw new Exception($"{psi.FileName} {psi.Arguments} failed. Exit code {process.ExitCode}."); - } - } - } - - public void Dispose() - { - if (_deleteSpecDirectoryOnDispose) - { - Directory.Delete(_packageSpecDirectory, recursive: true); - } - } - } -} diff --git a/build/tasks/VersionPinning/PinVersionUtility.cs b/build/tasks/VersionPinning/PinVersionUtility.cs deleted file mode 100644 index c94e19fc62..0000000000 --- a/build/tasks/VersionPinning/PinVersionUtility.cs +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Xml.Linq; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using NuGet.Common; -using NuGet.Frameworks; -using NuGet.LibraryModel; -using NuGet.ProjectModel; -using NuGet.Protocol; -using NuGet.Protocol.Core.Types; -using NuGet.Versioning; - -namespace RepoTasks.VersionPinning -{ - internal class PinVersionUtility - { - private readonly string _repositoryRoot; - private readonly FindPackageByIdResource[] _findPackageResources; - private readonly ConcurrentDictionary> _exactMatches = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); - private readonly DependencyGraphSpecProvider _provider; - private readonly SourceCacheContext _sourceCacheContext; - private readonly TaskLoggingHelper _logger; - - public PinVersionUtility( - string repositoryRoot, - List pinSources, - DependencyGraphSpecProvider provider, - TaskLoggingHelper logger) - { - _repositoryRoot = repositoryRoot; - _findPackageResources = new FindPackageByIdResource[pinSources.Count]; - for (var i = 0; i < pinSources.Count; i++) - { - var repository = FactoryExtensionsV3.GetCoreV3(Repository.Factory, pinSources[i].Trim()); - _findPackageResources[i] = repository.GetResource(); - } - _provider = provider; - _sourceCacheContext = new SourceCacheContext(); - _logger = logger; - } - - public void Execute() - { - _logger.LogMessage(MessageImportance.High, $"Pinning package references for projects in {_repositoryRoot}"); - - var solutionPinMetadata = GetProjectPinVersionMetadata(); - foreach (var cliToolReference in solutionPinMetadata.CLIToolReferences) - { - _logger.LogMessage(MessageImportance.Normal, $"Pinning CLI Tool {cliToolReference.Item1.Name}({cliToolReference.Item1.VersionRange} to {cliToolReference.Item2} for all projects in {_repositoryRoot}."); - } - - foreach (var item in solutionPinMetadata.PinVersionLookup) - { - var projectPinMetadata = item.Value; - var specProject = projectPinMetadata.PackageSpec; - - if (!(projectPinMetadata.Packages.Any() || solutionPinMetadata.CLIToolReferences.Any())) - { - _logger.LogMessage(MessageImportance.Normal, $"No package or tool references to pin for {specProject.FilePath}."); - continue; - } - - var projectFileInfo = new FileInfo(specProject.FilePath); - var pinnedReferencesFile = Path.Combine( - specProject.RestoreMetadata.OutputPath, - projectFileInfo.Name + ".pinnedversions.targets"); - - Directory.CreateDirectory(Path.GetDirectoryName(pinnedReferencesFile)); - - if (projectPinMetadata.Packages.Any()) - { - _logger.LogMessage(MessageImportance.Normal, $"Pinning package versions for {specProject.FilePath}."); - } - - var pinnedReferences = new XElement("ItemGroup", new XAttribute("Condition", "'$(PolicyDesignTimeBuild)' != 'true' AND !Exists('$(MSBuildThisFileDirectory)$(MSBuildProjectFile).nugetpolicy.g.targets')")); - foreach (var packageReference in projectPinMetadata.Packages) - { - (var tfm, var libraryRange, var exactVersion) = packageReference; - _logger.LogMessage(MessageImportance.Normal, $"Pinning reference {libraryRange.Name}({libraryRange.VersionRange} to {exactVersion}."); - var metadata = new List - { - new XAttribute("Update", libraryRange.Name), - new XAttribute("Version", exactVersion.ToNormalizedString()), - }; - - if (tfm != NuGetFramework.AnyFramework) - { - metadata.Add(new XAttribute("Condition", $"'$(TargetFramework)'=='{tfm.GetShortFolderName()}'")); - } - - pinnedReferences.Add(new XElement("PackageReference", metadata)); - } - - // CLI Tool references are specified at solution level. - foreach (var toolReference in solutionPinMetadata.CLIToolReferences) - { - (var libraryRange, var exactVersion) = toolReference; - var metadata = new List - { - new XAttribute("Update", libraryRange.Name), - new XAttribute("Version", exactVersion.ToNormalizedString()), - }; - - pinnedReferences.Add(new XElement("DotNetCliToolReference", metadata)); - } - - var pinnedVersionRoot = new XElement("Project", pinnedReferences); - File.WriteAllText(pinnedReferencesFile, pinnedVersionRoot.ToString()); - } - } - - private SolutionPinVersionMetadata GetProjectPinVersionMetadata() - { - var repositoryDirectoryInfo = new DirectoryInfo(_repositoryRoot); - var projects = new Dictionary(StringComparer.OrdinalIgnoreCase); - var cliToolReferences = new List<(LibraryRange, NuGetVersion)>(); - - foreach (var slnFile in repositoryDirectoryInfo.EnumerateFiles("*.sln")) - { - var graphSpec = _provider.GetDependencyGraphSpec(repositoryDirectoryInfo.Name, slnFile.FullName); - foreach (var specProject in graphSpec.Projects) - { - if (!projects.TryGetValue(specProject.FilePath, out var pinMetadata)) - { - pinMetadata = new ProjectPinVersionMetadata(specProject); - projects[specProject.FilePath] = pinMetadata; - } - - var allDependencies = specProject.Dependencies.Select(dependency => new { Dependency = dependency, FrameworkName = NuGetFramework.AnyFramework }) - .Concat(specProject.TargetFrameworks.SelectMany(tfm => tfm.Dependencies.Select(dependency => new { Dependency = dependency, tfm.FrameworkName }))) - .Where(d => d.Dependency.LibraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)); - - foreach (var dependency in allDependencies) - { - var reference = dependency.Dependency; - var versionRange = reference.LibraryRange.VersionRange; - if (!versionRange.IsFloating) - { - continue; - } - - var exactVersion = GetExactVersion(reference.Name, versionRange); - if (exactVersion == null) - { - continue; - } - - var projectStyle = specProject.RestoreMetadata.ProjectStyle; - if (projectStyle == ProjectStyle.PackageReference) - { - pinMetadata.Packages.Add((dependency.FrameworkName, reference.LibraryRange, exactVersion)); - } - else if (projectStyle == ProjectStyle.DotnetCliTool) - { - cliToolReferences.Add((reference.LibraryRange, exactVersion)); - } - else - { - throw new NotSupportedException($"Unknown project style '{projectStyle}'."); - } - } - } - } - - return new SolutionPinVersionMetadata(projects, cliToolReferences); - } - - private NuGetVersion GetExactVersion(string name, VersionRange range) - { - if (range.MinVersion == null) - { - throw new Exception($"Unsupported version range {range}."); - } - - if (!_exactMatches.TryGetValue(name, out var versionTask)) - { - versionTask = _exactMatches.GetOrAdd(name, GetExactVersionAsync(name, range.MinVersion)); - } - - return versionTask.Result; - } - - private async Task GetExactVersionAsync(string name, NuGetVersion floatingVersion) - { - foreach (var findPackageResource in _findPackageResources) - { - var packageVersions = await findPackageResource.GetAllVersionsAsync(name, _sourceCacheContext, NullLogger.Instance, default(CancellationToken)); - - var matchingVersions = packageVersions.Where(v => v.Version == floatingVersion.Version).ToList(); - switch (matchingVersions.Count) - { - case 0: - continue; - case 1: - return matchingVersions[0]; - default: - throw new Exception($"More than one version for {name} found that matches the specified version constraint: {string.Join(" ", matchingVersions)}."); - } - } - - return null; - } - - private struct SolutionPinVersionMetadata - { - public SolutionPinVersionMetadata( - IDictionary pinVersionLookup, - List<(LibraryRange, NuGetVersion)> cliToolReferences) - { - PinVersionLookup = pinVersionLookup; - CLIToolReferences = cliToolReferences; - } - - public IDictionary PinVersionLookup { get; } - - public List<(LibraryRange, NuGetVersion)> CLIToolReferences { get; } - } - - private struct ProjectPinVersionMetadata - { - public ProjectPinVersionMetadata(PackageSpec packageSpec) - { - PackageSpec = packageSpec; - Packages = new List<(NuGetFramework, LibraryRange, NuGetVersion)>(); - } - - public PackageSpec PackageSpec { get; } - - public List<(NuGetFramework, LibraryRange, NuGetVersion)> Packages { get; } - } - } -} From 4182b0b5d2e4f6dab6a48f9789fc19ae6a6afd64 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 12:31:08 -0700 Subject: [PATCH 023/115] When BuildGraphOf is used, only pin package references to packages currently building --- build/RepositoryBuild.targets | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index bef1c6b508..1e1251913c 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -20,10 +20,19 @@ + + + + + + Date: Mon, 18 Sep 2017 12:39:27 -0700 Subject: [PATCH 024/115] Remove dead code and ensure source code is cloned when building on CI --- build/RepositoryBuild.targets | 1 + build/repo.targets | 5 +- .../BuildGraph/DependencyGraphSpecProvider.cs | 30 ------ build/tasks/BuildGraph/Repository.cs | 101 ------------------ build/tasks/CalculateBuildGraph.cs | 71 ------------ build/tasks/RepoTasks.tasks | 2 - 6 files changed, 4 insertions(+), 206 deletions(-) delete mode 100644 build/tasks/BuildGraph/DependencyGraphSpecProvider.cs delete mode 100644 build/tasks/CalculateBuildGraph.cs diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 1e1251913c..6d119bbd50 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -52,6 +52,7 @@ $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash) + $(RepositoryBuildArguments) /noconsolelogger '/l:RepoTasks.FlowLogger,$(MSBuildThisFileDirectory)tasks\bin\publish\RepoTasks.dll;Summary;FlowId=$(RepositoryToBuild)' $(_RepositoryBuildTargets) $(RepositoryBuildArguments) $(BuildRepositoryRoot)artifacts diff --git a/build/repo.targets b/build/repo.targets index ad9f7d7c7f..ec54c6583b 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -17,7 +17,7 @@ <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify - $(PrepareDependsOn);CleanUniverseArtifacts + $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts $(BuildDependsOn);BuildRepositories @@ -25,6 +25,7 @@ + @@ -49,7 +50,7 @@ - + <_CloneRepositories Include="@(Repository);@(ShippedRepository)" /> <_CloneRepository Include="$(MSBuildProjectFullPath)"> diff --git a/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs b/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs deleted file mode 100644 index d16600578a..0000000000 --- a/build/tasks/BuildGraph/DependencyGraphSpecProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.IO; -using NuGet.ProjectModel; - -namespace RepoTools.BuildGraph -{ - public class DependencyGraphSpecProvider - { - readonly string _packageSpecDirectory; - - public DependencyGraphSpecProvider(string packageSpecDirectory) - { - _packageSpecDirectory = packageSpecDirectory; - } - - public DependencyGraphSpec GetDependencyGraphSpec(string repositoryName, string solutionPath) - { - var outputFile = Path.Combine(_packageSpecDirectory, repositoryName, Path.GetFileName(solutionPath) + ".json"); - - if (!File.Exists(outputFile)) - { - return null; - } - - return DependencyGraphSpec.Load(outputFile); - } - } -} diff --git a/build/tasks/BuildGraph/Repository.cs b/build/tasks/BuildGraph/Repository.cs index 5baaeedc92..e5bdaf414e 100644 --- a/build/tasks/BuildGraph/Repository.cs +++ b/build/tasks/BuildGraph/Repository.cs @@ -4,11 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Threading.Tasks; -using NuGet.LibraryModel; -using NuGet.ProjectModel; namespace RepoTools.BuildGraph { @@ -30,105 +26,8 @@ namespace RepoTools.BuildGraph public IEnumerable AllProjects => Projects.Concat(SupportProjects); - public static IList ReadAllRepositories(IList repositoryPaths, DependencyGraphSpecProvider provider) - { - var repositories = new Repository[repositoryPaths.Count]; - - Parallel.For(0, repositoryPaths.Count, new ParallelOptions { MaxDegreeOfParallelism = 6 }, i => - { - var repositoryPath = repositoryPaths[i]; - var repositoryName = Path.GetFileName(repositoryPath); - var repository = Read(provider, repositoryName, repositoryPath); - repositories[i] = repository; - }); - - return repositories; - } - public bool Equals(Repository other) => string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase); public override int GetHashCode() => StringComparer.OrdinalIgnoreCase.GetHashCode(Name); - - private static Repository Read(DependencyGraphSpecProvider provider, string name, string repositoryPath) - { - var repository = new Repository(name); - - ReadSharedSourceProjects(Path.Combine(repositoryPath, "shared"), repository, repository.Projects); - - var srcDirectory = Path.GetFullPath(Path.Combine(repositoryPath, "src")) - .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); - - var solutionFiles = Directory.EnumerateFiles(repositoryPath, "*.sln"); - foreach (var file in solutionFiles) - { - var spec = provider.GetDependencyGraphSpec(name, file); - if (spec == null) - { - continue; - } - - var projects = spec.Projects.OrderBy(p => p.RestoreMetadata.ProjectStyle == ProjectStyle.PackageReference ? 0 : 1); - foreach (var specProject in projects) - { - var projectPath = Path.GetFullPath(specProject.FilePath) - .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); - - var projectGroup = projectPath.StartsWith(srcDirectory, StringComparison.OrdinalIgnoreCase) ? - repository.Projects : - repository.SupportProjects; - - var project = projectGroup.FirstOrDefault(f => f.Path == specProject.FilePath); - if (project == null) - { - project = new Project(specProject.Name) - { - Repository = repository, - Path = specProject.FilePath, - Version = specProject.Version?.ToString(), - }; - - projectGroup.Add(project); - } - - foreach (var package in GetPackageReferences(specProject)) - { - project.PackageReferences.Add(package); - } - } - } - - return repository; - } - - private static List GetPackageReferences(PackageSpec specProject) - { - var allDependencies = Enumerable.Concat( - specProject.Dependencies, - specProject.TargetFrameworks.SelectMany(tfm => tfm.Dependencies)) - .Distinct(); - - var packageReferences = allDependencies - .Where(d => d.LibraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)) - .Select(d => d.Name) - .ToList(); - return packageReferences; - } - - private static void ReadSharedSourceProjects(string sharedSourceProjectsRoot, Repository repository, IList projects) - { - if (!Directory.Exists(sharedSourceProjectsRoot)) - { - return; - } - - foreach (var directory in new DirectoryInfo(sharedSourceProjectsRoot).EnumerateDirectories()) - { - var project = new Project(directory.Name) - { - Repository = repository, - }; - projects.Add(project); - } - } } } diff --git a/build/tasks/CalculateBuildGraph.cs b/build/tasks/CalculateBuildGraph.cs deleted file mode 100644 index 9475bd99ec..0000000000 --- a/build/tasks/CalculateBuildGraph.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using RepoTools.BuildGraph; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace RepoTasks -{ - public class CalculateBuildGraph : Task - { - [Required] - public ITaskItem[] Repositories { get; set; } - - /// - /// Directory that contains the package spec files. - /// - [Required] - public string PackageSpecsDirectory { get; set; } - - /// - /// The repository at which to root the graph at - /// - public string StartGraphAt { get; set; } - - [Output] - public ITaskItem[] RepositoriesToBuildInOrder { get; set; } - - public override bool Execute() - { - var graphSpecProvider = new DependencyGraphSpecProvider(PackageSpecsDirectory.Trim()); - - var repositoryPaths = Repositories.Select(r => r.GetMetadata("RepositoryPath")).ToList(); - var repositories = Repository.ReadAllRepositories(repositoryPaths, graphSpecProvider); - - var graph = GraphBuilder.Generate(repositories, StartGraphAt, Log); - var repositoriesWithOrder = new List<(ITaskItem repository, int order)>(); - foreach (var repositoryTaskItem in Repositories) - { - var repositoryName = repositoryTaskItem.ItemSpec; - var graphNodeRepository = graph.FirstOrDefault(g => g.Repository.Name == repositoryName); - if (graphNodeRepository == null) - { - // StartGraphAt was specified so the graph is incomplete. - continue; - } - - var order = TopologicalSort.GetOrder(graphNodeRepository); - repositoryTaskItem.SetMetadata("Order", order.ToString()); - repositoriesWithOrder.Add((repositoryTaskItem, order)); - } - - Log.LogMessage(MessageImportance.High, "Repository build order:"); - foreach (var buildGroup in repositoriesWithOrder.GroupBy(r => r.order).OrderBy(g => g.Key)) - { - var buildGroupRepos = buildGroup.Select(b => b.repository.ItemSpec); - Log.LogMessage(MessageImportance.High, $"{buildGroup.Key.ToString().PadLeft(2, ' ')}: {string.Join(", ", buildGroupRepos)}"); - } - - RepositoriesToBuildInOrder = repositoriesWithOrder - .OrderBy(r => r.order) - .Select(r => r.repository) - .ToArray(); - - return true; - } - } -} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index d6c6ae3488..505742d1e9 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -3,8 +3,6 @@ <_RepoTaskAssembly>$(MSBuildThisFileDirectory)bin\publish\RepoTasks.dll - - From 83adfd2484e1485c07a2c97b163618dec12cb32e Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 13:06:07 -0700 Subject: [PATCH 025/115] Find commit hashes from the TeamCity environment variables --- build/repo.targets | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build/repo.targets b/build/repo.targets index ec54c6583b..552c154969 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -202,6 +202,16 @@ + + + <_RepositoryName>$([System.String]::new('%(Repository.Identity)').Replace('.', '_')) + <_CloneUrl>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_URL_$(_RepositoryName)")) + <_CommitHash>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_NUMBER_$(_RepositoryName)")) + From b9e332c11bc612e2d2fe2fc48a67b92cff4824ea Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 13:10:08 -0700 Subject: [PATCH 026/115] Use KoreBuild to clone source code instead of TeamCity --- build/Repositories.props | 3 --- build/repo.targets | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index b95b7374c6..f6a75a3917 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -10,9 +10,6 @@ - - - true + + diff --git a/build/repo.targets b/build/repo.targets index 3845106a4f..72eaac18f2 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -19,7 +19,9 @@ $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts - $(BuildDependsOn);CloneRepositories;BuildRepositories + $(CompileDependsOn);CloneRepositories;BuildRepositories + $(PackageDependsOn);SplitPackages + $(VerifyDependsOn);VerifyCoherentVersions @@ -181,6 +183,26 @@ BuildNumber="$(BuildNumber)" /> + + + + + + + + + + + + + + + + $(BuildDir)$(_RepositoryListFileName) diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs new file mode 100644 index 0000000000..8f780e270f --- /dev/null +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -0,0 +1,100 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class CopyPackagesToSplitFolders : Microsoft.Build.Utilities.Task + { + /// + /// The item group containing the nuget packages to split in different folders. + /// + [Required] + public ITaskItem[] Packages { get; set; } + + [Required] + public ITaskItem[] Files { get; set; } + + /// + /// The folder where packages should be copied. Subfolders will be created based on package category. + /// + [Required] + public string DestinationFolder { get; set; } + + public override bool Execute() + { + if (Files?.Length == 0) + { + Log.LogError("No packages were found."); + return false; + } + + var expectedPackages = PackageCollection.FromItemGroup(Packages); + + Directory.CreateDirectory(DestinationFolder); + + foreach (var file in Files) + { + PackageIdentity identity; + using (var reader = new PackageArchiveReader(file.ItemSpec)) + { + identity = reader.GetIdentity(); + } + + if (!expectedPackages.TryGetCategory(identity.Id, out var category)) + { + Log.LogError($"Unexpected package artifact with id: {identity.Id}"); + return false; + } + + string destDir; + switch (category) + { + case PackageCategory.Unknown: + throw new InvalidOperationException($"Package {identity} does not have a recognized package category."); + case PackageCategory.Shipping: + destDir = Path.Combine(DestinationFolder, "ship"); + break; + case PackageCategory.NoShip: + destDir = Path.Combine(DestinationFolder, "noship"); + break; + case PackageCategory.ShipOob: + destDir = Path.Combine(DestinationFolder, "shipoob"); + break; + default: + throw new NotImplementedException(); + } + + Directory.CreateDirectory(destDir); + + var destFile = Path.Combine(destDir, Path.GetFileName(file.ItemSpec)); + + Log.LogMessage($"Copying {file.ItemSpec} to {destFile}"); + + File.Copy(file.ItemSpec, destFile); + expectedPackages.Remove(identity.Id); + } + + if (expectedPackages.Count != 0) + { + var error = new StringBuilder(); + foreach (var key in expectedPackages.Keys) + { + error.Append(" - ").AppendLine(key); + } + + Log.LogError($"Expected the following packages, but they were not found:" + error.ToString()); + return false; + } + + return true; + } + } +} diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs index 1acbd7c577..d1114814b5 100644 --- a/build/tasks/GenerateLineup.cs +++ b/build/tasks/GenerateLineup.cs @@ -47,7 +47,7 @@ namespace RepoTasks var root = new XElement("Project", props, items); var doc = new XDocument(root); - if (RestoreAdditionalSources.Length > 0) + if (RestoreAdditionalSources?.Length > 0) { var sources = RestoreAdditionalSources.Aggregate("$(RestoreAdditionalProjectSources)", (sum, piece) => sum + ";" + piece.ItemSpec); props.Add(new XElement("RestoreAdditionalProjectSources", sources)); diff --git a/build/tasks/ProjectModel/PackageInfo.cs b/build/tasks/ProjectModel/PackageInfo.cs index af1976686e..463fab690a 100644 --- a/build/tasks/ProjectModel/PackageInfo.cs +++ b/build/tasks/ProjectModel/PackageInfo.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; using NuGet.Frameworks; +using NuGet.Packaging; using NuGet.Versioning; namespace RepoTasks.ProjectModel @@ -12,8 +13,8 @@ namespace RepoTasks.ProjectModel internal class PackageInfo { public PackageInfo(string id, - string version, - IReadOnlyList frameworks, + NuGetVersion version, + IReadOnlyList dependencyGroups, string source, string packageType = "Dependency") { @@ -22,16 +23,11 @@ namespace RepoTasks.ProjectModel throw new ArgumentException(nameof(id)); } - if (string.IsNullOrEmpty(version)) - { - throw new ArgumentException(nameof(version)); - } - Id = id; - Version = NuGetVersion.Parse(version); - Frameworks = frameworks; + Version = version ?? throw new ArgumentNullException(nameof(version)); PackageType = packageType; Source = source; + DependencyGroups = dependencyGroups ?? Array.Empty(); } public string Id { get; } @@ -41,6 +37,6 @@ namespace RepoTasks.ProjectModel /// Can be a https feed or a file path. May be null. /// public string Source { get; } - public IReadOnlyList Frameworks { get; } + public IReadOnlyList DependencyGroups { get; } } } diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 505742d1e9..12ec53fe04 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -4,5 +4,7 @@ + + diff --git a/build/tasks/Utilities/ArtifactInfo.cs b/build/tasks/Utilities/ArtifactInfo.cs index f3bfa91ced..6e35289a3b 100644 --- a/build/tasks/Utilities/ArtifactInfo.cs +++ b/build/tasks/Utilities/ArtifactInfo.cs @@ -5,6 +5,9 @@ using System; using System.IO; using System.Linq; using NuGet.Frameworks; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; using Microsoft.Build.Framework; using RepoTasks.ProjectModel; @@ -50,10 +53,10 @@ namespace RepoTasks.Utilities { return new PackageInfo( item.GetMetadata("PackageId"), - item.GetMetadata("Version"), + NuGetVersion.Parse(item.GetMetadata("Version")), string.IsNullOrEmpty(item.GetMetadata("TargetFramework")) - ? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => NuGetFramework.Parse(s)).ToArray() - : new [] { NuGetFramework.Parse(item.GetMetadata("TargetFramework")) }, + ? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => new PackageDependencyGroup(NuGetFramework.Parse(s), Array.Empty())).ToArray() + : new [] { new PackageDependencyGroup(NuGetFramework.Parse(item.GetMetadata("TargetFramework")), Array.Empty()) }, Path.GetDirectoryName(item.ItemSpec), item.GetMetadata("PackageType")); } diff --git a/build/tasks/Utilities/PackageCategory.cs b/build/tasks/Utilities/PackageCategory.cs new file mode 100644 index 0000000000..80f2990244 --- /dev/null +++ b/build/tasks/Utilities/PackageCategory.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + + +namespace RepoTasks.Utilities +{ + public enum PackageCategory + { + Unknown = 0, + Shipping, + NoShip, + ShipOob + } +} diff --git a/build/tasks/Utilities/PackageCollection.cs b/build/tasks/Utilities/PackageCollection.cs new file mode 100644 index 0000000000..4aa778aa2a --- /dev/null +++ b/build/tasks/Utilities/PackageCollection.cs @@ -0,0 +1,66 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Build.Framework; + +namespace RepoTasks.Utilities +{ + public class PackageCollection + { + private readonly IDictionary _packages = new Dictionary(StringComparer.OrdinalIgnoreCase); + + private PackageCollection() + { + } + + public bool TryGetCategory(string packageId, out PackageCategory category) => _packages.TryGetValue(packageId, out category); + + public void Remove(string packageId) => _packages.Remove(packageId); + + public int Count => _packages.Count; + + public IEnumerable Keys => _packages.Keys; + + public static PackageCollection FromItemGroup(ITaskItem[] items) + { + var list = new PackageCollection(); + if (items == null) + { + return list; + } + + foreach (var item in items) + { + PackageCategory category; + switch (item.GetMetadata("Category")?.ToLowerInvariant()) + { + case "ship": + category = PackageCategory.Shipping; + break; + case "noship": + category = PackageCategory.NoShip; + break; + case "shipoob": + category = PackageCategory.ShipOob; + break; + default: + category = PackageCategory.Unknown; + break; + } + + if (list._packages.ContainsKey(item.ItemSpec)) + { + throw new InvalidDataException($"Duplicate package id detected: {item.ItemSpec}"); + } + + list._packages.Add(item.ItemSpec, category); + } + + return list; + } + } +} diff --git a/build/tasks/VerifyCoherentVersions.cs b/build/tasks/VerifyCoherentVersions.cs new file mode 100644 index 0000000000..6001041d0e --- /dev/null +++ b/build/tasks/VerifyCoherentVersions.cs @@ -0,0 +1,121 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using NuGet.Frameworks; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using RepoTasks.ProjectModel; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class VerifyCoherentVersions : Microsoft.Build.Utilities.Task + { + public ITaskItem[] PackageFiles { get; set; } + + public override bool Execute() + { + var packageLookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var file in PackageFiles) + { + PackageInfo package; + using (var reader = new PackageArchiveReader(file.ItemSpec)) + { + var identity = reader.GetIdentity(); + var metadata = new PackageBuilder(reader.GetNuspec(), basePath: null); + package = new PackageInfo(identity.Id, identity.Version, + source: Path.GetDirectoryName(file.ItemSpec), + dependencyGroups: metadata.DependencyGroups.ToArray()); + } + + if (packageLookup.TryGetValue(package.Id, out var existingPackage)) + { + throw new Exception("Multiple copies of the following package were found: " + + Environment.NewLine + + existingPackage + + Environment.NewLine + + package); + } + + packageLookup[package.Id] = package; + } + + var dependencyIssues = new List(); + foreach (var packageInfo in packageLookup.Values) + { + dependencyIssues.AddRange(Visit(packageLookup, packageInfo)); + } + + var success = true; + foreach (var mismatch in dependencyIssues) + { + var message = $"{mismatch.Info.Id} depends on {mismatch.Dependency.Id} " + + $"v{mismatch.Dependency.VersionRange} ({mismatch.TargetFramework}) when the latest build is v{mismatch.Info.Version}."; + Log.LogError(message); + success = false; + } + + Log.LogMessage(MessageImportance.High, $"Verified {PackageFiles.Length} package(s) have coherent versions"); + return success; + } + + private class DependencyWithIssue + { + public PackageDependency Dependency { get; set; } + public PackageInfo Info { get; set; } + public NuGetFramework TargetFramework { get; set; } + } + + private IEnumerable Visit(IDictionary packageLookup, PackageInfo packageInfo) + { + Log.LogMessage(MessageImportance.Low, $"Processing package {packageInfo.Id}"); + try + { + var issues = new List(); + foreach (var dependencySet in packageInfo.DependencyGroups) + { + // If the package doens't target any frameworks, just accept it + if (dependencySet.TargetFramework == null) + { + continue; + } + + foreach (var dependency in dependencySet.Packages) + { + if (!packageLookup.TryGetValue(dependency.Id, out var dependencyPackageInfo)) + { + // External dependency + continue; + } + + if (dependencyPackageInfo.Version != dependency.VersionRange.MinVersion) + { + // For any dependency in the universe + // Add a mismatch if the min version doesn't work out + // (we only really care about >= minVersion) + issues.Add(new DependencyWithIssue + { + Dependency = dependency, + TargetFramework = dependencySet.TargetFramework, + Info = dependencyPackageInfo + }); + } + } + } + return issues; + } + catch + { + Log.LogError($"Unable to verify package {packageInfo.Id}"); + throw; + } + } + } +} From cca1480f3a3e8fc83b2bc168202e060539e9f078 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 19 Sep 2017 10:48:15 -0700 Subject: [PATCH 030/115] Switch from feature/x to rel/x branches --- build/Repositories.props | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index f6a75a3917..ec23e8c91c 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -1,14 +1,14 @@ - - - - - - - - + + + + + + + + - - - dev - https://aspnetcore.blob.core.windows.net/buildtools - - +```json +{ + "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/dev/tools/korebuild.schema.json", + "channel": "dev", + "toolsSource": "https://aspnetcore.blob.core.windows.net/buildtools" +} ``` #> [CmdletBinding(PositionalBinding = $false)] @@ -58,7 +57,7 @@ param( [string]$ToolsSource, [Alias('u')] [switch]$Update, - [string]$ConfigFile = (Join-Path $PSScriptRoot 'version.props'), + [string]$ConfigFile = $null, [Parameter(ValueFromRemainingArguments = $true)] [string[]]$MSBuildArgs ) @@ -147,10 +146,20 @@ function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) { # Load configuration or set defaults +$Path = Resolve-Path $Path +if (!$ConfigFile) { $ConfigFile = Join-Path $Path 'korebuild.json' } + if (Test-Path $ConfigFile) { - [xml] $config = Get-Content $ConfigFile - if (!($Channel)) { [string] $Channel = Select-Xml -Xml $config -XPath '/Project/PropertyGroup/KoreBuildChannel' } - if (!($ToolsSource)) { [string] $ToolsSource = Select-Xml -Xml $config -XPath '/Project/PropertyGroup/KoreBuildToolsSource' } + try { + $config = Get-Content -Raw -Encoding UTF8 -Path $ConfigFile | ConvertFrom-Json + if ($config) { + if (!($Channel) -and (Get-Member -Name 'channel' -InputObject $config)) { [string] $Channel = $config.channel } + if (!($ToolsSource) -and (Get-Member -Name 'toolsSource' -InputObject $config)) { [string] $ToolsSource = $config.toolsSource} + } + } catch { + Write-Warning "$ConfigFile could not be read. Its settings will be ignored." + Write-Warning $Error[0] + } } if (!$DotNetHome) { diff --git a/build.sh b/build.sh index dc1cb5f14b..9e32358cf3 100755 --- a/build.sh +++ b/build.sh @@ -8,10 +8,11 @@ set -euo pipefail RESET="\033[0m" RED="\033[0;31m" +YELLOW="\033[0;33m" MAGENTA="\033[0;95m" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" [ -z "${DOTNET_HOME:-}"] && DOTNET_HOME="$HOME/.dotnet" -config_file="$DIR/version.props" +config_file="$DIR/korebuild.json" verbose=false update=false repo_path="$DIR" @@ -30,7 +31,7 @@ __usage() { echo "Options:" echo " --verbose Show verbose output." echo " -c|--channel The channel of KoreBuild to download. Overrides the value from the config file.." - echo " --config-file TThe path to the configuration file that stores values. Defaults to version.props." + echo " --config-file The path to the configuration file that stores values. Defaults to korebuild.json." echo " -d|--dotnet-home The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet." echo " --path The directory to build. Defaults to the directory containing the script." echo " -s|--tools-source The base url where build tools can be downloaded. Overrides the value from the config file." @@ -81,7 +82,11 @@ get_korebuild() { } __error() { - echo -e "${RED}$@${RESET}" 1>&2 + echo -e "${RED}error: $*${RESET}" 1>&2 +} + +__warn() { + echo -e "${YELLOW}warning: $*${RESET}" } __machine_has() { @@ -113,8 +118,6 @@ __get_remote_file() { fi } -__read_dom () { local IFS=\> ; read -d \< ENTITY CONTENT ;} - # # main # @@ -134,6 +137,10 @@ while [[ $# > 0 ]]; do shift config_file="${1:-}" [ -z "$config_file" ] && __usage + if [ ! -f "$config_file" ]; then + __error "Invalid value for --config-file. $config_file does not exist." + exit 1 + fi ;; -d|--dotnet-home|-DotNetHome) shift @@ -177,14 +184,28 @@ if ! __machine_has curl && ! __machine_has wget; then exit 1 fi -if [ -f $config_file ]; then - comment=false - while __read_dom; do - if [ "$comment" = true ]; then [[ $CONTENT == *'-->'* ]] && comment=false ; continue; fi - if [[ $ENTITY == '!--'* ]]; then comment=true; continue; fi - if [ -z "$channel" ] && [[ $ENTITY == "KoreBuildChannel" ]]; then channel=$CONTENT; fi - if [ -z "$tools_source" ] && [[ $ENTITY == "KoreBuildToolsSource" ]]; then tools_source=$CONTENT; fi - done < $config_file +[ -z "${config_file:-}" ] && config_file="$repo_path/korebuild.json" +if [ -f "$config_file" ]; then + if __machine_has jq ; then + if jq '.' "$config_file" >/dev/null ; then + config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")" + config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")" + else + __warn "$config_file is invalid JSON. Its settings will be ignored." + fi + elif __machine_has python ; then + if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then + config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")" + config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")" + else + __warn "$config_file is invalid JSON. Its settings will be ignored." + fi + else + __warn 'Missing required command: jq or pyton. Could not parse the JSON file. Its settings will be ignored.' + fi + + [ ! -z "${config_channel:-}" ] && channel="$config_channel" + [ ! -z "${config_tools_source:-}" ] && tools_source="$config_tools_source" fi [ -z "$channel" ] && channel='dev' diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 73a677fef7..3607f9fdd1 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-alpha-15495 -commithash:60338bc298ac6f70287e89256470cccf24c9bd80 +version:2.0.2-alpha-15498 +commithash:95739991d455287c157657744422cca9c023b0a2 diff --git a/korebuild.json b/korebuild.json new file mode 100644 index 0000000000..6bbc5eeb9c --- /dev/null +++ b/korebuild.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/rel/2.0.2/tools/korebuild.schema.json", + "channel": "rel/2.0.2" +} From 0f578ec2c543f38118c96ddd3c9951e06bfab935 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 12:49:27 -0700 Subject: [PATCH 032/115] Update to korebuild 2.0.2-beta-15503 --- korebuild-lock.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 3607f9fdd1..e608c0f7f0 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-alpha-15498 -commithash:95739991d455287c157657744422cca9c023b0a2 +version:2.0.2-beta-15503 +commithash:aa02c62a19780a960ce1bee2e453741df54f3703 From cb7540bd40fef26dba6aa0fc36b60b790a7742e4 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 13:34:51 -0700 Subject: [PATCH 033/115] Add aspnet/Security to the 2.0.x patch --- build/Repositories.props | 2 +- build/artifacts.props | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/build/Repositories.props b/build/Repositories.props index ec23e8c91c..793a7913f2 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -9,6 +9,7 @@ + From 72598ad07c034dfe18a4d8845ab17f3b00f0b9d6 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 14:07:26 -0700 Subject: [PATCH 035/115] Update env checked to see if TeamCity cloned the source --- build/repo.targets | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index 10419b8de8..f6fb26a5c9 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -16,6 +16,7 @@ <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify + Patch20_ $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts @@ -55,7 +56,7 @@ <_CloneRepositories Include="@(Repository)" /> - <_CloneRepositories Remove="%(Repository.Identity)" Condition="$([System.Environment]::GetEnvironmentVariable('BUILD_VCS_URL_%(Repository.Identity)')) != ''" /> + <_CloneRepositories Remove="%(Repository.Identity)" Condition="$([System.Environment]::GetEnvironmentVariable('BUILD_VCS_NUMBER_$(TcVcsIdPrefix)%(Repository.Identity)')) != ''" /> <_CloneRepositories Include="@(ShippedRepository)" /> <_CloneRepository Include="$(MSBuildProjectFullPath)"> @@ -235,8 +236,8 @@ * BUILD_VCS_NUMBER_ gives us the commit hash for a repository. --> <_RepositoryName>$([System.String]::new('%(Repository.Identity)').Replace('.', '_')) - <_CloneUrl>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_URL_$(_RepositoryName)")) - <_CommitHash>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_NUMBER_$(_RepositoryName)")) + <_CloneUrl>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_URL_$(TcVcsIdPrefix)$(_RepositoryName)")) + <_CommitHash>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_NUMBER_$(TcVcsIdPrefix)$(_RepositoryName)")) Date: Wed, 20 Sep 2017 14:58:15 -0700 Subject: [PATCH 036/115] Verify external dependencies (#561) --- build/dependencies.props | 178 +++++++++++++++++++++++ build/repo.beforecommon.props | 10 ++ build/repo.props | 1 + build/repo.targets | 11 +- build/tasks/AnalyzeBuildGraph.cs | 38 ++++- build/tasks/Utilities/KoreBuildErrors.cs | 2 + build/tasks/VerifyCoherentVersions.cs | 82 ++++++----- 7 files changed, 281 insertions(+), 41 deletions(-) create mode 100644 build/dependencies.props create mode 100644 build/repo.beforecommon.props diff --git a/build/dependencies.props b/build/dependencies.props new file mode 100644 index 0000000000..7094a9050b --- /dev/null +++ b/build/dependencies.props @@ -0,0 +1,178 @@ + + + + + + + + + + + + false + + + + + + https://dotnet.myget.org/F/roslyn/api/v3/index.json + + + + + + KRB2004 + + + + KRB2004 + + + + + + KRB2004 + + + + + + + + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json + + + + + + + + + https://dotnet.myget.org/F/aspnetcoremodule/api/v3/index.json + + + + + + + + + https://dotnet.myget.org/F/aspnetcore-ci-dev/api/v3/index.json + + + + + + + + + + https://api.nuget.org/v3/index.json + + + + + + + + + + + + + + + + + KRB2004 + + + + + KRB2004 + + + + KRB2004 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KRB2004 + + + + KRB2004 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KRB2004 + + + + KRB2004 + + + + + + KRB2004 + + + + KRB2004 + + + diff --git a/build/repo.beforecommon.props b/build/repo.beforecommon.props new file mode 100644 index 0000000000..0550358981 --- /dev/null +++ b/build/repo.beforecommon.props @@ -0,0 +1,10 @@ + + + + true + + diff --git a/build/repo.props b/build/repo.props index a98645c000..63a8d665d0 100644 --- a/build/repo.props +++ b/build/repo.props @@ -5,4 +5,5 @@ + diff --git a/build/repo.targets b/build/repo.targets index f6fb26a5c9..2ddd3f45e9 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -75,9 +75,9 @@ - + BuildInParallel="$(BuildInParallel)" /> @@ -161,6 +161,7 @@ @@ -199,13 +200,15 @@ DestinationFolder="$(ArtifactsDir)" /> - + + + PackageFiles="@(ShippingPackageFiles)" + ExternalDependencies="@(ExternalDependency);@(ShippedExternalDependency)" /> diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 7776f07319..1119b21dcf 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -31,6 +31,9 @@ namespace RepoTasks [Required] public ITaskItem[] Artifacts { get; set; } + [Required] + public ITaskItem[] Dependencies { get; set; } + // Artifacts that already shipped from repos [Required] public ITaskItem[] ShippedArtifacts { get; set; } @@ -95,6 +98,21 @@ namespace RepoTasks // ensure versions cascade var buildPackageMap = packages.ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); + var dependencyMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var dep in Dependencies) + { + if (!dependencyMap.TryGetValue(dep.ItemSpec, out var versions)) + { + dependencyMap[dep.ItemSpec] = versions = new List(); + } + else if (dep.GetMetadata("NoWarn") == null || dep.GetMetadata("NoWarn").IndexOf("KRB" + KoreBuildErrors.MultipleExternalDependencyVersions) < 0) + { + Log.LogKoreBuildWarning( + KoreBuildErrors.MultipleExternalDependencyVersions, + message: $"Multiple versions of external dependency '{dep.ItemSpec}' are defined. In most cases, there should only be one version of external dependencies."); + } + versions.Add(dep.GetMetadata("Version")); + } var inconsistentVersions = new List(); var reposThatShouldPatch = new HashSet(); @@ -107,7 +125,25 @@ namespace RepoTasks { if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) { - // this dependency is not a PackageReference to something that we build in Universe + // This dependency is not one of the packages that will be compiled by this run of Universe. + + var matchesExternalDependency = false; + if (shippedPackageMap.TryGetValue(dependency.Key, out var shippedPackage)) + { + matchesExternalDependency = shippedPackage.PackageInfo.Version.Equals(NuGetVersion.Parse(dependency.Value.Version)); + } + else if (dependencyMap.TryGetValue(dependency.Key, out var externalVersions)) + { + matchesExternalDependency = externalVersions.Contains(dependency.Value.Version); + } + + if (!matchesExternalDependency) + { + Log.LogKoreBuildError( + project.FullPath, + KoreBuildErrors.UndefinedExternalDependency, + message: $"Undefined external dependency on {dependency.Key}/{dependency.Value.Version}"); + } continue; } diff --git a/build/tasks/Utilities/KoreBuildErrors.cs b/build/tasks/Utilities/KoreBuildErrors.cs index 7751985c6f..b3693e5011 100644 --- a/build/tasks/Utilities/KoreBuildErrors.cs +++ b/build/tasks/Utilities/KoreBuildErrors.cs @@ -15,12 +15,14 @@ namespace RepoTasks.Utilities public const int RepoVersionDoesNotMatchProjectVersion = 2001; public const int RepoPackageVersionDoesNotMatchProjectPackageVersion = 2002; public const int DuplicatePackageReference = 2003; + public const int MultipleExternalDependencyVersions = 2004; // NuGet errors public const int InvalidNuspecFile = 4001; public const int PackageReferenceHasVersion = 4002; public const int DotNetCliReferenceReferenceHasVersion = 4003; public const int PackageVersionNotFoundInLineup = 4004; + public const int UndefinedExternalDependency = 4005; // Other unknown errors public const int PolicyFailedToApply = 5000; diff --git a/build/tasks/VerifyCoherentVersions.cs b/build/tasks/VerifyCoherentVersions.cs index 6001041d0e..1ea4afcfce 100644 --- a/build/tasks/VerifyCoherentVersions.cs +++ b/build/tasks/VerifyCoherentVersions.cs @@ -18,11 +18,27 @@ namespace RepoTasks { public class VerifyCoherentVersions : Microsoft.Build.Utilities.Task { + [Required] public ITaskItem[] PackageFiles { get; set; } + [Required] + public ITaskItem[] ExternalDependencies { get; set; } + public override bool Execute() { var packageLookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + + var dependencyMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var dep in ExternalDependencies) + { + if (!dependencyMap.TryGetValue(dep.ItemSpec, out var externalDep)) + { + dependencyMap[dep.ItemSpec] = externalDep = new ExternalDependency(); + } + externalDep.IsPrivate = bool.TryParse(dep.GetMetadata("Private"), out var isPrivate) && isPrivate; + externalDep.AllowedVersions.Add(dep.GetMetadata("Version")); + } + foreach (var file in PackageFiles) { PackageInfo package; @@ -47,51 +63,51 @@ namespace RepoTasks packageLookup[package.Id] = package; } - var dependencyIssues = new List(); foreach (var packageInfo in packageLookup.Values) { - dependencyIssues.AddRange(Visit(packageLookup, packageInfo)); - } - - var success = true; - foreach (var mismatch in dependencyIssues) - { - var message = $"{mismatch.Info.Id} depends on {mismatch.Dependency.Id} " + - $"v{mismatch.Dependency.VersionRange} ({mismatch.TargetFramework}) when the latest build is v{mismatch.Info.Version}."; - Log.LogError(message); - success = false; + Visit(packageLookup, dependencyMap, packageInfo); } Log.LogMessage(MessageImportance.High, $"Verified {PackageFiles.Length} package(s) have coherent versions"); - return success; + return !Log.HasLoggedErrors; } - private class DependencyWithIssue + private class ExternalDependency { - public PackageDependency Dependency { get; set; } - public PackageInfo Info { get; set; } - public NuGetFramework TargetFramework { get; set; } + public List AllowedVersions { get; } = new List(); + public bool IsPrivate { get; set; } } - private IEnumerable Visit(IDictionary packageLookup, PackageInfo packageInfo) + private void Visit( + IReadOnlyDictionary packageLookup, + IReadOnlyDictionary dependencyMap, + PackageInfo packageInfo) { Log.LogMessage(MessageImportance.Low, $"Processing package {packageInfo.Id}"); try { - var issues = new List(); foreach (var dependencySet in packageInfo.DependencyGroups) { - // If the package doens't target any frameworks, just accept it - if (dependencySet.TargetFramework == null) - { - continue; - } - foreach (var dependency in dependencySet.Packages) { - if (!packageLookup.TryGetValue(dependency.Id, out var dependencyPackageInfo)) + PackageInfo dependencyPackageInfo; + var depVersion = dependency.VersionRange.MinVersion.ToString(); + if (dependencyMap.TryGetValue(dependency.Id, out var externalDepInfo)) { - // External dependency + if (externalDepInfo.IsPrivate) + { + Log.LogError($"Package {packageInfo.Id} has an external dependency on {dependency.Id}/{depVersion} which is marked as Private=true."); + } + else if (!externalDepInfo.AllowedVersions.Any(a => depVersion.Equals(a))) + { + Log.LogError($"Package {packageInfo.Id} has an external dependency on the wrong version of {dependency.Id}. " + + $"It uses {depVersion} but only {string.Join(" or ", externalDepInfo.AllowedVersions)} is allowed."); + } + continue; + } + else if (!packageLookup.TryGetValue(dependency.Id, out dependencyPackageInfo)) + { + Log.LogError($"Package {packageInfo.Id} has an undefined external dependency on {dependency.Id}/{depVersion}"); continue; } @@ -100,21 +116,15 @@ namespace RepoTasks // For any dependency in the universe // Add a mismatch if the min version doesn't work out // (we only really care about >= minVersion) - issues.Add(new DependencyWithIssue - { - Dependency = dependency, - TargetFramework = dependencySet.TargetFramework, - Info = dependencyPackageInfo - }); + Log.LogError($"{packageInfo.Id} depends on {dependency.Id} " + + $"{dependency.VersionRange} ({dependencySet.TargetFramework}) when the latest build is {depVersion}."); } } } - return issues; } - catch + catch (Exception ex) { - Log.LogError($"Unable to verify package {packageInfo.Id}"); - throw; + Log.LogError($"Unexpected error while attempting to verify package {packageInfo.Id}.\r\n{ex}"); } } } From 8922f69532ead6470bcec2e7354410f0713fbe16 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 10:52:58 -0700 Subject: [PATCH 037/115] Mirror external dependencies --- build/dependencies.props | 10 +- build/repo.targets | 23 ++- build/tasks/CopyPackagesToSplitFolders.cs | 3 + build/tasks/DownloadNuGetPackages.cs | 196 ++++++++++++++++++++ build/tasks/Logger/MSBuildLogger.cs | 200 +++++++++++++++++++++ build/tasks/RepoTasks.csproj | 2 +- build/tasks/RepoTasks.tasks | 1 + build/tasks/Utilities/PackageCategory.cs | 3 +- build/tasks/Utilities/PackageCollection.cs | 3 + 9 files changed, 433 insertions(+), 8 deletions(-) create mode 100644 build/tasks/DownloadNuGetPackages.cs create mode 100644 build/tasks/Logger/MSBuildLogger.cs diff --git a/build/dependencies.props b/build/dependencies.props index 7094a9050b..4d8c14566c 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -10,6 +10,8 @@ false + + false @@ -52,17 +54,16 @@ - + - https://dotnet.myget.org/F/aspnetcore-ci-dev/api/v3/index.json + https://dotnet.myget.org/F/aspnetcore-release/api/v3/index.json - - + @@ -71,6 +72,7 @@ + diff --git a/build/repo.targets b/build/repo.targets index 2ddd3f45e9..73a7bb8306 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -16,10 +16,15 @@ <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify + + $(IntermediateDir)mirror\ + + $(IntermediateDir)ext\ Patch20_ $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts + $(RestoreDependsOn);RestoreExternalDependencies $(CompileDependsOn);CloneRepositories;BuildRepositories $(PackageDependsOn);SplitPackages $(VerifyDependsOn);VerifyCoherentVersions @@ -27,6 +32,19 @@ + + + + + + + + + @@ -192,11 +210,12 @@ + diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs index 8f780e270f..d9e43808ba 100644 --- a/build/tasks/CopyPackagesToSplitFolders.cs +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -68,6 +68,9 @@ namespace RepoTasks case PackageCategory.ShipOob: destDir = Path.Combine(DestinationFolder, "shipoob"); break; + case PackageCategory.Mirror: + destDir = Path.Combine(DestinationFolder, "mirror"); + break; default: throw new NotImplementedException(); } diff --git a/build/tasks/DownloadNuGetPackages.cs b/build/tasks/DownloadNuGetPackages.cs new file mode 100644 index 0000000000..8412cc5fbd --- /dev/null +++ b/build/tasks/DownloadNuGetPackages.cs @@ -0,0 +1,196 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Build; +using NuGet.Commands; +using NuGet.Configuration; +using NuGet.DependencyResolver; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; +using Task = System.Threading.Tasks.Task; + +namespace RepoTasks +{ + public class DownloadNuGetPackages : Microsoft.Build.Utilities.Task, ICancelableTask + { + private static readonly Task FalseTask = Task.FromResult(false); + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + + [Required] + public ITaskItem[] Packages { get; set; } + + [Required] + public string DestinationFolder { get; set; } + + [Output] + public ITaskItem[] Files { get; set; } + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + public async Task ExecuteAsync() + { + DestinationFolder = DestinationFolder.Replace('\\', '/'); + + var requests = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var files = new List(); + var downloadCount = 0; + foreach (var item in Packages) + { + var id = item.ItemSpec; + var rawVersion = item.GetMetadata("Version"); + if (!NuGetVersion.TryParse(rawVersion, out var version)) + { + Log.LogError($"Package '{id}' has an invalid 'Version' metadata value: '{rawVersion}'."); + return false; + } + + var source = item.GetMetadata("Source"); + if (string.IsNullOrEmpty(source)) + { + Log.LogError($"Package '{id}' is missing the 'Source' metadata value."); + return false; + } + + if (!requests.TryGetValue(source, out var packages)) + { + packages = requests[source] = new List(); + } + + var request = new PackageIdentity(id, version); + var dest = GetExpectedOutputPath(request); + files.Add(new TaskItem(dest)); + if (File.Exists(dest)) + { + Log.LogMessage($"Skipping {request.Id} {request.Version}. Already exists in '{dest}'"); + continue; + } + else + { + downloadCount++; + packages.Add(request); + } + } + + Files = files.ToArray(); + + if (downloadCount == 0) + { + Log.LogMessage("All packages are downloaded."); + return true; + } + + Directory.CreateDirectory(DestinationFolder); + var logger = new MSBuildLogger(Log); + var timer = Stopwatch.StartNew(); + + logger.LogMinimal($"Downloading {downloadCount} package(s)"); + + using (var cacheContext = new SourceCacheContext()) + { + var defaultSettings = Settings.LoadDefaultSettings(root: null, configFileName: null, machineWideSettings: null); + var sourceProvider = new CachingSourceProvider(new PackageSourceProvider(defaultSettings)); + var tasks = new List>(); + + foreach (var feed in requests) + { + var repo = sourceProvider.CreateRepository(new PackageSource(feed.Key)); + tasks.Add(DownloadPackagesAsync(repo, feed.Value, cacheContext, logger, _cts.Token)); + } + + var all = Task.WhenAll(tasks); + var wait = TimeSpan.FromSeconds(Math.Max(downloadCount * 5, 30)); + var delay = Task.Delay(wait); + + var finished = await Task.WhenAny(all, delay); + if (ReferenceEquals(delay, finished)) + { + Log.LogError($"Timed out after {wait.TotalSeconds}s"); + Cancel(); + return false; + } + + if (!tasks.All(a => a.Result)) + { + Log.LogError("Failed to download all packages"); + return false; + } + + timer.Stop(); + logger.LogMinimal($"Finished downloading {downloadCount} package(s) in {timer.ElapsedMilliseconds}ms"); + return true; + } + } + + private async Task DownloadPackagesAsync( + SourceRepository repo, + IEnumerable requests, + SourceCacheContext cacheContext, + NuGet.Common.ILogger logger, + CancellationToken cancellationToken) + { + var remoteLibraryProvider = new SourceRepositoryDependencyProvider(repo, logger, cacheContext, ignoreFailedSources: false, ignoreWarning: false); + var downloads = new List>(); + var metadataResource = await repo.GetResourceAsync(); + + foreach (var request in requests) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (metadataResource != null && !await metadataResource.Exists(request, logger, cancellationToken)) + { + logger.LogError($"Package {request.Id} {request.Version} is not available on '{repo}'"); + downloads.Add(FalseTask); + continue; + } + + var download = DownloadPackageAsync(cacheContext, logger, remoteLibraryProvider, request, cancellationToken); + downloads.Add(download); + } + + await Task.WhenAll(downloads); + return downloads.All(d => d.Result); + } + + private async Task DownloadPackageAsync(SourceCacheContext cacheContext, + NuGet.Common.ILogger logger, + SourceRepositoryDependencyProvider remoteLibraryProvider, + PackageIdentity request, + CancellationToken cancellationToken) + { + var dest = GetExpectedOutputPath(request); + logger.LogInformation($"Downloading {request.Id} {request.Version} to '{dest}'"); + + using (var packageDependency = await remoteLibraryProvider.GetPackageDownloaderAsync(request, cacheContext, logger, cancellationToken)) + { + if (!await packageDependency.CopyNupkgFileToAsync(dest, cancellationToken)) + { + logger.LogError($"Could not download {request.Id} {request.Version} from {remoteLibraryProvider.Source}"); + return false; + } + } + + return true; + } + + private string GetExpectedOutputPath(PackageIdentity request) + { + return Path.Combine(DestinationFolder, $"{request.Id.ToLowerInvariant()}.{request.Version.ToNormalizedString()}.nupkg"); + } + } +} diff --git a/build/tasks/Logger/MSBuildLogger.cs b/build/tasks/Logger/MSBuildLogger.cs new file mode 100644 index 0000000000..458bd698d9 --- /dev/null +++ b/build/tasks/Logger/MSBuildLogger.cs @@ -0,0 +1,200 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Common; + +namespace NuGet.Build +{ + /// + /// TaskLoggingHelper -> ILogger + /// + internal class MSBuildLogger : LoggerBase, Common.ILogger + { + private readonly TaskLoggingHelper _taskLogging; + + private delegate void LogMessageWithDetails(string subcategory, + string code, + string helpKeyword, + string file, + int lineNumber, + int columnNumber, + int endLineNumber, + int endColumnNumber, + MessageImportance importance, + string message, + params object[] messageArgs); + + private delegate void LogErrorWithDetails(string subcategory, + string code, + string helpKeyword, + string file, + int lineNumber, + int columnNumber, + int endLineNumber, + int endColumnNumber, + string message, + params object[] messageArgs); + + private delegate void LogMessageAsString(MessageImportance importance, + string message, + params object[] messageArgs); + + private delegate void LogErrorAsString(string message, + params object[] messageArgs); + + public MSBuildLogger(TaskLoggingHelper taskLogging) + { + _taskLogging = taskLogging ?? throw new ArgumentNullException(nameof(taskLogging)); + } + + public override void Log(ILogMessage message) + { + if (DisplayMessage(message.Level)) + { + if (RuntimeEnvironmentHelper.IsMono) + { + LogForMono(message); + return; + } + else + { + var logMessage = message as IRestoreLogMessage; + + if (logMessage == null) + { + logMessage = new RestoreLogMessage(message.Level, message.Message) + { + Code = message.Code, + FilePath = message.ProjectPath + }; + } + LogForNonMono(logMessage); + } + } + } + + /// + /// Log using with metadata for non mono platforms. + /// + private void LogForNonMono(IRestoreLogMessage message) + { + switch (message.Level) + { + case LogLevel.Error: + LogError(message, _taskLogging.LogError, _taskLogging.LogError); + break; + + case LogLevel.Warning: + LogError(message, _taskLogging.LogWarning, _taskLogging.LogWarning); + break; + + case LogLevel.Minimal: + LogMessage(message, MessageImportance.High, _taskLogging.LogMessage, _taskLogging.LogMessage); + break; + + case LogLevel.Information: + LogMessage(message, MessageImportance.Normal, _taskLogging.LogMessage, _taskLogging.LogMessage); + break; + + case LogLevel.Debug: + case LogLevel.Verbose: + default: + // Default to LogLevel.Debug and low importance + LogMessage(message, MessageImportance.Low, _taskLogging.LogMessage, _taskLogging.LogMessage); + break; + } + } + + /// + /// Log using basic methods to avoid missing methods on mono. + /// + private void LogForMono(ILogMessage message) + { + switch (message.Level) + { + case LogLevel.Error: + _taskLogging.LogError(message.Message); + break; + + case LogLevel.Warning: + _taskLogging.LogWarning(message.Message); + break; + + case LogLevel.Minimal: + _taskLogging.LogMessage(MessageImportance.High, message.Message); + break; + + case LogLevel.Information: + _taskLogging.LogMessage(MessageImportance.Normal, message.Message); + break; + + case LogLevel.Debug: + case LogLevel.Verbose: + default: + // Default to LogLevel.Debug and low importance + _taskLogging.LogMessage(MessageImportance.Low, message.Message); + break; + } + + return; + } + + private void LogMessage(IRestoreLogMessage logMessage, + MessageImportance importance, + LogMessageWithDetails logWithDetails, + LogMessageAsString logAsString) + { + if (logMessage.Code > NuGetLogCode.Undefined) + { + // NuGet does not currently have a subcategory while throwing logs, hence string.Empty + logWithDetails(string.Empty, + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + logMessage.FilePath, + logMessage.StartLineNumber, + logMessage.StartColumnNumber, + logMessage.EndLineNumber, + logMessage.EndColumnNumber, + importance, + logMessage.Message); + } + else + { + logAsString(importance, logMessage.Message); + } + } + + private void LogError(IRestoreLogMessage logMessage, + LogErrorWithDetails logWithDetails, + LogErrorAsString logAsString) + { + if (logMessage.Code > NuGetLogCode.Undefined) + { + // NuGet does not currently have a subcategory while throwing logs, hence string.Empty + logWithDetails(string.Empty, + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + logMessage.FilePath, + logMessage.StartLineNumber, + logMessage.StartColumnNumber, + logMessage.EndLineNumber, + logMessage.EndColumnNumber, + logMessage.Message); + } + else + { + logAsString(logMessage.Message); + } + } + + public override System.Threading.Tasks.Task LogAsync(ILogMessage message) + { + Log(message); + + return System.Threading.Tasks.Task.FromResult(0); + } + } +} diff --git a/build/tasks/RepoTasks.csproj b/build/tasks/RepoTasks.csproj index 084b189145..39574ccdb1 100644 --- a/build/tasks/RepoTasks.csproj +++ b/build/tasks/RepoTasks.csproj @@ -6,7 +6,7 @@ - + diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 12ec53fe04..d2df6c836a 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -5,6 +5,7 @@ + diff --git a/build/tasks/Utilities/PackageCategory.cs b/build/tasks/Utilities/PackageCategory.cs index 80f2990244..f51142bdb7 100644 --- a/build/tasks/Utilities/PackageCategory.cs +++ b/build/tasks/Utilities/PackageCategory.cs @@ -9,6 +9,7 @@ namespace RepoTasks.Utilities Unknown = 0, Shipping, NoShip, - ShipOob + ShipOob, + Mirror, } } diff --git a/build/tasks/Utilities/PackageCollection.cs b/build/tasks/Utilities/PackageCollection.cs index 4aa778aa2a..d066360f53 100644 --- a/build/tasks/Utilities/PackageCollection.cs +++ b/build/tasks/Utilities/PackageCollection.cs @@ -47,6 +47,9 @@ namespace RepoTasks.Utilities case "shipoob": category = PackageCategory.ShipOob; break; + case "mirror": + category = PackageCategory.Mirror; + break; default: category = PackageCategory.Unknown; break; From 8b112a95488c7a2338db799642b8da06f60f494a Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 15:43:11 -0700 Subject: [PATCH 038/115] Fix bug in external version verification causing Private=true to apply to all versions of that external dependency --- build/tasks/VerifyCoherentVersions.cs | 35 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/build/tasks/VerifyCoherentVersions.cs b/build/tasks/VerifyCoherentVersions.cs index 1ea4afcfce..38eddb87ff 100644 --- a/build/tasks/VerifyCoherentVersions.cs +++ b/build/tasks/VerifyCoherentVersions.cs @@ -27,16 +27,20 @@ namespace RepoTasks public override bool Execute() { var packageLookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + var dependencyMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); - var dependencyMap = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var dep in ExternalDependencies) { - if (!dependencyMap.TryGetValue(dep.ItemSpec, out var externalDep)) + if (!dependencyMap.TryGetValue(dep.ItemSpec, out var list)) { - dependencyMap[dep.ItemSpec] = externalDep = new ExternalDependency(); + dependencyMap[dep.ItemSpec] = list = new List(); } - externalDep.IsPrivate = bool.TryParse(dep.GetMetadata("Private"), out var isPrivate) && isPrivate; - externalDep.AllowedVersions.Add(dep.GetMetadata("Version")); + var externalDep = new ExternalDependency + { + Version = dep.GetMetadata("Version"), + IsPrivate = bool.TryParse(dep.GetMetadata("Private"), out var isPrivate) && isPrivate, + }; + list.Add(externalDep); } foreach (var file in PackageFiles) @@ -74,13 +78,13 @@ namespace RepoTasks private class ExternalDependency { - public List AllowedVersions { get; } = new List(); + public string Version { get; set; } public bool IsPrivate { get; set; } } private void Visit( IReadOnlyDictionary packageLookup, - IReadOnlyDictionary dependencyMap, + IReadOnlyDictionary> dependencyMap, PackageInfo packageInfo) { Log.LogMessage(MessageImportance.Low, $"Processing package {packageInfo.Id}"); @@ -92,17 +96,20 @@ namespace RepoTasks { PackageInfo dependencyPackageInfo; var depVersion = dependency.VersionRange.MinVersion.ToString(); - if (dependencyMap.TryGetValue(dependency.Id, out var externalDepInfo)) + if (dependencyMap.TryGetValue(dependency.Id, out var externalDependencies)) { - if (externalDepInfo.IsPrivate) + var matchedVersion = externalDependencies.FirstOrDefault(d => depVersion.Equals(d.Version)); + + if (matchedVersion == null) + { + var versions = string.Join(" or ", externalDependencies.Select(d => d.Version)); + Log.LogError($"Package {packageInfo.Id} has an external dependency on the wrong version of {dependency.Id}. " + + $"It uses {depVersion} but only {versions} is allowed."); + } + else if (matchedVersion.IsPrivate) { Log.LogError($"Package {packageInfo.Id} has an external dependency on {dependency.Id}/{depVersion} which is marked as Private=true."); } - else if (!externalDepInfo.AllowedVersions.Any(a => depVersion.Equals(a))) - { - Log.LogError($"Package {packageInfo.Id} has an external dependency on the wrong version of {dependency.Id}. " - + $"It uses {depVersion} but only {string.Join(" or ", externalDepInfo.AllowedVersions)} is allowed."); - } continue; } else if (!packageLookup.TryGetValue(dependency.Id, out dependencyPackageInfo)) From 34e10f7b2ca1c8677781ab0414d8127bc8d9b15e Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 17:39:54 -0700 Subject: [PATCH 039/115] Add additional repositories that are being patched --- build/Repositories.props | 40 ++-- build/artifacts.props | 227 ++++++++++++++-------- build/repo.targets | 9 +- build/tasks/CopyPackagesToSplitFolders.cs | 2 +- 4 files changed, 178 insertions(+), 100 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index 793a7913f2..4048a5a71d 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -1,53 +1,53 @@ + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - diff --git a/build/artifacts.props b/build/artifacts.props index 7e703fbfee..4ff91c3429 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -1,80 +1,155 @@ + + + false + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/repo.targets b/build/repo.targets index 73a7bb8306..b0e1916ba5 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -91,8 +91,6 @@ - - @@ -211,11 +209,16 @@ + + + + + diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs index d9e43808ba..9162492f6d 100644 --- a/build/tasks/CopyPackagesToSplitFolders.cs +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -51,7 +51,7 @@ namespace RepoTasks if (!expectedPackages.TryGetCategory(identity.Id, out var category)) { Log.LogError($"Unexpected package artifact with id: {identity.Id}"); - return false; + continue; } string destDir; From f87c9d9fb5028c770c7958eae75420303d1a26ad Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 17:44:34 -0700 Subject: [PATCH 040/115] Increase the minimum download timeout --- build/tasks/DownloadNuGetPackages.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tasks/DownloadNuGetPackages.cs b/build/tasks/DownloadNuGetPackages.cs index 8412cc5fbd..f1d0d6300b 100644 --- a/build/tasks/DownloadNuGetPackages.cs +++ b/build/tasks/DownloadNuGetPackages.cs @@ -114,7 +114,7 @@ namespace RepoTasks } var all = Task.WhenAll(tasks); - var wait = TimeSpan.FromSeconds(Math.Max(downloadCount * 5, 30)); + var wait = TimeSpan.FromSeconds(Math.Max(downloadCount * 5, 120)); var delay = Task.Delay(wait); var finished = await Task.WhenAny(all, delay); From 6af6dde460703b75003c060ef0136dc00f96041c Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 20 Sep 2017 19:08:32 -0700 Subject: [PATCH 041/115] Fix unexpected failures in CopyPackagesToSplitFolders when files exists --- build/repo.targets | 14 ++++---------- build/tasks/CopyPackagesToSplitFolders.cs | 13 ++++++++++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index b0e1916ba5..1349205d8e 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -39,10 +39,7 @@ - - - + DestinationFolder="$(IntermediateMirrorPackageDir)" /> @@ -209,17 +206,14 @@ - + <_MirroredPackageFiles Include="$(IntermediateMirrorPackageDir)*.nupkg" /> - - - - + DestinationFolder="$(ArtifactsDir)" + Overwrite="true" /> diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs index 9162492f6d..ebb1b37bf5 100644 --- a/build/tasks/CopyPackagesToSplitFolders.cs +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -28,6 +28,8 @@ namespace RepoTasks [Required] public string DestinationFolder { get; set; } + public bool Overwrite { get; set; } + public override bool Execute() { if (Files?.Length == 0) @@ -79,9 +81,14 @@ namespace RepoTasks var destFile = Path.Combine(destDir, Path.GetFileName(file.ItemSpec)); - Log.LogMessage($"Copying {file.ItemSpec} to {destFile}"); + if (!Overwrite && File.Exists(destFile)) + { + Log.LogError($"File already exists in {destFile}"); + continue; + } - File.Copy(file.ItemSpec, destFile); + Log.LogMessage($"Copying {file.ItemSpec} to {destFile}"); + File.Copy(file.ItemSpec, destFile, Overwrite); expectedPackages.Remove(identity.Id); } @@ -97,7 +104,7 @@ namespace RepoTasks return false; } - return true; + return !Log.HasLoggedErrors; } } } From 293b4b52fae3bcb8da5b2e2435498b4531b4aa71 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 21 Sep 2017 14:18:51 -0700 Subject: [PATCH 042/115] Bump to 2.0.3 --- version.props | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 version.props diff --git a/version.props b/version.props new file mode 100644 index 0000000000..7e4c6138cc --- /dev/null +++ b/version.props @@ -0,0 +1,9 @@ + + + 2.0.3 + rtm + $(VersionPrefix) + $(VersionPrefix)-$(VersionSuffix)-final + $(VersionSuffix)-$(BuildNumber) + + From 8204f17234922e46868354d7ba105c2ad639cee4 Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 22 Sep 2017 15:19:22 -0700 Subject: [PATCH 043/115] Add latest runtime packages --- build/dependencies.props | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/build/dependencies.props b/build/dependencies.props index 4d8c14566c..e6e0b0fad2 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -15,6 +15,17 @@ + + + https://dotnet.myget.org/F/dotnet-core/api/v3/index.json + + + + + + + + https://dotnet.myget.org/F/roslyn/api/v3/index.json @@ -119,7 +130,6 @@ - From 44b84a8da78708cd3259fd6596bc740365a7b53e Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 27 Sep 2017 16:24:38 -0700 Subject: [PATCH 044/115] Update KoreBuild and remove unused MSBuild property --- build/repo.targets | 2 -- korebuild-lock.txt | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index 1349205d8e..7a7de53283 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -2,8 +2,6 @@ - https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package - <_CloneRepositoryRoot>$(RepositoryRoot).r\ <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ <_DependencyPackagesDirectory>$(_DependencyBuildDirectory) diff --git a/korebuild-lock.txt b/korebuild-lock.txt index e608c0f7f0..2b2aa04d50 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-beta-15503 -commithash:aa02c62a19780a960ce1bee2e453741df54f3703 +version:2.0.2-beta-15512 +commithash:27029a108e010e6037d46331a8a3f52da8cb6076 From d2e49a726c0d83cf6baccb07b4e1e7d1ee0712f4 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 29 Sep 2017 13:32:24 -0700 Subject: [PATCH 045/115] Generate the package version props file --- build/Repositories.props | 26 ++-- build/RepositoryBuild.targets | 1 + build/dependencies.props | 11 +- build/repo.targets | 16 ++- .../tasks/GeneratePackageVersionPropsFile.cs | 118 ++++++++++++++++++ build/tasks/RepoTasks.tasks | 1 + 6 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 build/tasks/GeneratePackageVersionPropsFile.cs diff --git a/build/Repositories.props b/build/Repositories.props index 4048a5a71d..adba85b527 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -36,18 +36,18 @@ Build tools will *verify* that these repos will be unaffected by the patch update and do not need updating. --> - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 6d119bbd50..4ff447a430 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -51,6 +51,7 @@ + $(RepositoryBuildArguments) /p:DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath) $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash) $(RepositoryBuildArguments) /noconsolelogger '/l:RepoTasks.FlowLogger,$(MSBuildThisFileDirectory)tasks\bin\publish\RepoTasks.dll;Summary;FlowId=$(RepositoryToBuild)' diff --git a/build/dependencies.props b/build/dependencies.props index e6e0b0fad2..312df26335 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -21,7 +21,10 @@ - + + RuntimeFrameworkVersion + netcoreapp2.0 + @@ -35,16 +38,19 @@ KRB2004 + MicrosoftCodeAnalysisCommonToolingPackageVersion KRB2004 + MicrosoftCodeAnalysisCSharpToolingPackageVersion KRB2004 + MicrosoftCodeAnalysisWorkspacesCommonToolingPackageVersion @@ -135,6 +141,7 @@ KRB2004 + NewtonsoftJsonToolingPackageVersion @@ -171,6 +178,7 @@ KRB2004 + EFBenchmarksXunitAssertPackageVersion @@ -181,6 +189,7 @@ KRB2004 + XunitStablePackageVersion diff --git a/build/repo.targets b/build/repo.targets index 7a7de53283..0a20957aaf 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -18,6 +18,7 @@ $(IntermediateDir)mirror\ $(IntermediateDir)ext\ + $(IntermediateDir)dependencies.props Patch20_ $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts @@ -40,6 +41,19 @@ DestinationFolder="$(IntermediateMirrorPackageDir)" /> + + + <_LineupPackages Include="@(ExternalDependency)" /> + <_LineupPackages Include="%(ArtifactInfo.PackageId)" Version="%(ArtifactInfo.Version)" Condition=" '%(ArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> + <_LineupPackages Include="%(ShippedArtifactInfo.PackageId)" Version="%(ShippedArtifactInfo.Version)" Condition=" '%(ShippedArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> + + + + + + @@ -125,7 +139,7 @@ + DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;GeneratePackageVersionPropsFile;ComputeGraph;_BuildRepositories" /> (); + var versionElements = new List(); + foreach (var pkg in Packages) + { + var packageVersion = pkg.GetMetadata("Version"); + + if (string.IsNullOrEmpty(packageVersion)) + { + Log.LogError("Package {0} is missing the Version metadata", pkg.ItemSpec); + continue; + } + + + string packageVarName; + if (!string.IsNullOrEmpty(pkg.GetMetadata("VariableName"))) + { + packageVarName = pkg.GetMetadata("VariableName"); + if (!packageVarName.EndsWith("Version", StringComparison.Ordinal)) + { + Log.LogError("VariableName for {0} must end in 'Version'", pkg.ItemSpec); + continue; + } + } + else + { + var sb = new StringBuilder(); + var first = true; + foreach (var ch in pkg.ItemSpec) + { + if (ch == '.') + { + first = true; + continue; + } + + if (first) + { + first = false; + sb.Append(char.ToUpperInvariant(ch)); + } + else + { + sb.Append(ch); + } + } + sb.Append("PackageVersion"); + packageVarName = sb.ToString(); + } + + var packageTfm = pkg.GetMetadata("TargetFramework"); + var key = $"{packageVarName}/{packageTfm}"; + if (varNames.Contains(key)) + { + Log.LogError("Multiple packages would produce {0} in the generated dependencies.props file. Set VariableName to differentiate the packages manually", key); + continue; + } + varNames.Add(key); + var elem = new XElement(packageVarName, packageVersion); + if (!string.IsNullOrEmpty(packageTfm)) + { + elem.Add(new XAttribute("Condition", $" '$(TargetFramework)' == '{packageTfm}' ")); + } + versionElements.Add(elem); + } + + foreach (var item in versionElements.OrderBy(p => p.Name.ToString())) + { + props.Add(item); + } + + var settings = new XmlWriterSettings + { + OmitXmlDeclaration = true, + Indent = true, + }; + using (var writer = XmlWriter.Create(OutputPath, settings)) + { + Log.LogMessage(MessageImportance.High, $"Generate {OutputPath}"); + doc.Save(writer); + } + return !Log.HasLoggedErrors; + } + } +} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index d2df6c836a..da9cb26603 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -7,5 +7,6 @@ + From a6a01db7a6c3e7312239662067d97a167319f0ed Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 29 Sep 2017 17:03:35 -0700 Subject: [PATCH 046/115] Include DotNetPackageVersionPropsPath in the design-time build --- build/Repositories.props | 2 +- build/repo.targets | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build/Repositories.props b/build/Repositories.props index adba85b527..0c9265e5d8 100644 --- a/build/Repositories.props +++ b/build/Repositories.props @@ -17,7 +17,7 @@ - + diff --git a/build/repo.targets b/build/repo.targets index 0a20957aaf..989af75856 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -3,6 +3,7 @@ <_CloneRepositoryRoot>$(RepositoryRoot).r\ + false <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ <_DependencyPackagesDirectory>$(_DependencyBuildDirectory) @@ -80,7 +81,7 @@ - + <_CloneRepositories Include="@(Repository)" /> <_CloneRepositories Remove="%(Repository.Identity)" Condition="$([System.Environment]::GetEnvironmentVariable('BUILD_VCS_NUMBER_$(TcVcsIdPrefix)%(Repository.Identity)')) != ''" /> @@ -182,14 +183,14 @@ - + + Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber);DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath)"> From 372be2f638ab4a1d24972490f6e506ad77d7faf6 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 2 Oct 2017 12:59:06 -0700 Subject: [PATCH 047/115] Generate and set DotNetRestoreSourcePropsPath --- .vscode/launch.json | 11 ++++ build/RepositoryBuild.targets | 4 +- build/repo.targets | 22 +++++--- build/tasks/GenerateLineup.cs | 12 +---- .../tasks/GenerateRestoreSourcesPropsFile.cs | 52 +++++++++++++++++++ build/tasks/RepoTasks.tasks | 1 + 6 files changed, 82 insertions(+), 20 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 build/tasks/GenerateRestoreSourcesPropsFile.cs diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..396e7edb12 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,11 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach", + "processId": "${command:pickProcess}" + } + ] +} diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 4ff447a430..2f92b51d9a 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -18,8 +18,6 @@ - - @@ -35,7 +33,6 @@ Artifacts="@(PinnedArtifactInfo)" Repository="%(RepositoryBuildOrder.Identity)" OutputPath="%(RepositoryBuildOrder.RepositoryPath)\build\dependencies.g.targets" - RestoreAdditionalSources="@(PinPackageSources)" UseFloatingVersions="false" Condition="'%(RepositoryBuildOrder.RepositoryPath)' != ''" BuildNumber="$(BuildNumber)" /> @@ -51,6 +48,7 @@ + $(RepositoryBuildArguments) /p:DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath) $(RepositoryBuildArguments) /p:DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath) $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash) $(RepositoryBuildArguments) /noconsolelogger '/l:RepoTasks.FlowLogger,$(MSBuildThisFileDirectory)tasks\bin\publish\RepoTasks.dll;Summary;FlowId=$(RepositoryToBuild)' diff --git a/build/repo.targets b/build/repo.targets index 989af75856..fde013f088 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -20,10 +20,10 @@ $(IntermediateDir)ext\ $(IntermediateDir)dependencies.props + $(IntermediateDir)sources.props Patch20_ - $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts - $(CleanDependsOn);CleanUniverseArtifacts + $(CleanDependsOn);CleanArtifacts;CleanUniverseArtifacts $(RestoreDependsOn);RestoreExternalDependencies $(CompileDependsOn);CloneRepositories;BuildRepositories $(PackageDependsOn);SplitPackages @@ -42,17 +42,27 @@ DestinationFolder="$(IntermediateMirrorPackageDir)" /> - + <_LineupPackages Include="@(ExternalDependency)" /> <_LineupPackages Include="%(ArtifactInfo.PackageId)" Version="%(ArtifactInfo.Version)" Condition=" '%(ArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> <_LineupPackages Include="%(ShippedArtifactInfo.PackageId)" Version="%(ShippedArtifactInfo.Version)" Condition=" '%(ShippedArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> + + <_LineupSources Include="$(_DependencyPackagesDirectory)" Condition="'$(_DependencyPackagesDirectory)' != '' AND Exists('$(_DependencyPackagesDirectory)')" /> + <_LineupSources Include="$(BuildDir)" /> + <_LineupSources Include="$(IntermediateExternalPackageDir)" /> + <_LineupSources Include="$(IntermediateMirrorPackageDir)" /> + + + @@ -140,7 +150,7 @@ + DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;GeneratePropsFiles;ComputeGraph;_BuildRepositories" /> - + + Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber);DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath)"> diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs index d1114814b5..5053ebdfa2 100644 --- a/build/tasks/GenerateLineup.cs +++ b/build/tasks/GenerateLineup.cs @@ -25,9 +25,6 @@ namespace RepoTasks // Can be set to filter the lists of packages when produce a list for a specific repository public string Repository { get; set; } - // Items to add to the RestoreAdditionalProjectSources list in project - public ITaskItem[] RestoreAdditionalSources { get; set; } - public bool UseFloatingVersions { get; set; } public string BuildNumber { get; set; } @@ -43,16 +40,9 @@ namespace RepoTasks } var items = new XElement("ItemGroup"); - var props = new XElement("PropertyGroup"); - var root = new XElement("Project", props, items); + var root = new XElement("Project", items); var doc = new XDocument(root); - if (RestoreAdditionalSources?.Length > 0) - { - var sources = RestoreAdditionalSources.Aggregate("$(RestoreAdditionalProjectSources)", (sum, piece) => sum + ";" + piece.ItemSpec); - props.Add(new XElement("RestoreAdditionalProjectSources", sources)); - } - var packages = new List(); foreach (var pkg in Artifacts.Select(ArtifactInfo.Parse) diff --git a/build/tasks/GenerateRestoreSourcesPropsFile.cs b/build/tasks/GenerateRestoreSourcesPropsFile.cs new file mode 100644 index 0000000000..75343a16bc --- /dev/null +++ b/build/tasks/GenerateRestoreSourcesPropsFile.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.IO; +using System.Text; +using System.Xml; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + public class GenerateRestoreSourcesPropsFile : Task + { + [Required] + public ITaskItem[] Sources { get; set; } + + [Required] + public string OutputPath { get; set; } + + public override bool Execute() + { + OutputPath = OutputPath.Replace('\\', '/'); + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + var sources = new XElement("DotNetRestoreSources"); + var doc = new XDocument(new XElement("Project", new XElement("PropertyGroup", sources))); + + var sb = new StringBuilder(); + + foreach (var source in Sources) + { + sb.Append(source.ItemSpec).AppendLine(";"); + } + + sources.SetValue(sb.ToString()); + + var settings = new XmlWriterSettings + { + OmitXmlDeclaration = true, + }; + using (var writer = XmlWriter.Create(OutputPath, settings)) + { + Log.LogMessage(MessageImportance.High, $"Generate {OutputPath}"); + doc.Save(writer); + } + return !Log.HasLoggedErrors; + } + } +} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index da9cb26603..4990d23c33 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -8,5 +8,6 @@ + From c3c891952346aeb88ffe09489a4393c786f2ed5c Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 2 Oct 2017 12:59:32 -0700 Subject: [PATCH 048/115] Mirror Microsoft.NETCore.App 2.0.2 and its dependencies --- build/dependencies.props | 57 ++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index 312df26335..63a64327c9 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -18,42 +18,78 @@ https://dotnet.myget.org/F/dotnet-core/api/v3/index.json + 2.0.2-servicing-25728-02 - + + + RuntimeFrameworkVersion netcoreapp2.0 - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https://dotnet.myget.org/F/roslyn/api/v3/index.json + 2.6.0-beta1-61924-08 - + KRB2004 MicrosoftCodeAnalysisCommonToolingPackageVersion - + KRB2004 MicrosoftCodeAnalysisCSharpToolingPackageVersion - - - + + + KRB2004 MicrosoftCodeAnalysisWorkspacesCommonToolingPackageVersion - - + + @@ -116,7 +152,6 @@ - From 11b25e7c877d1a99767996531e0e83b623eaf962 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 21 Sep 2017 01:09:13 -0700 Subject: [PATCH 049/115] Build and pack Runtime Store - Also add targets to build all metapackage. --- .gitignore | 1 + NuGet.Config | 3 +- build/Key.snk | Bin 0 -> 596 bytes build/RuntimeStore.targets | 191 ++++++++++++++++++ build/artifacts.props | 8 +- build/common.props | 20 ++ build/dependencies.props | 4 + build/repo.props | 1 - build/repo.targets | 2 +- build/tasks/AddMetapackageReferences.cs | 64 ++++++ build/tasks/ComposeNewStore.cs | 131 ++++++++++++ build/tasks/ConsolidateManifests.cs | 57 ++++++ build/tasks/CreateCommonManifest.cs | 61 ++++++ build/tasks/RepoTasks.csproj | 1 + build/tasks/RepoTasks.tasks | 6 + build/tasks/ResolveHostingStartupPackages.cs | 30 +++ build/tasks/TrimDeps.cs | 60 ++++++ .../scripts/GetSharedFrameworkVersion.ps1 | 6 + .../scripts/GetSharedFrameworkVersion.sh | 2 + .../HostingStartup/HostingStartup.csproj | 20 ++ .../tools/templates/HostingStartup/Program.cs | 7 + .../templates/RS.Manifest/RS.Manifest.csproj | 22 ++ .../Microsoft.AspNetCore.All.csproj | 19 ++ ...ublishWithAspNetCoreTargetManifest.targets | 29 +++ .../build/aspnetcore-store-2.0.0-common.xml | 3 + .../aspnetcore-store-2.0.0-linux-x64.xml | 171 ++++++++++++++++ .../build/aspnetcore-store-2.0.0-osx-x64.xml | 171 ++++++++++++++++ .../build/aspnetcore-store-2.0.0-win7-x64.xml | 172 ++++++++++++++++ .../build/aspnetcore-store-2.0.0-win7-x86.xml | 172 ++++++++++++++++ .../Microsoft.AspNetCore.All.targets | 3 + .../lib/netcoreapp2.0/_._ | 0 31 files changed, 1430 insertions(+), 7 deletions(-) create mode 100644 build/Key.snk create mode 100644 build/RuntimeStore.targets create mode 100644 build/common.props create mode 100644 build/tasks/AddMetapackageReferences.cs create mode 100644 build/tasks/ComposeNewStore.cs create mode 100644 build/tasks/ConsolidateManifests.cs create mode 100644 build/tasks/CreateCommonManifest.cs create mode 100644 build/tasks/ResolveHostingStartupPackages.cs create mode 100644 build/tasks/TrimDeps.cs create mode 100644 build/tools/scripts/GetSharedFrameworkVersion.ps1 create mode 100755 build/tools/scripts/GetSharedFrameworkVersion.sh create mode 100644 build/tools/templates/HostingStartup/HostingStartup.csproj create mode 100644 build/tools/templates/HostingStartup/Program.cs create mode 100644 build/tools/templates/RS.Manifest/RS.Manifest.csproj create mode 100644 src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj create mode 100644 src/Microsoft.AspNetCore.All/build/PublishWithAspNetCoreTargetManifest.targets create mode 100755 src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-common.xml create mode 100755 src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-linux-x64.xml create mode 100755 src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-osx-x64.xml create mode 100755 src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x64.xml create mode 100755 src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x86.xml create mode 100644 src/Microsoft.AspNetCore.All/build/netcoreapp2.0/Microsoft.AspNetCore.All.targets create mode 100644 src/Microsoft.AspNetCore.All/lib/netcoreapp2.0/_._ diff --git a/.gitignore b/.gitignore index 9aed881a4b..a68780e24f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ node_modules .build .nuget .r +.w .deps global.json korebuild-lock.txt diff --git a/NuGet.Config b/NuGet.Config index cbeb8fa20d..c4bc056c4d 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -3,6 +3,7 @@ + - + \ No newline at end of file diff --git a/build/Key.snk b/build/Key.snk new file mode 100644 index 0000000000000000000000000000000000000000..e10e4889c125d3120cd9e81582243d70f7cbb806 GIT binary patch literal 596 zcmV-a0;~N80ssI2Bme+XQ$aES1ONa50098=Iw=HCsnz~#iVhm& zj%TU(_THUee?3yHBjk$37ysB?i5#7WD$={H zV4B!OxRPrb|8)HPg~A}8P>^=#y<)56#=E&NzcjOtPK~<4n6GHt=K$ro*T(lhby_@U zEk(hLzk1H)0yXj{A_5>fk-TgNoP|q6(tP2xo8zt8i%212CWM#AeCd?`hS|4~L({h~Moo(~vy&3Z z1uI}`fd^*>o=rwbAGymj6RM^pZm(*Kfhs+Y1#`-2JPWZMK8@;ZWCk2+9bX4YP);~fj-BU*R zQPvWv$89!{Rl9wM+zR>_TSkn^voYxA?2G iKnV#iZ6Ah`K>b=@=IjYJXrxL124zR(38)nxe+&q_$QXwJ literal 0 HcmV?d00001 diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets new file mode 100644 index 0000000000..599882ca5e --- /dev/null +++ b/build/RuntimeStore.targets @@ -0,0 +1,191 @@ + + + + + <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ + <_BuildScriptsDirectory>$(MSBuildThisFileDirectory)tools\scripts\ + <_WorkRoot>$(RepositoryRoot).w\ + <_RuntimeStoreWorkDirectory>$(_WorkRoot).rw\ + <_RuntimeStoreOutputDirectory>$(_WorkRoot).ro\ + <_TemplatesDirectory>$(MSBuildThisFileDirectory)tools\templates\ + <_SrcDirectory>$(RepositoryRoot)src\ + <_AllMetapackageDirectory>$(_SrcDirectory)Microsoft.AspNetCore.All\ + <_ExistingManifestsDirectory>$(_AllMetapackageDirectory)build\ + <_ArtifactsZipDirectory>$(ArtifactsDir)zip\ + <_StoreZipDirectory>$(_ArtifactsZipDirectory)rs\ + <_SymbolsZipDirectory>$(_ArtifactsZipDirectory)symbols\ + <_DepsOutputDirectory>$(ArtifactsDir)deps\ + + + + + + + + + $(_WorkRoot)Microsoft.AspNetCore.All\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + win7-$(RuntimeStoreArch) + linux-$(RuntimeStoreArch) + osx-$(RuntimeStoreArch) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(ArtifactsDir)aspnetcore-store-$(VersionPrefix)-$(VersionSuffix)-$(RuntimeStoreRID).zip + $(ArtifactsDir)aspnetcore-symbols-$(VersionPrefix)-$(VersionSuffix)-$(RuntimeStoreRID).zip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + aspnetcore-store-$(VersionPrefix)-$(VersionSuffix)-common.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build/artifacts.props b/build/artifacts.props index 4ff91c3429..5aee8bffa3 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -9,7 +9,7 @@ - + @@ -23,7 +23,7 @@ - + @@ -37,7 +37,7 @@ - + @@ -46,7 +46,7 @@ - + diff --git a/build/common.props b/build/common.props new file mode 100644 index 0000000000..f7b678dcf2 --- /dev/null +++ b/build/common.props @@ -0,0 +1,20 @@ + + + + + + Microsoft ASP.NET Core + https://github.com/aspnet/MetaPackages + git + false + ..\..\build\Key.snk + true + true + true + + + + + + + \ No newline at end of file diff --git a/build/dependencies.props b/build/dependencies.props index 63a64327c9..f0452cf9db 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -231,4 +231,8 @@ KRB2004 + + + 2.0.1-rtm-15400 + diff --git a/build/repo.props b/build/repo.props index 63a8d665d0..a98645c000 100644 --- a/build/repo.props +++ b/build/repo.props @@ -5,5 +5,4 @@ - diff --git a/build/repo.targets b/build/repo.targets index fde013f088..e7e6c89de6 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -1,5 +1,6 @@ + <_CloneRepositoryRoot>$(RepositoryRoot).r\ @@ -302,5 +303,4 @@ <_CloneUrl> - diff --git a/build/tasks/AddMetapackageReferences.cs b/build/tasks/AddMetapackageReferences.cs new file mode 100644 index 0000000000..3db68880cc --- /dev/null +++ b/build/tasks/AddMetapackageReferences.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Xml; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class AddMetapackageReferences : Task + { + [Required] + public string ReferencePackagePath { get; set; } + + [Required] + public ITaskItem[] BuildArtifacts { get; set; } + + [Required] + public ITaskItem[] PackageArtifacts { get; set; } + + public override bool Execute() + { + // Parse input + var metapackageArtifacts = PackageArtifacts.Where(p => p.GetMetadata("Metapackage") != "false"); + var buildArtifacts = BuildArtifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact); + + var xmlDoc = new XmlDocument(); + xmlDoc.Load(ReferencePackagePath); + + // Project + var projectElement = xmlDoc.FirstChild; + + // Items + var itemGroupElement = xmlDoc.CreateElement("ItemGroup"); + Log.LogMessage(MessageImportance.Normal, $"Runtime store will include the following packages"); + foreach (var package in metapackageArtifacts) + { + var packageName = package.GetMetadata("Identity"); + var packageVersion = buildArtifacts + .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) + .PackageInfo.Version.ToString(); + Log.LogMessage(MessageImportance.Normal, $" - Package: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + + itemGroupElement.AppendChild(packageReferenceElement); + } + projectElement.AppendChild(itemGroupElement); + + // Save updated file + xmlDoc.AppendChild(projectElement); + xmlDoc.Save(ReferencePackagePath); + + return true; + } + } +} diff --git a/build/tasks/ComposeNewStore.cs b/build/tasks/ComposeNewStore.cs new file mode 100644 index 0000000000..1694f5ef25 --- /dev/null +++ b/build/tasks/ComposeNewStore.cs @@ -0,0 +1,131 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.IO; +using System.Xml; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + public class ComposeNewStore : Task + { + [Required] + public ITaskItem[] ExistingManifests { get; set; } + + [Required] + public ITaskItem[] NewManifests { get; set; } + + [Required] + public ITaskItem[] RuntimeStoreFiles { get; set; } + + [Required] + public ITaskItem[] RuntimeStoreSymbolFiles { get; set; } + + [Required] + public string ManifestDestination { get; set; } + + [Required] + public string StoreDestination { get; set; } + + [Required] + public string SymbolsDestination { get; set; } + + public override bool Execute() + { + var existingFiles = new Dictionary>(); + var newRuntimeStoreFiles = new List(); + var newRuntimeStoreSymbolFiles = new List(); + + // Construct database of existing assets + foreach (var manifest in ExistingManifests) + { + var xmlDoc = new XmlDocument(); + xmlDoc.Load(manifest.GetMetadata("FullPath")); + var storeArtifacts = xmlDoc.SelectSingleNode("/StoreArtifacts"); + + foreach (XmlNode artifact in storeArtifacts.ChildNodes) + { + if (existingFiles.TryGetValue(artifact.Attributes["Id"].Value, out var versions)) + { + versions.Add(artifact.Attributes["Version"].Value); + } + else + { + existingFiles[artifact.Attributes["Id"].Value] = new HashSet{ artifact.Attributes["Version"].Value }; + } + } + } + + // Insert new runtime store files + foreach (var storeFile in RuntimeStoreFiles) + { + // format: {bitness}}/{tfm}}/{id}/{version}}/... + var recursiveDir = storeFile.GetMetadata("RecursiveDir"); + var components = recursiveDir.Split(Path.DirectorySeparatorChar); + var id = components[2]; + var version = components[3]; + + if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) + { + var destinationDir = Path.Combine(StoreDestination, recursiveDir); + if (!Directory.Exists(Path.Combine(StoreDestination, recursiveDir))) + { + Directory.CreateDirectory(destinationDir); + } + + File.Copy(storeFile.GetMetadata("FullPath"), Path.Combine(destinationDir, $"{storeFile.GetMetadata("Filename")}{storeFile.GetMetadata("Extension")}"), overwrite: true); + } + } + + // Insert new runtime store files + foreach (var symbolFile in RuntimeStoreSymbolFiles) + { + // format: {bitness}}/{tfm}}/{id}/{version}}/... + var recursiveDir = symbolFile.GetMetadata("RecursiveDir"); + var components = recursiveDir.Split(Path.DirectorySeparatorChar); + var id = components[2]; + var version = components[3]; + + if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) + { + var destinationDir = Path.Combine(StoreDestination, recursiveDir); + if (!Directory.Exists(Path.Combine(StoreDestination, recursiveDir))) + { + Directory.CreateDirectory(destinationDir); + } + + File.Copy(symbolFile.GetMetadata("FullPath"), Path.Combine(destinationDir, $"{symbolFile.GetMetadata("Filename")}{symbolFile.GetMetadata("Extension")}"), overwrite: true); + } + } + + // Purge existing packages from manifest + foreach (var newManifest in NewManifests) + { + var newManifestPath = newManifest.GetMetadata("FullPath"); + var xmlDoc = new XmlDocument(); + xmlDoc.Load(newManifestPath); + var storeArtifacts = xmlDoc.SelectSingleNode("/StoreArtifacts"); + var artifactsToRemove = new List(); + + foreach (XmlNode artifact in storeArtifacts.ChildNodes) + { + if (existingFiles.TryGetValue(artifact.Attributes["Id"].Value, out var versions) && versions.Contains(artifact.Attributes["Version"].Value)) + { + artifactsToRemove.Add(artifact); + } + } + + foreach (var artifactToRemove in artifactsToRemove) + { + storeArtifacts.RemoveChild(artifactToRemove); + } + + xmlDoc.Save(ManifestDestination); + } + + return true; + } + } +} diff --git a/build/tasks/ConsolidateManifests.cs b/build/tasks/ConsolidateManifests.cs new file mode 100644 index 0000000000..12838855a6 --- /dev/null +++ b/build/tasks/ConsolidateManifests.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Xml; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + // Takes multiple runtime store manifests and create a consolidated manifest containing all unique entries. + public class ConsolidateManifests : Task + { + [Required] + public ITaskItem[] Manifests { get; set; } + + [Required] + public string ManifestDestination { get; set; } + + public override bool Execute() + { + var artifacts = new HashSet>(); + + // Construct database of all artifacts + foreach (var manifest in Manifests) + { + var xmlDoc = new XmlDocument(); + xmlDoc.Load(manifest.GetMetadata("FullPath")); + var storeArtifacts = xmlDoc.SelectSingleNode("/StoreArtifacts"); + + foreach (XmlNode artifact in storeArtifacts.ChildNodes) + { + artifacts.Add(new Tuple(artifact.Attributes["Id"].Value, artifact.Attributes["Version"].Value)); + } + } + + + var consolidatedXmlDoc = new XmlDocument(); + var packagesElement = consolidatedXmlDoc.CreateElement("StoreArtifacts"); + + foreach (var artifact in artifacts) + { + var packageElement = consolidatedXmlDoc.CreateElement("Package"); + packageElement.SetAttribute("Id", artifact.Item1); + packageElement.SetAttribute("Version", artifact.Item2); + + packagesElement.AppendChild(packageElement); + } + + consolidatedXmlDoc.AppendChild(packagesElement); + consolidatedXmlDoc.Save(ManifestDestination); + + return true; + } + } +} diff --git a/build/tasks/CreateCommonManifest.cs b/build/tasks/CreateCommonManifest.cs new file mode 100644 index 0000000000..5a412fa8e9 --- /dev/null +++ b/build/tasks/CreateCommonManifest.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Xml; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + /// + /// Creates a common manifest file used for trimming publish output given a list of packages and package definitions containing their versions. + /// + public class CreateCommonManifest : Task + { + /// + /// The path for the common manifest file to be created. + /// + /// + [Required] + public string DestinationFilePath { get; set; } + + /// + /// The packages to include in the common manifest file. + /// + /// + [Required] + public ITaskItem[] Packages { get; set; } + + /// + /// The package definitions used for resolving package versions. + /// + /// + [Required] + public ITaskItem[] PackageDefinitions { get; set; } + + public override bool Execute() + { + var xmlDoc = new XmlDocument(); + var packagesElement = xmlDoc.CreateElement("StoreArtifacts"); + + foreach (var package in Packages) + { + var packageName = package.ItemSpec; + var packageElement = xmlDoc.CreateElement("Package"); + packageElement.SetAttribute("Id", packageName); + packageElement.SetAttribute("Version", PackageDefinitions + .Where(p => string.Equals(p.GetMetadata("Name"), packageName, StringComparison.OrdinalIgnoreCase)) + .Select(p => p.GetMetadata("Version")).Single()); + + packagesElement.AppendChild(packageElement); + } + + xmlDoc.AppendChild(packagesElement); + xmlDoc.Save(DestinationFilePath); + + return true; + } + } +} diff --git a/build/tasks/RepoTasks.csproj b/build/tasks/RepoTasks.csproj index 39574ccdb1..a52ecec4f4 100644 --- a/build/tasks/RepoTasks.csproj +++ b/build/tasks/RepoTasks.csproj @@ -7,6 +7,7 @@ + diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 4990d23c33..5dda060bf5 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -10,4 +10,10 @@ + + + + + + diff --git a/build/tasks/ResolveHostingStartupPackages.cs b/build/tasks/ResolveHostingStartupPackages.cs new file mode 100644 index 0000000000..0d5b0838f4 --- /dev/null +++ b/build/tasks/ResolveHostingStartupPackages.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + public class ResolveHostingStartupPackages : Task + { + [Required] + public ITaskItem[] BuildArtifacts { get; set; } + + [Required] + public ITaskItem[] PackageArtifacts { get; set; } + + [Output] + public ITaskItem[] HostingStartupArtifacts { get; set; } + + public override bool Execute() + { + // Parse input + var hostingStartupArtifacts = PackageArtifacts.Where(p => p.GetMetadata("Metapackage") == "hostingstartup"); + HostingStartupArtifacts = BuildArtifacts.Where(p => hostingStartupArtifacts.Any(h => h.GetMetadata("Identity") == p.GetMetadata("PackageId"))).ToArray(); + + return true; + } + } +} diff --git a/build/tasks/TrimDeps.cs b/build/tasks/TrimDeps.cs new file mode 100644 index 0000000000..b63bdb35cc --- /dev/null +++ b/build/tasks/TrimDeps.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace RepoTasks +{ + public class TrimDeps : Task + { + [Required] + public ITaskItem[] DepsFiles { get; set; } + + public override bool Execute() + { + foreach (var depsFile in DepsFiles) + { + ChangeEntryPointLibraryName(depsFile.GetMetadata("Identity")); + } + + // Parse input + return true; + } + + + private void ChangeEntryPointLibraryName(string depsFile) + { + JToken deps; + using (var file = File.OpenText(depsFile)) + using (JsonTextReader reader = new JsonTextReader(file)) + { + deps = JObject.ReadFrom(reader); + } + + foreach (JProperty target in deps["targets"]) + { + var targetLibrary = target.Value.Children().FirstOrDefault(); + if (targetLibrary == null) + { + continue; + } + + targetLibrary.Remove(); + } + + var library = deps["libraries"].Children().First(); + library.Remove(); + + using (var file = File.CreateText(depsFile)) + using (var writer = new JsonTextWriter(file) { Formatting = Formatting.Indented }) + { + deps.WriteTo(writer); + } + } + } +} diff --git a/build/tools/scripts/GetSharedFrameworkVersion.ps1 b/build/tools/scripts/GetSharedFrameworkVersion.ps1 new file mode 100644 index 0000000000..37772625fe --- /dev/null +++ b/build/tools/scripts/GetSharedFrameworkVersion.ps1 @@ -0,0 +1,6 @@ +$infoOutput = dotnet --info +$versions = $infoOutput | Select-String -Pattern "version" +$FXVersionRaw = $versions | Select-Object -Last 1 +$FXVersionString = $FXVersionRaw.ToString() +$FXVersion = $FXVersionString.SubString($FXVersionString.IndexOf(':') + 1).Trim() +Write-Host $FXVersion diff --git a/build/tools/scripts/GetSharedFrameworkVersion.sh b/build/tools/scripts/GetSharedFrameworkVersion.sh new file mode 100755 index 0000000000..dee9ea2db8 --- /dev/null +++ b/build/tools/scripts/GetSharedFrameworkVersion.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +dotnet --info | grep -i version | tail -1 | cut -f 2 -d ":" | tr -d ' ' \ No newline at end of file diff --git a/build/tools/templates/HostingStartup/HostingStartup.csproj b/build/tools/templates/HostingStartup/HostingStartup.csproj new file mode 100644 index 0000000000..dbb96336d5 --- /dev/null +++ b/build/tools/templates/HostingStartup/HostingStartup.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp2.0 + Exe + + + + + + + + + $(RuntimeFrameworkVersion) + $(DepsOutputPath)\$(HostingStartupPackageName)\shared\Microsoft.NETCore.App\$(DepsRuntimeFrameworkVersion)\$(HostingStartupPackageName).deps.json + + + + + diff --git a/build/tools/templates/HostingStartup/Program.cs b/build/tools/templates/HostingStartup/Program.cs new file mode 100644 index 0000000000..8cd3c0026c --- /dev/null +++ b/build/tools/templates/HostingStartup/Program.cs @@ -0,0 +1,7 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +public class Program +{ + public static void Main() { } +} \ No newline at end of file diff --git a/build/tools/templates/RS.Manifest/RS.Manifest.csproj b/build/tools/templates/RS.Manifest/RS.Manifest.csproj new file mode 100644 index 0000000000..0e9d1c5186 --- /dev/null +++ b/build/tools/templates/RS.Manifest/RS.Manifest.csproj @@ -0,0 +1,22 @@ + + + + + + netcoreapp2.0 + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj new file mode 100644 index 0000000000..bdbf65814c --- /dev/null +++ b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj @@ -0,0 +1,19 @@ + + + + + + false + netcoreapp2.0 + aspnetcore + Microsoft.AspNetCore.All + false + + + + + + + + + diff --git a/src/Microsoft.AspNetCore.All/build/PublishWithAspNetCoreTargetManifest.targets b/src/Microsoft.AspNetCore.All/build/PublishWithAspNetCoreTargetManifest.targets new file mode 100644 index 0000000000..a590cf01f1 --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/PublishWithAspNetCoreTargetManifest.targets @@ -0,0 +1,29 @@ + + + true + + + + + + + + + + + + + $(TargetManifestFiles);@(AspNetCoreTargetManifestFiles) + + + diff --git a/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-common.xml b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-common.xml new file mode 100755 index 0000000000..3b0c4bda7c --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-common.xml @@ -0,0 +1,3 @@ + + + diff --git a/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-linux-x64.xml b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-linux-x64.xml new file mode 100755 index 0000000000..e548a00368 --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-linux-x64.xml @@ -0,0 +1,171 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-osx-x64.xml b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-osx-x64.xml new file mode 100755 index 0000000000..369db3ef5b --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-osx-x64.xml @@ -0,0 +1,171 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x64.xml b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x64.xml new file mode 100755 index 0000000000..0155a8e72c --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x64.xml @@ -0,0 +1,172 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x86.xml b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x86.xml new file mode 100755 index 0000000000..362dab9895 --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.0-win7-x86.xml @@ -0,0 +1,172 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.AspNetCore.All/build/netcoreapp2.0/Microsoft.AspNetCore.All.targets b/src/Microsoft.AspNetCore.All/build/netcoreapp2.0/Microsoft.AspNetCore.All.targets new file mode 100644 index 0000000000..0bed51f606 --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/netcoreapp2.0/Microsoft.AspNetCore.All.targets @@ -0,0 +1,3 @@ + + + diff --git a/src/Microsoft.AspNetCore.All/lib/netcoreapp2.0/_._ b/src/Microsoft.AspNetCore.All/lib/netcoreapp2.0/_._ new file mode 100644 index 0000000000..e69de29bb2 From 8cba694aa40886f9399bd3902cdd248f533409e3 Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 2 Oct 2017 16:48:09 -0700 Subject: [PATCH 050/115] Build all metapackage before split packages --- build/repo.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/repo.targets b/build/repo.targets index e7e6c89de6..8fac6e8176 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -27,7 +27,7 @@ $(CleanDependsOn);CleanArtifacts;CleanUniverseArtifacts $(RestoreDependsOn);RestoreExternalDependencies $(CompileDependsOn);CloneRepositories;BuildRepositories - $(PackageDependsOn);SplitPackages + $(PackageDependsOn);BuildAllMetapackage;SplitPackages $(VerifyDependsOn);VerifyCoherentVersions From e3efdd5feb101ca1b1bf6cf9d0400d73fd1bbdbd Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 2 Oct 2017 17:18:27 -0700 Subject: [PATCH 051/115] Add BuildDir as restore source --- build/RuntimeStore.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 599882ca5e..54487281b9 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -40,7 +40,7 @@ + Properties="Configuration=$(Configuration);RestoreFallbackFolders=$(BuildDir)" /> From e24913ae4632542cf08a2a5f0b1ab088366ed3b1 Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 2 Oct 2017 18:16:40 -0700 Subject: [PATCH 052/115] Try add restore sources when building metapackage --- build/RuntimeStore.targets | 2 +- build/common.props | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 54487281b9..599882ca5e 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -40,7 +40,7 @@ + Properties="Configuration=$(Configuration)" /> diff --git a/build/common.props b/build/common.props index f7b678dcf2..3c035106e3 100644 --- a/build/common.props +++ b/build/common.props @@ -1,6 +1,7 @@ + Microsoft ASP.NET Core From f6fa111333f4014bac6bde9f7cfcc386eac260d6 Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 2 Oct 2017 19:18:12 -0700 Subject: [PATCH 053/115] Another attempt --- build/RuntimeStore.targets | 2 +- build/common.props | 1 - src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 599882ca5e..6888bd3106 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -40,7 +40,7 @@ + Properties="Configuration=$(Configuration);AdditionalImports=$(GeneratedRestoreSourcesPropsPath)" /> diff --git a/build/common.props b/build/common.props index 3c035106e3..f7b678dcf2 100644 --- a/build/common.props +++ b/build/common.props @@ -1,7 +1,6 @@ - Microsoft ASP.NET Core diff --git a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj index bdbf65814c..79df5e1ea1 100644 --- a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj +++ b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj @@ -1,6 +1,7 @@ + false From 37c12d5a648ec2e7b7e0477bc1655e1bcae0fb33 Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 3 Oct 2017 11:46:57 -0700 Subject: [PATCH 054/115] Add sources for the final time --- build/RuntimeStore.targets | 2 +- .../Microsoft.AspNetCore.All.csproj | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 6888bd3106..96f092c204 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -40,7 +40,7 @@ + Properties="Configuration=$(Configuration);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath)" /> diff --git a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj index 79df5e1ea1..95e31dbeb0 100644 --- a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj +++ b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj @@ -1,9 +1,18 @@ - + + $(DotNetRestoreSources) + + $(RestoreSources); + https://dotnet.myget.org/F/aspnet-2-0-2-october2017-patch/api/v3/index.json; + https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json; + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; + https://api.nuget.org/v3/index.json; + + false netcoreapp2.0 aspnetcore From 15bfc962b77f4aedbf9c2a2153c9d5b38a86f42d Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 3 Oct 2017 12:18:39 -0700 Subject: [PATCH 055/115] Fix typo --- build/RuntimeStore.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 96f092c204..304d8bf154 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -44,7 +44,7 @@ - + From 955cd09ddcf8178db288774b3f76f04b53e925ae Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 3 Oct 2017 15:12:48 -0700 Subject: [PATCH 056/115] Add mirror dependencies --- build/RuntimeStore.targets | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 304d8bf154..b6ad613cd2 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -3,6 +3,7 @@ <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ + <_DependencyMirrorDirectory>$(RepositoryRoot).deps\mirror\ <_BuildScriptsDirectory>$(MSBuildThisFileDirectory)tools\scripts\ <_WorkRoot>$(RepositoryRoot).w\ <_RuntimeStoreWorkDirectory>$(_WorkRoot).rw\ @@ -65,6 +66,11 @@ SourceName="Dependencies" SourceUri="$(_DependencyBuildDirectory)" Condition="Exists('$(_DependencyBuildDirectory)')" /> + From f7ff4455010b1adafcbca905e413a77b52c6c8b3 Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 3 Oct 2017 15:52:06 -0700 Subject: [PATCH 057/115] Casing of NuGet.config --- NuGet.Config => NuGet.config | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename NuGet.Config => NuGet.config (100%) diff --git a/NuGet.Config b/NuGet.config similarity index 100% rename from NuGet.Config rename to NuGet.config From 5fe87fcc6bd6ce79e2dfca867fda4e8324543e1b Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 3 Oct 2017 16:05:32 -0700 Subject: [PATCH 058/115] Update KoreBuild to 2.0.2-beta-15516 --- korebuild-lock.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 2b2aa04d50..a09c6b1ed7 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-beta-15512 -commithash:27029a108e010e6037d46331a8a3f52da8cb6076 +version:2.0.2-beta-15516 +commithash:b7022c60cb24c59a29ca4bd9ad8f6dd29dc41509 From 0c1352dd27d7babdbc3ef89b2a240aa900bd11ea Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 3 Oct 2017 17:48:01 -0700 Subject: [PATCH 059/115] Update metapackage via artifact dependencies --- build/RuntimeStore.targets | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index b6ad613cd2..28118dbf02 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -171,27 +171,23 @@ - - - + - - - + + + ManifestDestination="$(_WorkRoot)\build\aspnetcore-store-$(VersionPrefix)-$(VersionSuffix).xml"/> - + - + \ No newline at end of file From 2b6989bc09c0234f0ee97f2c09047f37829dda80 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 4 Oct 2017 11:24:40 -0700 Subject: [PATCH 060/115] Add /t:Push --- build/push.targets | 39 +++++++++++++++++++++++++++++++++++++++ build/repo.targets | 1 + 2 files changed, 40 insertions(+) create mode 100644 build/push.targets diff --git a/build/push.targets b/build/push.targets new file mode 100644 index 0000000000..b2a563794e --- /dev/null +++ b/build/push.targets @@ -0,0 +1,39 @@ + + + + + + + + <_PackagesToPush Include="$(BuildDir)*.nupkg" /> + <_PackagesToPush Include="$(ArtifactsDir)mirror\*.nupkg" /> + + + + + + + + + + $(NpmRegistry.Replace("https:", "")):_authToken + + + + + + + + + + + + + + diff --git a/build/repo.targets b/build/repo.targets index 8fac6e8176..008abfd375 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -1,6 +1,7 @@ + <_CloneRepositoryRoot>$(RepositoryRoot).r\ From 8fbac32ed84842fa0cb83ba812d0f5f9878863df Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 4 Oct 2017 11:57:26 -0700 Subject: [PATCH 061/115] Only attempt to publish npm when npm artifacts exist --- build/push.targets | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/build/push.targets b/build/push.targets index b2a563794e..d52845fc48 100644 --- a/build/push.targets +++ b/build/push.targets @@ -1,35 +1,32 @@ - + + + + + + - - - <_PackagesToPush Include="$(BuildDir)*.nupkg" /> - <_PackagesToPush Include="$(ArtifactsDir)mirror\*.nupkg" /> - - - + + + + + $(NpmRegistry.Replace("https:", "")):_authToken - - - - From c53c133bcbd38b9d797fec0fa6dee8218d61eb62 Mon Sep 17 00:00:00 2001 From: John Luo Date: Thu, 5 Oct 2017 19:47:45 -0700 Subject: [PATCH 062/115] Compose store symbols to the correct location --- build/tasks/ComposeNewStore.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/tasks/ComposeNewStore.cs b/build/tasks/ComposeNewStore.cs index 1694f5ef25..90f601e0a2 100644 --- a/build/tasks/ComposeNewStore.cs +++ b/build/tasks/ComposeNewStore.cs @@ -90,8 +90,8 @@ namespace RepoTasks if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) { - var destinationDir = Path.Combine(StoreDestination, recursiveDir); - if (!Directory.Exists(Path.Combine(StoreDestination, recursiveDir))) + var destinationDir = Path.Combine(SymbolsDestination, recursiveDir); + if (!Directory.Exists(Path.Combine(SymbolsDestination, recursiveDir))) { Directory.CreateDirectory(destinationDir); } From 611b6a23ad3bdd722f1fc7aeaa407818c8553b22 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 6 Oct 2017 16:44:49 -0700 Subject: [PATCH 063/115] Update how PackageReference versions are set Changes: - Remove floating versions - Disable myget feeds during a Universe build - Use package-specific MSBuild variables. Pattern = `packageId.Pascalize() + "PackageVersion"`, with a few exceptions. - Remove myget feeds during build - Remove obsolete 'GenerateLineup' targets/tasks --- NuGet.config | 4 +- build/RepositoryBuild.targets | 10 +-- build/RuntimeStore.targets | 4 +- build/common.props | 4 +- build/dependencies.props | 34 ++++++-- build/repo.targets | 20 ----- build/tasks/AnalyzeBuildGraph.cs | 11 ++- build/tasks/GenerateLineup.cs | 87 ------------------- .../tasks/GeneratePackageVersionPropsFile.cs | 51 ++++++----- .../tasks/GenerateRestoreSourcesPropsFile.cs | 2 +- build/tasks/RepoTasks.tasks | 1 - build/tasks/Utilities/KoreBuildErrors.cs | 1 + korebuild-lock.txt | 4 +- .../Microsoft.AspNetCore.All.csproj | 8 +- 14 files changed, 80 insertions(+), 161 deletions(-) delete mode 100644 build/tasks/GenerateLineup.cs diff --git a/NuGet.config b/NuGet.config index c4bc056c4d..7604d0051e 100644 --- a/NuGet.config +++ b/NuGet.config @@ -2,8 +2,6 @@ - - - \ No newline at end of file + diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 2f92b51d9a..1ee1df93a5 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -29,14 +29,6 @@ - - + + $(RepositoryBuildArguments) /p:AspNetUniverseBuildOffline=true $(RepositoryBuildArguments) /p:DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath) $(RepositoryBuildArguments) /p:DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 28118dbf02..81f949a2c3 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -41,7 +41,7 @@ + Properties="Configuration=$(Configuration);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath);AspNetUniverseBuildOffline=true" /> @@ -190,4 +190,4 @@ - \ No newline at end of file + diff --git a/build/common.props b/build/common.props index f7b678dcf2..07f22d4eb3 100644 --- a/build/common.props +++ b/build/common.props @@ -14,7 +14,7 @@ - + - \ No newline at end of file + diff --git a/build/dependencies.props b/build/dependencies.props index f0452cf9db..c5d8370df2 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -22,6 +22,7 @@ + @@ -95,10 +96,12 @@ https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json + 2.0.2-beta-15522 - + + @@ -171,7 +174,23 @@ - + + KRB2004 + NETStandardImplicitPackageVersion + netstandard2.0 + + + + KRB2004 + NETStandardLibraryPackageVersion + net461 + + + + KRB2004 + NETStandardLibraryPackageVersion + net46 + @@ -181,6 +200,7 @@ KRB2004 + NewtonsoftJsonRuntimePackageVersion @@ -213,14 +233,16 @@ KRB2004 - EFBenchmarksXunitAssertPackageVersion + XunitAssertStablePackageVersion KRB2004 - + + XunitRunnerVisualStudioPackageVersion + KRB2004 @@ -231,8 +253,4 @@ KRB2004 - - - 2.0.1-rtm-15400 - diff --git a/build/repo.targets b/build/repo.targets index 008abfd375..b2f0305756 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -207,26 +207,6 @@ - - - - - - - - - - - diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 1119b21dcf..3a7a6a3ded 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -130,6 +130,15 @@ namespace RepoTasks var matchesExternalDependency = false; if (shippedPackageMap.TryGetValue(dependency.Key, out var shippedPackage)) { + if (string.IsNullOrEmpty(dependency.Value.Version)) + { + Log.LogKoreBuildError( + project.FullPath, + KoreBuildErrors.EmptyPackageReferenceVersion, + message: $"Package reference to {dependency.Key} has an empty version"); + continue; + } + matchesExternalDependency = shippedPackage.PackageInfo.Version.Equals(NuGetVersion.Parse(dependency.Value.Version)); } else if (dependencyMap.TryGetValue(dependency.Key, out var externalVersions)) @@ -152,7 +161,7 @@ namespace RepoTasks { continue; } - else if (package.PackageInfo.Version.Equals(refVersion)) + else if (package.PackageInfo.Version.Equals(refVersion.MinVersion)) { continue; } diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs deleted file mode 100644 index 5053ebdfa2..0000000000 --- a/build/tasks/GenerateLineup.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.IO; -using System.Xml; -using System.Xml.Linq; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using RepoTasks.ProjectModel; -using RepoTasks.Utilities; - -namespace RepoTasks -{ - public class GenerateLineup : Task - { - [Required] - public ITaskItem[] Artifacts { get; set; } - - [Required] - public string OutputPath { get; set; } - - // Can be set to filter the lists of packages when produce a list for a specific repository - public string Repository { get; set; } - - public bool UseFloatingVersions { get; set; } - - public string BuildNumber { get; set; } - - public override bool Execute() - { - OutputPath = OutputPath.Replace('\\', '/'); - Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); - - if (UseFloatingVersions && string.IsNullOrEmpty(BuildNumber)) - { - Log.LogWarning("Cannot compute floating versions when BuildNumber is not specified"); - } - - var items = new XElement("ItemGroup"); - var root = new XElement("Project", items); - var doc = new XDocument(root); - - var packages = new List(); - - foreach (var pkg in Artifacts.Select(ArtifactInfo.Parse) - .OfType() - .Where(p => !p.IsSymbolsArtifact - && (string.IsNullOrEmpty(Repository) - || !Repository.Equals(p.RepoName, StringComparison.OrdinalIgnoreCase)))) - { - packages.Add(pkg.PackageInfo); - } - - foreach (var pkg in packages.OrderBy(i => i.Id)) - { - var version = pkg.Version.ToString(); - if (UseFloatingVersions && version.EndsWith(BuildNumber)) - { - version = version.Substring(0, version.Length - BuildNumber.Length) + "*"; - } - - var refType = "DotNetCliTool".Equals(pkg.PackageType, StringComparison.OrdinalIgnoreCase) - ? "DotNetCliToolReference" - : "PackageReference"; - - items.Add(new XElement(refType, - new XAttribute("Update", pkg.Id), - new XAttribute("Version", version))); - } - - var settings = new XmlWriterSettings - { - OmitXmlDeclaration = true, - Indent = true, - }; - using (var writer = XmlWriter.Create(OutputPath, settings)) - { - Log.LogMessage(MessageImportance.High, $"Generate {OutputPath}"); - doc.Save(writer); - } - return true; - } - } -} diff --git a/build/tasks/GeneratePackageVersionPropsFile.cs b/build/tasks/GeneratePackageVersionPropsFile.cs index 4f7f444db8..43a12b6bde 100644 --- a/build/tasks/GeneratePackageVersionPropsFile.cs +++ b/build/tasks/GeneratePackageVersionPropsFile.cs @@ -57,28 +57,7 @@ namespace RepoTasks } else { - var sb = new StringBuilder(); - var first = true; - foreach (var ch in pkg.ItemSpec) - { - if (ch == '.') - { - first = true; - continue; - } - - if (first) - { - first = false; - sb.Append(char.ToUpperInvariant(ch)); - } - else - { - sb.Append(ch); - } - } - sb.Append("PackageVersion"); - packageVarName = sb.ToString(); + packageVarName = GetVariableName(pkg.ItemSpec); } var packageTfm = pkg.GetMetadata("TargetFramework"); @@ -109,10 +88,36 @@ namespace RepoTasks }; using (var writer = XmlWriter.Create(OutputPath, settings)) { - Log.LogMessage(MessageImportance.High, $"Generate {OutputPath}"); + Log.LogMessage(MessageImportance.Normal, $"Generate {OutputPath}"); doc.Save(writer); } return !Log.HasLoggedErrors; } + + private string GetVariableName(string packageId) + { + var sb = new StringBuilder(); + var first = true; + foreach (var ch in packageId) + { + if (ch == '.') + { + first = true; + continue; + } + + if (first) + { + first = false; + sb.Append(char.ToUpperInvariant(ch)); + } + else + { + sb.Append(ch); + } + } + sb.Append("PackageVersion"); + return sb.ToString(); + } } } diff --git a/build/tasks/GenerateRestoreSourcesPropsFile.cs b/build/tasks/GenerateRestoreSourcesPropsFile.cs index 75343a16bc..253420dfae 100644 --- a/build/tasks/GenerateRestoreSourcesPropsFile.cs +++ b/build/tasks/GenerateRestoreSourcesPropsFile.cs @@ -43,7 +43,7 @@ namespace RepoTasks }; using (var writer = XmlWriter.Create(OutputPath, settings)) { - Log.LogMessage(MessageImportance.High, $"Generate {OutputPath}"); + Log.LogMessage(MessageImportance.Normal, $"Generate {OutputPath}"); doc.Save(writer); } return !Log.HasLoggedErrors; diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 5dda060bf5..58b2008d5d 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -6,7 +6,6 @@ - diff --git a/build/tasks/Utilities/KoreBuildErrors.cs b/build/tasks/Utilities/KoreBuildErrors.cs index b3693e5011..f21bde3436 100644 --- a/build/tasks/Utilities/KoreBuildErrors.cs +++ b/build/tasks/Utilities/KoreBuildErrors.cs @@ -23,6 +23,7 @@ namespace RepoTasks.Utilities public const int DotNetCliReferenceReferenceHasVersion = 4003; public const int PackageVersionNotFoundInLineup = 4004; public const int UndefinedExternalDependency = 4005; + public const int EmptyPackageReferenceVersion = 4006; // Other unknown errors public const int PolicyFailedToApply = 5000; diff --git a/korebuild-lock.txt b/korebuild-lock.txt index a09c6b1ed7..2a51b344b6 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-beta-15516 -commithash:b7022c60cb24c59a29ca4bd9ad8f6dd29dc41509 +version:2.0.2-beta-15522 +commithash:f26cb086ebd8b5a187381bb9a431794f11d69025 diff --git a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj index 95e31dbeb0..67dda6b16a 100644 --- a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj +++ b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj @@ -5,11 +5,12 @@ $(DotNetRestoreSources) + + $(RestoreSources); + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; + $(RestoreSources); - https://dotnet.myget.org/F/aspnet-2-0-2-october2017-patch/api/v3/index.json; - https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json; - https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; https://api.nuget.org/v3/index.json; @@ -18,6 +19,7 @@ aspnetcore Microsoft.AspNetCore.All false + $(CoreSetupPackageVersion) From 869d7626418eae94ef19e249323e99ad5e1bfad9 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 9 Oct 2017 10:31:42 -0700 Subject: [PATCH 064/115] Copy symbol packages to artifacts/symbols --- build/repo.targets | 2 +- build/tasks/CopyPackagesToSplitFolders.cs | 11 ++++++++++- build/tasks/Utilities/PackageCategory.cs | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/build/repo.targets b/build/repo.targets index b2f0305756..1aa7ad2e18 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -209,7 +209,7 @@ - + <_MirroredPackageFiles Include="$(IntermediateMirrorPackageDir)*.nupkg" /> diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs index ebb1b37bf5..b9ead3f5b6 100644 --- a/build/tasks/CopyPackagesToSplitFolders.cs +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -50,7 +50,13 @@ namespace RepoTasks identity = reader.GetIdentity(); } - if (!expectedPackages.TryGetCategory(identity.Id, out var category)) + var isSymbolsPackage = file.ItemSpec.EndsWith(".symbols.nupkg", StringComparison.OrdinalIgnoreCase); + PackageCategory category; + if (isSymbolsPackage) + { + category = PackageCategory.Symbols; + } + else if (!expectedPackages.TryGetCategory(identity.Id, out category)) { Log.LogError($"Unexpected package artifact with id: {identity.Id}"); continue; @@ -73,6 +79,9 @@ namespace RepoTasks case PackageCategory.Mirror: destDir = Path.Combine(DestinationFolder, "mirror"); break; + case PackageCategory.Symbols: + destDir = Path.Combine(DestinationFolder, "symbols"); + break; default: throw new NotImplementedException(); } diff --git a/build/tasks/Utilities/PackageCategory.cs b/build/tasks/Utilities/PackageCategory.cs index f51142bdb7..78afe9cbce 100644 --- a/build/tasks/Utilities/PackageCategory.cs +++ b/build/tasks/Utilities/PackageCategory.cs @@ -11,5 +11,6 @@ namespace RepoTasks.Utilities NoShip, ShipOob, Mirror, + Symbols, } } From c76125af0ec297d7011e450d94bf56ddbeeb88a6 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 9 Oct 2017 12:54:22 -0700 Subject: [PATCH 065/115] Add previously-built aspnetcore artifacts as external dependencies --- build/dependencies.props | 73 +++++++++++++++++++++++++++++++- build/repo.targets | 2 - build/tasks/AnalyzeBuildGraph.cs | 43 ++----------------- 3 files changed, 75 insertions(+), 43 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index c5d8370df2..93fa713183 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -132,7 +132,7 @@ - + @@ -253,4 +253,75 @@ KRB2004 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json + + + + + + + + + + + + + + + + + + + diff --git a/build/repo.targets b/build/repo.targets index 1aa7ad2e18..07d6aad469 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -48,7 +48,6 @@ <_LineupPackages Include="@(ExternalDependency)" /> <_LineupPackages Include="%(ArtifactInfo.PackageId)" Version="%(ArtifactInfo.Version)" Condition=" '%(ArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> - <_LineupPackages Include="%(ShippedArtifactInfo.PackageId)" Version="%(ShippedArtifactInfo.Version)" Condition=" '%(ShippedArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> <_LineupSources Include="$(_DependencyPackagesDirectory)" Condition="'$(_DependencyPackagesDirectory)' != '' AND Exists('$(_DependencyPackagesDirectory)')" /> <_LineupSources Include="$(BuildDir)" /> @@ -200,7 +199,6 @@ Solutions="@(Solution)" Artifacts="@(ArtifactInfo)" Dependencies="@(ExternalDependency)" - ShippedArtifacts="@(ShippedArtifactInfo)" StartGraphAt="$(BuildGraphOf)" Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber);DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath)"> diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 3a7a6a3ded..d72fd62fc4 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -34,10 +34,6 @@ namespace RepoTasks [Required] public ITaskItem[] Dependencies { get; set; } - // Artifacts that already shipped from repos - [Required] - public ITaskItem[] ShippedArtifacts { get; set; } - [Required] public string Properties { get; set; } @@ -90,12 +86,6 @@ namespace RepoTasks private void EnsureConsistentGraph(IEnumerable packages, IEnumerable solutions) { - var shippedPackageMap = ShippedArtifacts - .Select(ArtifactInfo.Parse) - .OfType() - .Where(p => !p.IsSymbolsArtifact) - .ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); - // ensure versions cascade var buildPackageMap = packages.ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); var dependencyMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); @@ -126,27 +116,8 @@ namespace RepoTasks if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) { // This dependency is not one of the packages that will be compiled by this run of Universe. - - var matchesExternalDependency = false; - if (shippedPackageMap.TryGetValue(dependency.Key, out var shippedPackage)) - { - if (string.IsNullOrEmpty(dependency.Value.Version)) - { - Log.LogKoreBuildError( - project.FullPath, - KoreBuildErrors.EmptyPackageReferenceVersion, - message: $"Package reference to {dependency.Key} has an empty version"); - continue; - } - - matchesExternalDependency = shippedPackage.PackageInfo.Version.Equals(NuGetVersion.Parse(dependency.Value.Version)); - } - else if (dependencyMap.TryGetValue(dependency.Key, out var externalVersions)) - { - matchesExternalDependency = externalVersions.Contains(dependency.Value.Version); - } - - if (!matchesExternalDependency) + if (!dependencyMap.TryGetValue(dependency.Key, out var externalVersions) + || !externalVersions.Contains(dependency.Value.Version)) { Log.LogKoreBuildError( project.FullPath, @@ -168,15 +139,7 @@ namespace RepoTasks if (!solution.ShouldBuild) { - if (!shippedPackageMap.TryGetValue(project.PackageId, out _)) - { - Log.LogMessage(MessageImportance.Normal, $"Detected inconsistent in a sample or test project {project.FullPath}"); - continue; - } - else - { - reposThatShouldPatch.Add(Path.GetFileName(Path.GetDirectoryName(solution.FullPath))); - } + reposThatShouldPatch.Add(Path.GetFileName(Path.GetDirectoryName(solution.FullPath))); } inconsistentVersions.Add(new VersionMismatch From bda1eb2ab9435a91e48c6ef3f203651988d6b0b1 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 9 Oct 2017 13:04:15 -0700 Subject: [PATCH 066/115] Update restore source for aspnet testing utility packages --- build/dependencies.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index 93fa713183..2c63adc8a5 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -265,7 +265,6 @@ not building again in this patch. - @@ -294,7 +293,6 @@ not building again in this patch. - @@ -308,11 +306,13 @@ not building again in this patch. + + From be7c5c775239cb9ed1ffad2933d33d18cf884b4d Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 9 Oct 2017 13:51:29 -0700 Subject: [PATCH 067/115] Mirror everything that isn't on nuget.org --- build/dependencies.props | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index 2c63adc8a5..c439ea7377 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -72,25 +72,25 @@ - + KRB2004 MicrosoftCodeAnalysisCommonToolingPackageVersion - + KRB2004 MicrosoftCodeAnalysisCSharpToolingPackageVersion - - - + + + KRB2004 MicrosoftCodeAnalysisWorkspacesCommonToolingPackageVersion - - + + @@ -119,7 +119,7 @@ - + From 5e4e386a84ed27e78e1d6ac044b7fe1f7c8b8fb9 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 9 Oct 2017 14:30:20 -0700 Subject: [PATCH 068/115] List more external dependencies --- build/dependencies.props | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index c439ea7377..826de7c7ed 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -72,6 +72,11 @@ + + + KRB2004 + MicrosoftCodeAnalysisToolingPackageVersion + KRB2004 @@ -82,13 +87,29 @@ KRB2004 MicrosoftCodeAnalysisCSharpToolingPackageVersion - - + + + KRB2004 + MicrosoftCodeAnalysisCSharpWorkspacesToolingPackageVersion + + + + KRB2004 + MicrosoftCodeAnalysisVisualBasicToolingPackageVersion + + + + KRB2004 + MicrosoftCodeAnalysisVisualBasicWorkspacesToolingPackageVersion + KRB2004 MicrosoftCodeAnalysisWorkspacesCommonToolingPackageVersion + + + From d7d0e31b36025b9c7947374dd41d7c8b8ba5aa66 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 9 Oct 2017 15:02:02 -0700 Subject: [PATCH 069/115] Fix build warning about Microsoft.CodeAnalysis.CSharp.Workspaces --- .gitignore | 2 +- build/dependencies.props | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a68780e24f..ed69f6667e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,4 @@ node_modules .w .deps global.json -korebuild-lock.txt +msbuild.ProjectImports.zip diff --git a/build/dependencies.props b/build/dependencies.props index 826de7c7ed..9971961ad3 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -165,7 +165,10 @@ KRB2004 - + + + KRB2004 + KRB2004 From 0bf81fd82adce4a8c133e3defdbf3ccbd00f4e58 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 10 Oct 2017 09:59:42 -0700 Subject: [PATCH 070/115] Invoke dotnet-store as an MSBuild target, and update its restore sources --- build/RuntimeStore.targets | 44 +++++++++++++++---- .../templates/RS.Manifest/RS.Manifest.csproj | 16 ++++++- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 81f949a2c3..528485da7c 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -1,6 +1,12 @@ + + 1.0.0 + $(VersionPrefix)-$(VersionSuffix) + $(VersionPrefix) + + <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ <_DependencyMirrorDirectory>$(RepositoryRoot).deps\mirror\ @@ -84,16 +90,38 @@ + + <_RuntimeStoreRestoreSources Include="$(_DependencyBuildDirectory)" Condition="Exists($(_DependencyBuildDirectory))" /> + <_RuntimeStoreRestoreSources Include="$(BuildDir)" /> + + + + + + <__ComposeStoreProps /> + <_ComposeStoreProps Condition="Exists($(_DependencyBuildDirectory))">$(_ComposeStoreProps);AspNetUniverseBuildOffline=true + <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeIdentifier=$(RuntimeStoreRID) + <_ComposeStoreProps>$(_ComposeStoreProps);ComposeDir=$(_RuntimeStoreOutputDirectory) + <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeFrameworkVersion=$(SharedFrameworkVersion) + <_ComposeStoreProps>$(_ComposeStoreProps);ComposeWorkingDir=$(_RuntimeStoreWorkDirectory) + <_ComposeStoreProps>$(_ComposeStoreProps);MicrosoftAspNetCoreAllPackageVersion=$(Version) + <_ComposeStoreProps>$(_ComposeStoreProps);DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath) + + - + - + - + @@ -118,8 +146,8 @@ - $(ArtifactsDir)aspnetcore-store-$(VersionPrefix)-$(VersionSuffix)-$(RuntimeStoreRID).zip - $(ArtifactsDir)aspnetcore-symbols-$(VersionPrefix)-$(VersionSuffix)-$(RuntimeStoreRID).zip + $(ArtifactsDir)aspnetcore-store-$(Version)-$(RuntimeStoreRID).zip + $(ArtifactsDir)aspnetcore-symbols-$(Version)-$(RuntimeStoreRID).zip @@ -136,7 +164,7 @@ NewManifests="@(RuntimeStoreManifestFiles)" RuntimeStoreFiles="@(RuntimeStoreFiles)" RuntimeStoreSymbolFiles="@(RuntimeStoreSymbolFiles)" - ManifestDestination="$(ArtifactsDir)aspnetcore-store-$(VersionPrefix)-$(VersionSuffix)-$(RuntimeStoreRID).xml" + ManifestDestination="$(ArtifactsDir)aspnetcore-store-$(Version)-$(RuntimeStoreRID).xml" StoreDestination="$(_StoreZipDirectory)store\" SymbolsDestination="$(_SymbolsZipDirectory)"/> @@ -155,7 +183,7 @@ - aspnetcore-store-$(VersionPrefix)-$(VersionSuffix)-common.xml + aspnetcore-store-$(Version)-common.xml @@ -182,7 +210,7 @@ + ManifestDestination="$(_WorkRoot)\build\aspnetcore-store-$(Version).xml"/> diff --git a/build/tools/templates/RS.Manifest/RS.Manifest.csproj b/build/tools/templates/RS.Manifest/RS.Manifest.csproj index 0e9d1c5186..514fd38948 100644 --- a/build/tools/templates/RS.Manifest/RS.Manifest.csproj +++ b/build/tools/templates/RS.Manifest/RS.Manifest.csproj @@ -1,13 +1,25 @@ + netcoreapp2.0 + $(DotNetRestoreSources) + + $(RestoreSources); + https://dotnet.myget.org/F/aspnet-2-0-2-october2017-patch/api/v3/index.json; + https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json; + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; + + + $(RestoreSources); + https://api.nuget.org/v3/index.json; + - + @@ -19,4 +31,4 @@ - \ No newline at end of file + From d0cab06f08d492f4763e367cb7ed407373e31843 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 10 Oct 2017 10:20:01 -0700 Subject: [PATCH 071/115] Only add sources that exist --- build/RuntimeStore.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 528485da7c..c6b75c6484 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -92,7 +92,7 @@ <_RuntimeStoreRestoreSources Include="$(_DependencyBuildDirectory)" Condition="Exists($(_DependencyBuildDirectory))" /> - <_RuntimeStoreRestoreSources Include="$(BuildDir)" /> + <_RuntimeStoreRestoreSources Include="$(BuildDir)" Condition="Exists($(BuildDir))" /> Date: Tue, 10 Oct 2017 10:29:01 -0700 Subject: [PATCH 072/115] Add .deps/mirror and stop generating a nuget.config file for the RS build --- build/RuntimeStore.targets | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index c6b75c6484..8e319f3e76 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -63,20 +63,7 @@ - - - - - @@ -92,6 +79,7 @@ <_RuntimeStoreRestoreSources Include="$(_DependencyBuildDirectory)" Condition="Exists($(_DependencyBuildDirectory))" /> + <_RuntimeStoreRestoreSources Include="$(_DependencyMirrorDirectory)" Condition="Exists($(_DependencyMirrorDirectory))" /> <_RuntimeStoreRestoreSources Include="$(BuildDir)" Condition="Exists($(BuildDir))" /> From 32a65fdb622509bff7f07ca15d6a2c27436a1f67 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 10 Oct 2017 10:56:28 -0700 Subject: [PATCH 073/115] Invoke /t:CollectDeps on HostingStartup.csproj via MSBuild task, not Exec --- build/RuntimeStore.targets | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 8e319f3e76..316d9303de 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -64,6 +64,21 @@ + + + <_RuntimeStoreRestoreSources Include="$(_DependencyBuildDirectory)" Condition="Exists($(_DependencyBuildDirectory))" /> + <_RuntimeStoreRestoreSources Include="$(_DependencyMirrorDirectory)" Condition="Exists($(_DependencyMirrorDirectory))" /> + <_RuntimeStoreRestoreSources Include="$(BuildDir)" Condition="Exists($(BuildDir))" /> + + + + + + <_RsManifestProps>MicrosoftAspNetCoreAllPackageVersion=$(Version) + <_RsManifestProps>$(_RsManifestProps);DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath) + @@ -77,25 +92,13 @@ - - <_RuntimeStoreRestoreSources Include="$(_DependencyBuildDirectory)" Condition="Exists($(_DependencyBuildDirectory))" /> - <_RuntimeStoreRestoreSources Include="$(_DependencyMirrorDirectory)" Condition="Exists($(_DependencyMirrorDirectory))" /> - <_RuntimeStoreRestoreSources Include="$(BuildDir)" Condition="Exists($(BuildDir))" /> - - - - <__ComposeStoreProps /> - <_ComposeStoreProps Condition="Exists($(_DependencyBuildDirectory))">$(_ComposeStoreProps);AspNetUniverseBuildOffline=true - <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeIdentifier=$(RuntimeStoreRID) + <_ComposeStoreProps>$(_ComposeStoreProps);$(_RsManifestProps) <_ComposeStoreProps>$(_ComposeStoreProps);ComposeDir=$(_RuntimeStoreOutputDirectory) - <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeFrameworkVersion=$(SharedFrameworkVersion) <_ComposeStoreProps>$(_ComposeStoreProps);ComposeWorkingDir=$(_RuntimeStoreWorkDirectory) - <_ComposeStoreProps>$(_ComposeStoreProps);MicrosoftAspNetCoreAllPackageVersion=$(Version) - <_ComposeStoreProps>$(_ComposeStoreProps);DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath) + <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeIdentifier=$(RuntimeStoreRID) + <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeFrameworkVersion=$(SharedFrameworkVersion) @@ -123,7 +126,9 @@ - + @@ -179,7 +184,9 @@ - + From 509210bcac00ac07ab8dfa8ed111a07badc671ab Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 11 Oct 2017 10:14:18 -0700 Subject: [PATCH 074/115] Fix syntax errors in bootstrapper script --- build.sh | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/build.sh b/build.sh index 9e32358cf3..14d84a8773 100755 --- a/build.sh +++ b/build.sh @@ -11,7 +11,7 @@ RED="\033[0;31m" YELLOW="\033[0;33m" MAGENTA="\033[0;95m" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -[ -z "${DOTNET_HOME:-}"] && DOTNET_HOME="$HOME/.dotnet" +[ -z "${DOTNET_HOME:-}" ] && DOTNET_HOME="$HOME/.dotnet" config_file="$DIR/korebuild.json" verbose=false update=false @@ -23,7 +23,7 @@ tools_source='' # Functions # __usage() { - echo "Usage: $(basename ${BASH_SOURCE[0]}) [options] [[--] ...]" + echo "Usage: $(basename "${BASH_SOURCE[0]}") [options] [[--] ...]" echo "" echo "Arguments:" echo " ... Arguments passed to MSBuild. Variable number of arguments allowed." @@ -47,16 +47,17 @@ __usage() { } get_korebuild() { + local version local lock_file="$repo_path/korebuild-lock.txt" - if [ ! -f $lock_file ] || [ "$update" = true ]; then - __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" $lock_file + if [ ! -f "$lock_file" ] || [ "$update" = true ]; then + __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" "$lock_file" fi - local version="$(grep 'version:*' -m 1 $lock_file)" + version="$(grep 'version:*' -m 1 "$lock_file")" if [[ "$version" == '' ]]; then __error "Failed to parse version from $lock_file. Expected a line that begins with 'version:'" return 1 fi - version="$(echo ${version#version:} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + version="$(echo "${version#version:}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version" { @@ -65,10 +66,10 @@ get_korebuild() { local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip" tmpfile="$(mktemp)" echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}" - if __get_remote_file $remote_path $tmpfile; then - unzip -q -d "$korebuild_path" $tmpfile + if __get_remote_file "$remote_path" "$tmpfile"; then + unzip -q -d "$korebuild_path" "$tmpfile" fi - rm $tmpfile || true + rm "$tmpfile" || true fi source "$korebuild_path/KoreBuild.sh" @@ -99,19 +100,22 @@ __get_remote_file() { local local_path=$2 if [[ "$remote_path" != 'http'* ]]; then - cp $remote_path $local_path + cp "$remote_path" "$local_path" return 0 fi - failed=false - if __machine_has curl ; then - curl --retry 10 -sSL -f --create-dirs -o $local_path $remote_path || failed=true - elif __machine_has wget; then - wget --tries 10 -O $local_path $remote_path || failed=true + local failed=false + if __machine_has wget; then + wget --tries 10 --quiet -O "$local_path" "$remote_path" || failed=true else failed=true fi + if [ "$failed" = true ] && __machine_has curl; then + failed=false + curl --retry 10 -sSL -f --create-dirs -o "$local_path" "$remote_path" || failed=true + fi + if [ "$failed" = true ]; then __error "Download failed: $remote_path" 1>&2 return 1 @@ -122,7 +126,7 @@ __get_remote_file() { # main # -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do case $1 in -\?|-h|--help) __usage --no-exit @@ -130,7 +134,7 @@ while [[ $# > 0 ]]; do ;; -c|--channel|-Channel) shift - channel=${1:-} + channel="${1:-}" [ -z "$channel" ] && __usage ;; --config-file|-ConfigFile) @@ -144,7 +148,7 @@ while [[ $# > 0 ]]; do ;; -d|--dotnet-home|-DotNetHome) shift - DOTNET_HOME=${1:-} + DOTNET_HOME="${1:-}" [ -z "$DOTNET_HOME" ] && __usage ;; --path|-Path) @@ -213,4 +217,4 @@ fi get_korebuild install_tools "$tools_source" "$DOTNET_HOME" -invoke_repository_build "$repo_path" $@ +invoke_repository_build "$repo_path" "$@" From c8068b7d1cf9ba3bfb456d5b8f4cd100e415b1f1 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 11 Oct 2017 09:56:59 -0700 Subject: [PATCH 075/115] Use submodules for source dependencies As a part of making builds more reproducible, we want to start committing the exact sha of source dependencies used to build aspnetcore. This implements that using git submodules, and removes targets that were previously used to clone source on demand. --- .gitignore | 1 - .gitmodules | 168 +++++++++++++++++++++++++++++++ README.md | 51 ++-------- build/Repositories.props | 53 ---------- build/RepositoryBuild.targets | 7 +- build/repo.props | 1 + build/repo.targets | 147 ++++----------------------- build/submodules.props | 53 ++++++++++ build/tasks/AnalyzeBuildGraph.cs | 2 +- modules/Antiforgery | 1 + modules/AzureIntegration | 1 + modules/BasicMiddleware | 1 + modules/BrowserLink | 1 + modules/CORS | 1 + modules/Caching | 1 + modules/Common | 1 + modules/Configuration | 1 + modules/DataProtection | 1 + modules/DependencyInjection | 1 + modules/Diagnostics | 1 + modules/DotNetTools | 1 + modules/EntityFrameworkCore | 1 + modules/EventNotification | 1 + modules/FileSystem | 1 + modules/Hosting | 1 + modules/HtmlAbstractions | 1 + modules/HttpAbstractions | 1 + modules/HttpSysServer | 1 + modules/IISIntegration | 1 + modules/Identity | 1 + modules/JavaScriptServices | 1 + modules/JsonPatch | 1 + modules/KestrelHttpServer | 1 + modules/Localization | 1 + modules/Logging | 1 + modules/MetaPackages | 1 + modules/Microsoft.Data.Sqlite | 1 + modules/Mvc | 1 + modules/MvcPrecompilation | 1 + modules/Options | 1 + modules/Proxy | 1 + modules/Razor | 1 + modules/ResponseCaching | 1 + modules/Routing | 1 + modules/Scaffolding | 1 + modules/Security | 1 + modules/ServerTests | 1 + modules/Session | 1 + modules/StaticFiles | 1 + modules/Testing | 1 + modules/WebSockets | 1 + 51 files changed, 294 insertions(+), 231 deletions(-) create mode 100644 .gitmodules delete mode 100644 build/Repositories.props create mode 100644 build/submodules.props create mode 160000 modules/Antiforgery create mode 160000 modules/AzureIntegration create mode 160000 modules/BasicMiddleware create mode 160000 modules/BrowserLink create mode 160000 modules/CORS create mode 160000 modules/Caching create mode 160000 modules/Common create mode 160000 modules/Configuration create mode 160000 modules/DataProtection create mode 160000 modules/DependencyInjection create mode 160000 modules/Diagnostics create mode 160000 modules/DotNetTools create mode 160000 modules/EntityFrameworkCore create mode 160000 modules/EventNotification create mode 160000 modules/FileSystem create mode 160000 modules/Hosting create mode 160000 modules/HtmlAbstractions create mode 160000 modules/HttpAbstractions create mode 160000 modules/HttpSysServer create mode 160000 modules/IISIntegration create mode 160000 modules/Identity create mode 160000 modules/JavaScriptServices create mode 160000 modules/JsonPatch create mode 160000 modules/KestrelHttpServer create mode 160000 modules/Localization create mode 160000 modules/Logging create mode 160000 modules/MetaPackages create mode 160000 modules/Microsoft.Data.Sqlite create mode 160000 modules/Mvc create mode 160000 modules/MvcPrecompilation create mode 160000 modules/Options create mode 160000 modules/Proxy create mode 160000 modules/Razor create mode 160000 modules/ResponseCaching create mode 160000 modules/Routing create mode 160000 modules/Scaffolding create mode 160000 modules/Security create mode 160000 modules/ServerTests create mode 160000 modules/Session create mode 160000 modules/StaticFiles create mode 160000 modules/Testing create mode 160000 modules/WebSockets diff --git a/.gitignore b/.gitignore index ed69f6667e..aa0331923d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ node_modules *.snk .build .nuget -.r .w .deps global.json diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..76a2fb3298 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,168 @@ +[submodule "modules/Antiforgery"] + path = modules/Antiforgery + url = https://github.com/aspnet/Antiforgery.git + branch = rel/2.0.1 +[submodule "modules/AzureIntegration"] + path = modules/AzureIntegration + url = https://github.com/aspnet/AzureIntegration.git + branch = rel/2.0.1 +[submodule "modules/BasicMiddleware"] + path = modules/BasicMiddleware + url = https://github.com/aspnet/BasicMiddleware.git + branch = rel/2.0.1 +[submodule "modules/BrowserLink"] + path = modules/BrowserLink + url = https://github.com/aspnet/BrowserLink.git + branch = rel/2.0.1 +[submodule "modules/CORS"] + path = modules/CORS + url = https://github.com/aspnet/CORS.git + branch = rel/2.0.1 +[submodule "modules/DataProtection"] + path = modules/DataProtection + url = https://github.com/aspnet/DataProtection.git + branch = rel/2.0.1 +[submodule "modules/Diagnostics"] + path = modules/Diagnostics + url = https://github.com/aspnet/Diagnostics.git + branch = rel/2.0.1 +[submodule "modules/EntityFrameworkCore"] + path = modules/EntityFrameworkCore + url = https://github.com/aspnet/EntityFrameworkCore.git + branch = rel/2.0.1 +[submodule "modules/Hosting"] + path = modules/Hosting + url = https://github.com/aspnet/Hosting.git + branch = rel/2.0.1 +[submodule "modules/HttpAbstractions"] + path = modules/HttpAbstractions + url = https://github.com/aspnet/HttpAbstractions.git + branch = rel/2.0.1 +[submodule "modules/HttpSysServer"] + path = modules/HttpSysServer + url = https://github.com/aspnet/HttpSysServer.git + branch = rel/2.0.1 +[submodule "modules/Identity"] + path = modules/Identity + url = https://github.com/aspnet/Identity.git + branch = rel/2.0.1 +[submodule "modules/IISIntegration"] + path = modules/IISIntegration + url = https://github.com/aspnet/IISIntegration.git + branch = rel/2.0.1 +[submodule "modules/JavaScriptServices"] + path = modules/JavaScriptServices + url = https://github.com/aspnet/JavaScriptServices.git + branch = rel/2.0.1 +[submodule "modules/KestrelHttpServer"] + path = modules/KestrelHttpServer + url = https://github.com/aspnet/KestrelHttpServer.git + branch = rel/2.0.1 +[submodule "modules/Localization"] + path = modules/Localization + url = https://github.com/aspnet/Localization.git + branch = rel/2.0.1 +[submodule "modules/MetaPackages"] + path = modules/MetaPackages + url = https://github.com/aspnet/MetaPackages.git + branch = rel/2.0.1 +[submodule "modules/Mvc"] + path = modules/Mvc + url = https://github.com/aspnet/Mvc.git + branch = rel/2.0.1 +[submodule "modules/MvcPrecompilation"] + path = modules/MvcPrecompilation + url = https://github.com/aspnet/MvcPrecompilation.git + branch = rel/2.0.1 +[submodule "modules/Proxy"] + path = modules/Proxy + url = https://github.com/aspnet/Proxy.git + branch = rel/2.0.1 +[submodule "modules/Razor"] + path = modules/Razor + url = https://github.com/aspnet/Razor.git + branch = rel/2.0.1 +[submodule "modules/ResponseCaching"] + path = modules/ResponseCaching + url = https://github.com/aspnet/ResponseCaching.git + branch = rel/2.0.1 +[submodule "modules/Routing"] + path = modules/Routing + url = https://github.com/aspnet/Routing.git + branch = rel/2.0.1 +[submodule "modules/Scaffolding"] + path = modules/Scaffolding + url = https://github.com/aspnet/Scaffolding.git + branch = rel/2.0.1 +[submodule "modules/Security"] + path = modules/Security + url = https://github.com/aspnet/Security.git + branch = rel/2.0.1 +[submodule "modules/ServerTests"] + path = modules/ServerTests + url = https://github.com/aspnet/ServerTests.git + branch = rel/2.0.1 +[submodule "modules/Session"] + path = modules/Session + url = https://github.com/aspnet/Session.git + branch = rel/2.0.1 +[submodule "modules/StaticFiles"] + path = modules/StaticFiles + url = https://github.com/aspnet/StaticFiles.git + branch = rel/2.0.1 +[submodule "modules/WebSockets"] + path = modules/WebSockets + url = https://github.com/aspnet/WebSockets.git + branch = rel/2.0.1 +[submodule "modules/Caching"] + path = modules/Caching + url = https://github.com/aspnet/Caching.git + branch = patch/2.0.1 +[submodule "modules/Common"] + path = modules/Common + url = https://github.com/aspnet/Common.git + branch = patch/2.0.1 +[submodule "modules/Configuration"] + path = modules/Configuration + url = https://github.com/aspnet/Configuration.git + branch = patch/2.0.1 +[submodule "modules/DependencyInjection"] + path = modules/DependencyInjection + url = https://github.com/aspnet/DependencyInjection.git + branch = patch/2.0.1 +[submodule "modules/DotNetTools"] + path = modules/DotNetTools + url = https://github.com/aspnet/DotNetTools.git + branch = patch/2.0.1 +[submodule "modules/EventNotification"] + path = modules/EventNotification + url = https://github.com/aspnet/EventNotification.git + branch = patch/2.0.1 +[submodule "modules/FileSystem"] + path = modules/FileSystem + url = https://github.com/aspnet/FileSystem.git + branch = patch/2.0.1 +[submodule "modules/HtmlAbstractions"] + path = modules/HtmlAbstractions + url = https://github.com/aspnet/HtmlAbstractions.git + branch = patch/2.0.1 +[submodule "modules/JsonPatch"] + path = modules/JsonPatch + url = https://github.com/aspnet/JsonPatch.git + branch = patch/2.0.1 +[submodule "modules/Logging"] + path = modules/Logging + url = https://github.com/aspnet/Logging.git + branch = patch/2.0.1 +[submodule "modules/Microsoft.Data.Sqlite"] + path = modules/Microsoft.Data.Sqlite + url = https://github.com/aspnet/Microsoft.Data.Sqlite.git + branch = patch/2.0.1 +[submodule "modules/Options"] + path = modules/Options + url = https://github.com/aspnet/Options.git + branch = patch/2.0.1 +[submodule "modules/Testing"] + path = modules/Testing + url = https://github.com/aspnet/Testing.git + branch = patch/2.0.1 diff --git a/README.md b/README.md index 1b049850fa..65b631381b 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,16 @@ Universe -========= +======== -This repo is to build the whole ASP.NET Core stack. +Build infrastructure used to produce the whole ASP.NET Core stack. ## Getting started - git clone git@github.com:aspnet/Universe.git - cd Universe - build - -The default build will clone all known repos as subfolders. The clone will be the dev branch. - -If the build is run subsequently it will `git pull` the dev branch rather than clone. Note! This will cause a -merge if you have local changes. We may tweak how this is done if it causes problems. - -After folders are up to date, `build.cmd compile` is executed in each of the enlisted subfolders. - -If there are errors the build will continue with the next repo. - -The last output is a list of which repos succeeded or failed. - -## build targets - -`build pull` will only clone or pull all repos. - -`build compile` this is the default target, described above. - -`build install` works like build compile, but will run `build.cmd install` in each subfolder. This means -any nupkg produced by the repo are copied into the local `.nuget` folder to be picked up by subsequent -repositories. The subfolders are built in dependency order. - - -## Verifying cross repo changes -You can use the Universe repo to preemptively verify and prepare follow ups for your breaking changes: -- Clone the Universe repo https://github.com/aspnet/Universe -- Add a branch attribute to the `build\Repositories.props` file to point to your branch in the repo you’re trying to verify. For instance, - `` - becomes - `` - **Note**: This branch should have been pushed to the server already. -- Run from the root of Universe - `build.cmd /p:CompileOnly=true /p:ShallowClone=true /p:BuildGraphOf=HtmlAbstractions` - - This should clone and compile all the repos against your breaking changes branch. If you’d like to additionally run tests in all your dependencies (this will take a while and also you could hit some flaky tests), remove the first parameter: - `build.cmd /p:ShallowClone=true /p:BuildGraphOf=HtmlAbstractions` - - The ShallowClone property speeds up git clone and is optional in both cases. +``` +git clone --recursive https://github.com/aspnet/Universe.git +cd Universe +./build.cmd +``` +## More info This project is part of ASP.NET Core. You can find samples, documentation and getting started instructions for ASP.NET Core at the [Home](https://github.com/aspnet/home) repo. diff --git a/build/Repositories.props b/build/Repositories.props deleted file mode 100644 index 0c9265e5d8..0000000000 --- a/build/Repositories.props +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 1ee1df93a5..9c5cd93425 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -6,8 +6,7 @@ %(RepositoryBuildOrder.Identity) RepositoryToBuild=%(RepositoryBuildOrder.Identity); - BuildRepositoryRoot=%(RepositoryBuildOrder.RepositoryPath)\; - CommitHash=%(RepositoryBuildOrder.Commit) + BuildRepositoryRoot=$([MSBuild]::NormalizeDirectory(%(RepositoryBuildOrder.RootPath))) @@ -24,7 +23,7 @@ @@ -44,7 +43,7 @@ $(RepositoryBuildArguments) /p:DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath) $(RepositoryBuildArguments) /p:DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath) - $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash) + $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) $(RepositoryBuildArguments) /noconsolelogger '/l:RepoTasks.FlowLogger,$(MSBuildThisFileDirectory)tasks\bin\publish\RepoTasks.dll;Summary;FlowId=$(RepositoryToBuild)' $(_RepositoryBuildTargets) $(RepositoryBuildArguments) diff --git a/build/repo.props b/build/repo.props index a98645c000..180925e696 100644 --- a/build/repo.props +++ b/build/repo.props @@ -5,4 +5,5 @@ + diff --git a/build/repo.targets b/build/repo.targets index 07d6aad469..16f7b77936 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -4,17 +4,10 @@ - <_CloneRepositoryRoot>$(RepositoryRoot).r\ - false + $(RepositoryRoot)modules\ <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ <_DependencyPackagesDirectory>$(_DependencyBuildDirectory) - <_RepositoryListFileName>Repositories.props - <_DefaultRepositoryList>$(MSBuildThisFileDirectory)$(_RepositoryListFileName) - <_DependencyRepositoryList>$(_DependencyBuildDirectory)$(_RepositoryListFileName) - <_RepositoryListToImport Condition="Exists('$(_DependencyRepositoryList)')">$(_DependencyRepositoryList) - <_RepositoryListToImport Condition="!Exists('$(_DependencyRepositoryList)')">$(_DefaultRepositoryList) - <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify @@ -23,16 +16,18 @@ $(IntermediateDir)ext\ $(IntermediateDir)dependencies.props $(IntermediateDir)sources.props - Patch20_ + $(PrepareDependsOn);PrepareOutputPath $(CleanDependsOn);CleanArtifacts;CleanUniverseArtifacts $(RestoreDependsOn);RestoreExternalDependencies - $(CompileDependsOn);CloneRepositories;BuildRepositories + $(CompileDependsOn);BuildRepositories $(PackageDependsOn);BuildAllMetapackage;SplitPackages $(VerifyDependsOn);VerifyCoherentVersions - + + + - @@ -88,82 +82,29 @@ Condition="'$(KOREBUILD_REPOSITORY_INCLUDE)' != '' AND '$(KOREBUILD_REPOSITORY_EXCLUDE)' != ''" /> - + + - - - <_CloneRepositories Include="@(Repository)" /> - <_CloneRepositories Remove="%(Repository.Identity)" Condition="$([System.Environment]::GetEnvironmentVariable('BUILD_VCS_NUMBER_$(TcVcsIdPrefix)%(Repository.Identity)')) != ''" /> - <_CloneRepositories Include="@(ShippedRepository)" /> - <_CloneRepository Include="$(MSBuildProjectFullPath)"> - - CloneRepository=%(_CloneRepositories.Identity); - CloneUrl=%(_CloneRepositories.CloneUrl); - CloneBranch=%(_CloneRepositories.Branch); - CloneRepositoryCommit=%(_CloneRepositories.Commit); - UseGateBranch=$(UseGateBranch) - - - - - - - - - - - - - - - false - <_CloneArguments>git clone --quiet $(CloneUrl) $(CloneRepository) - <_CloneArguments Condition="'$(ShallowClone)'=='true'">$(_CloneArguments) --depth 1 - - - - - - - - - - - - - - - + DependsOnTargets="_PrepareRepositories;GeneratePropsFiles;ComputeGraph;_BuildRepositories" /> + + + + @@ -174,13 +115,13 @@ --> @@ -191,7 +132,7 @@ - + @@ -230,56 +171,4 @@ ExternalDependencies="@(ExternalDependency);@(ShippedExternalDependency)" /> - - - $(BuildDir)$(_RepositoryListFileName) - - - - - - - - - - - - - <_RepositoryName>$([System.String]::new('%(Repository.Identity)').Replace('.', '_')) - <_CloneUrl>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_URL_$(TcVcsIdPrefix)$(_RepositoryName)")) - <_CommitHash>$([System.Environment]::GetEnvironmentVariable("BUILD_VCS_NUMBER_$(TcVcsIdPrefix)$(_RepositoryName)")) - - - - - - - - - - - - - - - - <_CommitHash> - <_CloneUrl> - - diff --git a/build/submodules.props b/build/submodules.props new file mode 100644 index 0000000000..fb0ebcca81 --- /dev/null +++ b/build/submodules.props @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index d72fd62fc4..94710b1aa3 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -225,7 +225,7 @@ namespace RepoTasks var order = TopologicalSort.GetOrder(graphNodeRepository); var repositoryTaskItem = new TaskItem(repository.Name); repositoryTaskItem.SetMetadata("Order", order.ToString()); - repositoryTaskItem.SetMetadata("RepositoryPath", repository.RootDir); + repositoryTaskItem.SetMetadata("RootPath", repository.RootDir); repositoriesWithOrder.Add((repositoryTaskItem, order)); } diff --git a/modules/Antiforgery b/modules/Antiforgery new file mode 160000 index 0000000000..a45c919344 --- /dev/null +++ b/modules/Antiforgery @@ -0,0 +1 @@ +Subproject commit a45c919344bd40ee719e41b205d7418728a3c7fa diff --git a/modules/AzureIntegration b/modules/AzureIntegration new file mode 160000 index 0000000000..7f86cd58fd --- /dev/null +++ b/modules/AzureIntegration @@ -0,0 +1 @@ +Subproject commit 7f86cd58fd91ff8bb8c4286eda5444188d002002 diff --git a/modules/BasicMiddleware b/modules/BasicMiddleware new file mode 160000 index 0000000000..47e4b45852 --- /dev/null +++ b/modules/BasicMiddleware @@ -0,0 +1 @@ +Subproject commit 47e4b458528c2d05c8b0004de1718b5134687211 diff --git a/modules/BrowserLink b/modules/BrowserLink new file mode 160000 index 0000000000..0c496c3652 --- /dev/null +++ b/modules/BrowserLink @@ -0,0 +1 @@ +Subproject commit 0c496c3652faf6f8091394f31ffc03e8f3199ffa diff --git a/modules/CORS b/modules/CORS new file mode 160000 index 0000000000..a95ab767fd --- /dev/null +++ b/modules/CORS @@ -0,0 +1 @@ +Subproject commit a95ab767fda039777a81fc712f6986cde15bff8d diff --git a/modules/Caching b/modules/Caching new file mode 160000 index 0000000000..19dc192591 --- /dev/null +++ b/modules/Caching @@ -0,0 +1 @@ +Subproject commit 19dc192591c77f998c0856213ab7ae088997d043 diff --git a/modules/Common b/modules/Common new file mode 160000 index 0000000000..7309bc455b --- /dev/null +++ b/modules/Common @@ -0,0 +1 @@ +Subproject commit 7309bc455b15eb9cf5634fd281a227b744d29b68 diff --git a/modules/Configuration b/modules/Configuration new file mode 160000 index 0000000000..ebc12e335e --- /dev/null +++ b/modules/Configuration @@ -0,0 +1 @@ +Subproject commit ebc12e335e194893f9e1818ac32078f02ebc6b4a diff --git a/modules/DataProtection b/modules/DataProtection new file mode 160000 index 0000000000..f5004b5ad8 --- /dev/null +++ b/modules/DataProtection @@ -0,0 +1 @@ +Subproject commit f5004b5ad8c8d802d584d9a186cd96e362e66857 diff --git a/modules/DependencyInjection b/modules/DependencyInjection new file mode 160000 index 0000000000..b5e7f41cd9 --- /dev/null +++ b/modules/DependencyInjection @@ -0,0 +1 @@ +Subproject commit b5e7f41cd9a2ac2650f3168d09de491fa10f2336 diff --git a/modules/Diagnostics b/modules/Diagnostics new file mode 160000 index 0000000000..0372c762cf --- /dev/null +++ b/modules/Diagnostics @@ -0,0 +1 @@ +Subproject commit 0372c762cf5df2084e9b9cd2ac299a63d707afc6 diff --git a/modules/DotNetTools b/modules/DotNetTools new file mode 160000 index 0000000000..72add92b43 --- /dev/null +++ b/modules/DotNetTools @@ -0,0 +1 @@ +Subproject commit 72add92b43df8327d14f38b25013ad035d8b71a1 diff --git a/modules/EntityFrameworkCore b/modules/EntityFrameworkCore new file mode 160000 index 0000000000..196400e9ca --- /dev/null +++ b/modules/EntityFrameworkCore @@ -0,0 +1 @@ +Subproject commit 196400e9ca7ebaac92e6e1e7d638fc1c82446385 diff --git a/modules/EventNotification b/modules/EventNotification new file mode 160000 index 0000000000..bd62d77546 --- /dev/null +++ b/modules/EventNotification @@ -0,0 +1 @@ +Subproject commit bd62d775466ff5f7286ba847681451f3ecaa3b0a diff --git a/modules/FileSystem b/modules/FileSystem new file mode 160000 index 0000000000..befc6c1a86 --- /dev/null +++ b/modules/FileSystem @@ -0,0 +1 @@ +Subproject commit befc6c1a86f36bcc66ed4ab6562b9904b56a4802 diff --git a/modules/Hosting b/modules/Hosting new file mode 160000 index 0000000000..befe353db8 --- /dev/null +++ b/modules/Hosting @@ -0,0 +1 @@ +Subproject commit befe353db8a376c1a2c6b58c62f08d315d90e0b2 diff --git a/modules/HtmlAbstractions b/modules/HtmlAbstractions new file mode 160000 index 0000000000..dc04ba8ee7 --- /dev/null +++ b/modules/HtmlAbstractions @@ -0,0 +1 @@ +Subproject commit dc04ba8ee7137289afe4c64d8b1b7177ceb2e780 diff --git a/modules/HttpAbstractions b/modules/HttpAbstractions new file mode 160000 index 0000000000..4afe4480b4 --- /dev/null +++ b/modules/HttpAbstractions @@ -0,0 +1 @@ +Subproject commit 4afe4480b4909141160ded50487b575cc8fd4cd8 diff --git a/modules/HttpSysServer b/modules/HttpSysServer new file mode 160000 index 0000000000..7ea2905820 --- /dev/null +++ b/modules/HttpSysServer @@ -0,0 +1 @@ +Subproject commit 7ea2905820aac58488466ad4671d9397566c3173 diff --git a/modules/IISIntegration b/modules/IISIntegration new file mode 160000 index 0000000000..f8a04f2d4e --- /dev/null +++ b/modules/IISIntegration @@ -0,0 +1 @@ +Subproject commit f8a04f2d4e7a0d401e7fed445262d57fcc64b81e diff --git a/modules/Identity b/modules/Identity new file mode 160000 index 0000000000..8a880adf72 --- /dev/null +++ b/modules/Identity @@ -0,0 +1 @@ +Subproject commit 8a880adf7299777d4ef025fa78c23be072fb8a08 diff --git a/modules/JavaScriptServices b/modules/JavaScriptServices new file mode 160000 index 0000000000..86e94d7812 --- /dev/null +++ b/modules/JavaScriptServices @@ -0,0 +1 @@ +Subproject commit 86e94d7812d0c8790ebcb32a6ae6935b85648d9c diff --git a/modules/JsonPatch b/modules/JsonPatch new file mode 160000 index 0000000000..bfd5a994cb --- /dev/null +++ b/modules/JsonPatch @@ -0,0 +1 @@ +Subproject commit bfd5a994cb68dd090694d38b5695958665bcc876 diff --git a/modules/KestrelHttpServer b/modules/KestrelHttpServer new file mode 160000 index 0000000000..39b56770a9 --- /dev/null +++ b/modules/KestrelHttpServer @@ -0,0 +1 @@ +Subproject commit 39b56770a97cdc4943cb2d652329b3ca9afc84a5 diff --git a/modules/Localization b/modules/Localization new file mode 160000 index 0000000000..6be6b8fe1d --- /dev/null +++ b/modules/Localization @@ -0,0 +1 @@ +Subproject commit 6be6b8fe1d44cbc04e85d06ae69e4eef94091f28 diff --git a/modules/Logging b/modules/Logging new file mode 160000 index 0000000000..527a2809bf --- /dev/null +++ b/modules/Logging @@ -0,0 +1 @@ +Subproject commit 527a2809bf4739a3502fdc87626411fd04f483a8 diff --git a/modules/MetaPackages b/modules/MetaPackages new file mode 160000 index 0000000000..3d5b69063f --- /dev/null +++ b/modules/MetaPackages @@ -0,0 +1 @@ +Subproject commit 3d5b69063fa85a4c504f7d9208e260b8c33bd683 diff --git a/modules/Microsoft.Data.Sqlite b/modules/Microsoft.Data.Sqlite new file mode 160000 index 0000000000..e911180fea --- /dev/null +++ b/modules/Microsoft.Data.Sqlite @@ -0,0 +1 @@ +Subproject commit e911180feacb80daddeb374e28169fb5d85e62aa diff --git a/modules/Mvc b/modules/Mvc new file mode 160000 index 0000000000..3bfb023679 --- /dev/null +++ b/modules/Mvc @@ -0,0 +1 @@ +Subproject commit 3bfb023679e73b19b0a513002ccffd42d820eecb diff --git a/modules/MvcPrecompilation b/modules/MvcPrecompilation new file mode 160000 index 0000000000..9d07f205f2 --- /dev/null +++ b/modules/MvcPrecompilation @@ -0,0 +1 @@ +Subproject commit 9d07f205f2bda99f7b62c0d65ad70043b22e7a22 diff --git a/modules/Options b/modules/Options new file mode 160000 index 0000000000..35c1a7a57b --- /dev/null +++ b/modules/Options @@ -0,0 +1 @@ +Subproject commit 35c1a7a57b1d71ab3feb40db353b4505c4d53793 diff --git a/modules/Proxy b/modules/Proxy new file mode 160000 index 0000000000..2875d8e4a1 --- /dev/null +++ b/modules/Proxy @@ -0,0 +1 @@ +Subproject commit 2875d8e4a14f9375d409ee42cd5d9a2d49363295 diff --git a/modules/Razor b/modules/Razor new file mode 160000 index 0000000000..e45618eb1b --- /dev/null +++ b/modules/Razor @@ -0,0 +1 @@ +Subproject commit e45618eb1b0d5b37091dc38f169c24b4f546cdaf diff --git a/modules/ResponseCaching b/modules/ResponseCaching new file mode 160000 index 0000000000..bf08e66cee --- /dev/null +++ b/modules/ResponseCaching @@ -0,0 +1 @@ +Subproject commit bf08e66ceeabd7a8ef8eef9fe3d8492b1b6c23ba diff --git a/modules/Routing b/modules/Routing new file mode 160000 index 0000000000..d104dd8f5f --- /dev/null +++ b/modules/Routing @@ -0,0 +1 @@ +Subproject commit d104dd8f5fc9658214a49a3ceb2a7b1ad448ef54 diff --git a/modules/Scaffolding b/modules/Scaffolding new file mode 160000 index 0000000000..2f754921f5 --- /dev/null +++ b/modules/Scaffolding @@ -0,0 +1 @@ +Subproject commit 2f754921f5b58450fe901a179cc9b313ce083a1b diff --git a/modules/Security b/modules/Security new file mode 160000 index 0000000000..30b629edfd --- /dev/null +++ b/modules/Security @@ -0,0 +1 @@ +Subproject commit 30b629edfdc20cdc2d11df01657b94f1e0b9772c diff --git a/modules/ServerTests b/modules/ServerTests new file mode 160000 index 0000000000..369b64b1b6 --- /dev/null +++ b/modules/ServerTests @@ -0,0 +1 @@ +Subproject commit 369b64b1b6f2a816669e4f9929211b0780485ac3 diff --git a/modules/Session b/modules/Session new file mode 160000 index 0000000000..c2b35d75a3 --- /dev/null +++ b/modules/Session @@ -0,0 +1 @@ +Subproject commit c2b35d75a3c0358e7773697252552ab396b0120b diff --git a/modules/StaticFiles b/modules/StaticFiles new file mode 160000 index 0000000000..3368112e15 --- /dev/null +++ b/modules/StaticFiles @@ -0,0 +1 @@ +Subproject commit 3368112e15f76f688555968ea16bf4f766dc3ed8 diff --git a/modules/Testing b/modules/Testing new file mode 160000 index 0000000000..abd5da6ccf --- /dev/null +++ b/modules/Testing @@ -0,0 +1 @@ +Subproject commit abd5da6ccf89ff70f52df7060d7f8dbb39dc5779 diff --git a/modules/WebSockets b/modules/WebSockets new file mode 160000 index 0000000000..b956ad3143 --- /dev/null +++ b/modules/WebSockets @@ -0,0 +1 @@ +Subproject commit b956ad3143ef4d7144f4f55a9d783679fe60a044 From afba40b573172cdd2890d46defcc8c0012e8aa87 Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 10 Oct 2017 11:54:14 -0700 Subject: [PATCH 076/115] Build LZMA - Update metapackage reference insertion --- build/PackageArchive.targets | 78 +++++++++++ build/RuntimeStore.targets | 3 +- build/artifacts.props | 10 +- build/dependencies.props | 81 +++++++----- build/repo.targets | 1 + build/tasks/AddArchiveReferences.cs | 132 +++++++++++++++++++ build/tasks/AddMetapackageReferences.cs | 25 +++- build/tasks/RepoTasks.tasks | 1 + build/tasks/Utilities/VersionUtilities.cs | 41 ++++++ build/tools/templates/Archive/Archive.csproj | 20 +++ 10 files changed, 349 insertions(+), 43 deletions(-) create mode 100644 build/PackageArchive.targets create mode 100644 build/tasks/AddArchiveReferences.cs create mode 100644 build/tasks/Utilities/VersionUtilities.cs create mode 100644 build/tools/templates/Archive/Archive.csproj diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets new file mode 100644 index 0000000000..a5f9c15ab6 --- /dev/null +++ b/build/PackageArchive.targets @@ -0,0 +1,78 @@ + + + <_TemplatesDirectory>$(MSBuildThisFileDirectory)tools\templates\ + <_WorkRoot>$(RepositoryRoot).w\ + + + + + + + + + + + + + + + + + + + $(_WorkRoot)obj\$(OutputPackageName) + $(RepositoryRoot)artifacts\$(OutputPackageName).lzma + + + + + + + + + + + + + + + + $(RepositoryRoot).deps\Signed\Packages\ + $(RepositoryRoot).deps\Signed\Packages-NoTimeStamp\ + + + + + + + + + @(TimestampMetapackageVersion->Metadata('Filename')) + $(TimeStampVersion.TrimStart('Microsoft.AspNetCore.All')) + @(TimestampFreeMetapackageVersion->Metadata('Filename')) + $(TimeStampFreeVersion.TrimStart('Microsoft.AspNetCore.All')) + nuGetPackagesArchive-$(TimestampVersion) + nuGetPackagesArchive-$(TimestampFreeVersion) + + + + + + + + + diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 316d9303de..f695ae6033 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -42,7 +42,8 @@ + PackageArtifacts="@(PackageArtifact)" + ExternalDependencies="@(ExternalDependency)" /> false + false + false - + @@ -46,7 +48,7 @@ - + @@ -128,7 +130,7 @@ - + @@ -142,7 +144,7 @@ - + diff --git a/build/dependencies.props b/build/dependencies.props index 9971961ad3..2ae47f4ba3 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -12,6 +12,12 @@ false false + + false + + false + + false @@ -287,41 +293,46 @@ not building again in this patch. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/repo.targets b/build/repo.targets index 16f7b77936..67ca16bf53 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -1,6 +1,7 @@ + diff --git a/build/tasks/AddArchiveReferences.cs b/build/tasks/AddArchiveReferences.cs new file mode 100644 index 0000000000..9d12c34270 --- /dev/null +++ b/build/tasks/AddArchiveReferences.cs @@ -0,0 +1,132 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Xml; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Versioning; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class AddArchiveReferences : Task + { + [Required] + public string ReferencePackagePath { get; set; } + + [Required] + public bool RemoveTimestamp { get; set; } + + [Required] + public ITaskItem[] BuildArtifacts { get; set; } + + [Required] + public ITaskItem[] PackageArtifacts { get; set; } + + [Required] + public ITaskItem[] ExternalDependencies { get; set; } + + public override bool Execute() + { + // Parse input + var externalArchiveArtifacts = ExternalDependencies.Where(p => p.GetMetadata("LZMA") == "true"); + var externalArchiveTools = ExternalDependencies.Where(p => p.GetMetadata("LZMATools") == "true"); + var archiveArtifacts = PackageArtifacts.Where(p => p.GetMetadata("LZMA") == "true"); + var archiveTools = PackageArtifacts.Where(p => p.GetMetadata("LZMATools") == "true"); + var buildArtifacts = BuildArtifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact); + + var xmlDoc = new XmlDocument(); + xmlDoc.Load(ReferencePackagePath); + + // Project + var projectElement = xmlDoc.FirstChild; + + // Items + var itemGroupElement = xmlDoc.CreateElement("ItemGroup"); + Log.LogMessage(MessageImportance.High, $"Archive will include the following packages"); + + foreach (var package in archiveArtifacts) + { + var packageName = package.ItemSpec; + var packageVersion = buildArtifacts + .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) + .PackageInfo.Version.ToString(); + + if (string.Equals(RemoveTimestamp, "true", StringComparison.OrdinalIgnoreCase)) + { + var version = new NuGetVersion(packageVersion); + var updatedVersion = new NuGetVersion(version.Version, VersionUtilities.GetNoTimestampReleaseLabel(version.Release)); + packageVersion = updatedVersion.ToNormalizedString(); + } + + Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + + itemGroupElement.AppendChild(packageReferenceElement); + } + + foreach (var package in externalArchiveArtifacts) + { + var packageName = package.ItemSpec; + var packageVersion = package.GetMetadata("Version"); + Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + + itemGroupElement.AppendChild(packageReferenceElement); + } + + foreach (var package in archiveTools) + { + var packageName = package.ItemSpec; + var packageVersion = buildArtifacts + .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) + .PackageInfo.Version.ToString(); + + if (string.Equals(RemoveTimestamp, "true", StringComparison.OrdinalIgnoreCase)) + { + var version = new NuGetVersion(packageVersion); + var updatedVersion = new NuGetVersion(version.Version, VersionUtilities.GetNoTimestampReleaseLabel(version.Release)); + packageVersion = updatedVersion.ToNormalizedString(); + } + + Log.LogMessage(MessageImportance.High, $" - Tool: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("DotNetCliToolReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + + itemGroupElement.AppendChild(packageReferenceElement); + } + + foreach (var package in externalArchiveTools) + { + var packageName = package.ItemSpec; + var packageVersion = package.GetMetadata("Version"); + Log.LogMessage(MessageImportance.High, $" - Tool: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("DotNetCliToolReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + + itemGroupElement.AppendChild(packageReferenceElement); + } + projectElement.AppendChild(itemGroupElement); + + // Save updated file + xmlDoc.AppendChild(projectElement); + xmlDoc.Save(ReferencePackagePath); + + return true; + } + } +} diff --git a/build/tasks/AddMetapackageReferences.cs b/build/tasks/AddMetapackageReferences.cs index 3db68880cc..f6edbcf1f5 100644 --- a/build/tasks/AddMetapackageReferences.cs +++ b/build/tasks/AddMetapackageReferences.cs @@ -21,10 +21,14 @@ namespace RepoTasks [Required] public ITaskItem[] PackageArtifacts { get; set; } + [Required] + public ITaskItem[] ExternalDependencies { get; set; } + public override bool Execute() { // Parse input var metapackageArtifacts = PackageArtifacts.Where(p => p.GetMetadata("Metapackage") != "false"); + var externalArtifacts = ExternalDependencies.Where(p => p.GetMetadata("Metapackage") != "false"); var buildArtifacts = BuildArtifacts.Select(ArtifactInfo.Parse) .OfType() .Where(p => !p.IsSymbolsArtifact); @@ -37,14 +41,15 @@ namespace RepoTasks // Items var itemGroupElement = xmlDoc.CreateElement("ItemGroup"); - Log.LogMessage(MessageImportance.Normal, $"Runtime store will include the following packages"); + Log.LogMessage(MessageImportance.High, $"Runtime store will include the following packages"); + foreach (var package in metapackageArtifacts) { - var packageName = package.GetMetadata("Identity"); + var packageName = package.ItemSpec; var packageVersion = buildArtifacts .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) .PackageInfo.Version.ToString(); - Log.LogMessage(MessageImportance.Normal, $" - Package: {packageName} Version: {packageVersion}"); + Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); packageReferenceElement.SetAttribute("Include", packageName); @@ -52,6 +57,20 @@ namespace RepoTasks itemGroupElement.AppendChild(packageReferenceElement); } + + foreach (var package in externalArtifacts) + { + var packageName = package.ItemSpec; + var packageVersion = package.GetMetadata("Version"); + Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + + itemGroupElement.AppendChild(packageReferenceElement); + } + projectElement.AppendChild(itemGroupElement); // Save updated file diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 58b2008d5d..0d8e3fca8c 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -10,6 +10,7 @@ + diff --git a/build/tasks/Utilities/VersionUtilities.cs b/build/tasks/Utilities/VersionUtilities.cs new file mode 100644 index 0000000000..9bcd94956c --- /dev/null +++ b/build/tasks/Utilities/VersionUtilities.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace RepoTasks.Utilities +{ + public class VersionUtilities + { + public static string GetNoTimestampReleaseLabel(string releaseLabel) + { + if (releaseLabel.StartsWith("rtm-", StringComparison.OrdinalIgnoreCase)) + { + // E.g. change version 2.5.0-rtm-123123 to 2.5.0. + releaseLabel = string.Empty; + } + else + { + var timeStampFreeVersion = Environment.GetEnvironmentVariable("TIMESTAMP_FREE_VERSION"); + if (string.IsNullOrEmpty(timeStampFreeVersion)) + { + timeStampFreeVersion = "final"; + } + + if (!timeStampFreeVersion.StartsWith("-")) + { + timeStampFreeVersion = "-" + timeStampFreeVersion; + } + + // E.g. change version 2.5.0-rc2-123123 to 2.5.0-rc2-final. + var index = releaseLabel.LastIndexOf('-'); + if (index != -1) + { + releaseLabel = releaseLabel.Substring(0, index) + timeStampFreeVersion; + } + } + + return releaseLabel; + } + } +} \ No newline at end of file diff --git a/build/tools/templates/Archive/Archive.csproj b/build/tools/templates/Archive/Archive.csproj new file mode 100644 index 0000000000..252842049b --- /dev/null +++ b/build/tools/templates/Archive/Archive.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp2.0 + false + netcoreapp2.0 + $(DotNetRestoreSources) + + $(RestoreSources); + https://dotnet.myget.org/F/aspnet-2-0-2-october2017-patch/api/v3/index.json; + https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json; + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; + + + $(RestoreSources); + https://api.nuget.org/v3/index.json; + + + + \ No newline at end of file From 9aeefd3fd51152c6a767fd70a915ed5ef991f09a Mon Sep 17 00:00:00 2001 From: John Luo Date: Wed, 11 Oct 2017 16:34:00 -0700 Subject: [PATCH 077/115] Fix compile error --- build/tasks/AddArchiveReferences.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/tasks/AddArchiveReferences.cs b/build/tasks/AddArchiveReferences.cs index 9d12c34270..9ad3c815dc 100644 --- a/build/tasks/AddArchiveReferences.cs +++ b/build/tasks/AddArchiveReferences.cs @@ -56,7 +56,7 @@ namespace RepoTasks .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) .PackageInfo.Version.ToString(); - if (string.Equals(RemoveTimestamp, "true", StringComparison.OrdinalIgnoreCase)) + if (RemoveTimestamp) { var version = new NuGetVersion(packageVersion); var updatedVersion = new NuGetVersion(version.Version, VersionUtilities.GetNoTimestampReleaseLabel(version.Release)); @@ -92,7 +92,7 @@ namespace RepoTasks .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) .PackageInfo.Version.ToString(); - if (string.Equals(RemoveTimestamp, "true", StringComparison.OrdinalIgnoreCase)) + if (RemoveTimestamp) { var version = new NuGetVersion(packageVersion); var updatedVersion = new NuGetVersion(version.Version, VersionUtilities.GetNoTimestampReleaseLabel(version.Release)); From a48a3be6d73939eb6f167ee636bee20090e20a1c Mon Sep 17 00:00:00 2001 From: John Luo Date: Wed, 11 Oct 2017 17:04:09 -0700 Subject: [PATCH 078/115] Explicitly specify metapackage version --- build/PackageArchive.targets | 7 ++++--- build/tasks/AddArchiveReferences.cs | 11 ++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index a5f9c15ab6..a78b6c6408 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -22,7 +22,8 @@ BuildArtifacts="@(ArtifactInfo)" PackageArtifacts="@(PackageArtifact)" ExternalDependencies="@(ExternalDependency)" - RemoveTimestamp="$(RemoveTimestamp)" /> + RemoveTimestamp="$(RemoveTimestamp)" + MetapackageVersion="$(MetapackageVersion)" /> $(_WorkRoot)obj\$(OutputPackageName) @@ -69,10 +70,10 @@ + Properties="DotNetRestoreSources=$(TimestampSource);OutputPackageName=$(TimestampOutputPackageName);RemoveTimestamp=false;MetapackageVersion=$(TimeStampVersion)" /> + Properties="DotNetRestoreSources=$(TimestampFreeSource);OutputPackageName=$(TimestampFreeOutputPackageName);RemoveTimestamp=true;MetapackageVersion=$(TimeStampFreeVersion)" /> diff --git a/build/tasks/AddArchiveReferences.cs b/build/tasks/AddArchiveReferences.cs index 9ad3c815dc..8bc770b694 100644 --- a/build/tasks/AddArchiveReferences.cs +++ b/build/tasks/AddArchiveReferences.cs @@ -16,6 +16,9 @@ namespace RepoTasks [Required] public string ReferencePackagePath { get; set; } + [Required] + public string MetaPackageVersion { get; set; } + [Required] public bool RemoveTimestamp { get; set; } @@ -52,9 +55,11 @@ namespace RepoTasks foreach (var package in archiveArtifacts) { var packageName = package.ItemSpec; - var packageVersion = buildArtifacts - .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) - .PackageInfo.Version.ToString(); + var packageVersion = string.Equals(packageName, "Microsoft.AspNetCore.All", StringComparison.OrdinalIgnoreCase) ? + MetaPackageVersion : + buildArtifacts + .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) + .PackageInfo.Version.ToString(); if (RemoveTimestamp) { From 84157041bca3953591988421abdbe97a823358ec Mon Sep 17 00:00:00 2001 From: John Luo Date: Wed, 11 Oct 2017 17:43:59 -0700 Subject: [PATCH 079/115] Update properties fore restore on CI --- build/PackageArchive.targets | 4 ++-- build/tools/templates/Archive/Archive.csproj | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index a78b6c6408..0465a73e55 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -4,7 +4,7 @@ <_WorkRoot>$(RepositoryRoot).w\ - + @@ -37,7 +37,7 @@ + Properties="RestorePackagesPath=$(FallbackStagingDir);RuntimeFrameworkVersion=$(SharedFrameworkVersion);DotNetRestoreSources=$(DotNetRestoreSources);AspNetUniverseBuildOffline=true" /> diff --git a/build/tools/templates/Archive/Archive.csproj b/build/tools/templates/Archive/Archive.csproj index 252842049b..baa0ab2068 100644 --- a/build/tools/templates/Archive/Archive.csproj +++ b/build/tools/templates/Archive/Archive.csproj @@ -5,6 +5,7 @@ false netcoreapp2.0 $(DotNetRestoreSources) + $(RestoreSources);https://dotnet.myget.org/F/dotnet-core/api/v3/index.json; $(RestoreSources); https://dotnet.myget.org/F/aspnet-2-0-2-october2017-patch/api/v3/index.json; From c2a0010eda778a7e348ac5fa23f0e58b43a71d3f Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 12 Oct 2017 08:36:40 -0700 Subject: [PATCH 080/115] Fix the metapackage generation --- build/RuntimeStore.targets | 10 +++++++--- build/artifacts.props | 6 +++--- build/common.props | 2 +- build/dependencies.props | 7 +++++-- build/tasks/AddMetapackageReferences.cs | 2 ++ 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index f695ae6033..e5d4e1c67f 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -45,9 +45,14 @@ PackageArtifacts="@(PackageArtifact)" ExternalDependencies="@(ExternalDependency)" /> - + + + + @@ -56,7 +61,6 @@ - diff --git a/build/artifacts.props b/build/artifacts.props index f55587e6d0..4216072dcc 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -26,7 +26,7 @@ - + @@ -114,7 +114,7 @@ - + @@ -147,7 +147,7 @@ - + diff --git a/build/common.props b/build/common.props index 07f22d4eb3..20fe8aea4b 100644 --- a/build/common.props +++ b/build/common.props @@ -4,7 +4,7 @@ Microsoft ASP.NET Core - https://github.com/aspnet/MetaPackages + https://github.com/aspnet/Universe git false ..\..\build\Key.snk diff --git a/build/dependencies.props b/build/dependencies.props index 2ae47f4ba3..6add83592f 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -303,12 +303,15 @@ not building again in this patch. + + + @@ -323,16 +326,16 @@ not building again in this patch. + + - - diff --git a/build/tasks/AddMetapackageReferences.cs b/build/tasks/AddMetapackageReferences.cs index f6edbcf1f5..fab06c38ba 100644 --- a/build/tasks/AddMetapackageReferences.cs +++ b/build/tasks/AddMetapackageReferences.cs @@ -54,6 +54,7 @@ namespace RepoTasks var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); packageReferenceElement.SetAttribute("Include", packageName); packageReferenceElement.SetAttribute("Version", packageVersion); + packageReferenceElement.SetAttribute("PrivateAssets", "None"); itemGroupElement.AppendChild(packageReferenceElement); } @@ -67,6 +68,7 @@ namespace RepoTasks var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); packageReferenceElement.SetAttribute("Include", packageName); packageReferenceElement.SetAttribute("Version", packageVersion); + packageReferenceElement.SetAttribute("PrivateAssets", "None"); itemGroupElement.AppendChild(packageReferenceElement); } From 87b3cc6879b9d022ef2c0bb180806f11aa68355e Mon Sep 17 00:00:00 2001 From: Smit Patel Date: Thu, 12 Oct 2017 15:35:56 -0700 Subject: [PATCH 081/115] Update EF --- modules/EntityFrameworkCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/EntityFrameworkCore b/modules/EntityFrameworkCore index 196400e9ca..81d6c37d36 160000 --- a/modules/EntityFrameworkCore +++ b/modules/EntityFrameworkCore @@ -1 +1 @@ -Subproject commit 196400e9ca7ebaac92e6e1e7d638fc1c82446385 +Subproject commit 81d6c37d360701a30c2c0bbb8a4492f4308be35a From 446de15f0bc1b8d7d651c5e90c11c0479512e8b8 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 12 Oct 2017 12:06:36 -0700 Subject: [PATCH 082/115] Add aspnet/Templating to the universe build --- .gitmodules | 4 ++ build/RuntimeStore.targets | 19 ++--- build/Templating.targets | 69 +++++++++++++++++++ build/artifacts.props | 5 ++ build/common.props | 1 - build/dependencies.props | 4 +- build/repo.targets | 4 +- .../tasks/GeneratePackageVersionPropsFile.cs | 2 + .../tasks/GenerateRestoreSourcesPropsFile.cs | 5 +- korebuild-lock.txt | 4 +- modules/Templating | 1 + src/Directory.Build.props | 4 ++ .../Microsoft.AspNetCore.All.csproj | 2 +- version.props | 2 + 14 files changed, 106 insertions(+), 20 deletions(-) create mode 100644 build/Templating.targets create mode 160000 modules/Templating create mode 100644 src/Directory.Build.props diff --git a/.gitmodules b/.gitmodules index 76a2fb3298..297e22efd4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -166,3 +166,7 @@ path = modules/Testing url = https://github.com/aspnet/Testing.git branch = patch/2.0.1 +[submodule "modules/Templating"] + path = modules/Templating + url = https://github.com/aspnet/Templating.git + branch = rel/2.0.3 diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index e5d4e1c67f..a6f91b12c0 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -1,12 +1,6 @@ - - 1.0.0 - $(VersionPrefix)-$(VersionSuffix) - $(VersionPrefix) - - <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\ <_DependencyMirrorDirectory>$(RepositoryRoot).deps\mirror\ @@ -37,6 +31,7 @@ + - <_RsManifestProps>MicrosoftAspNetCoreAllPackageVersion=$(Version) + <_RsManifestProps>MicrosoftAspNetCoreAllPackageVersion=$(PackageVersion) <_RsManifestProps>$(_RsManifestProps);DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath) @@ -144,8 +139,8 @@ - $(ArtifactsDir)aspnetcore-store-$(Version)-$(RuntimeStoreRID).zip - $(ArtifactsDir)aspnetcore-symbols-$(Version)-$(RuntimeStoreRID).zip + $(ArtifactsDir)aspnetcore-store-$(PackageVersion)-$(RuntimeStoreRID).zip + $(ArtifactsDir)aspnetcore-symbols-$(PackageVersion)-$(RuntimeStoreRID).zip @@ -162,7 +157,7 @@ NewManifests="@(RuntimeStoreManifestFiles)" RuntimeStoreFiles="@(RuntimeStoreFiles)" RuntimeStoreSymbolFiles="@(RuntimeStoreSymbolFiles)" - ManifestDestination="$(ArtifactsDir)aspnetcore-store-$(Version)-$(RuntimeStoreRID).xml" + ManifestDestination="$(ArtifactsDir)aspnetcore-store-$(PackageVersion)-$(RuntimeStoreRID).xml" StoreDestination="$(_StoreZipDirectory)store\" SymbolsDestination="$(_SymbolsZipDirectory)"/> @@ -181,7 +176,7 @@ - aspnetcore-store-$(Version)-common.xml + aspnetcore-store-$(PackageVersion)-common.xml @@ -210,7 +205,7 @@ + ManifestDestination="$(_WorkRoot)\build\aspnetcore-store-$(PackageVersion).xml"/> diff --git a/build/Templating.targets b/build/Templating.targets new file mode 100644 index 0000000000..31e026120b --- /dev/null +++ b/build/Templating.targets @@ -0,0 +1,69 @@ + + + $(MSBuildThisFileDirectory)..\modules\Templating\ + + + + + $(IntermediateDir)dependencies.notimestamp.props + + RepositoryRoot=$(TemplatingProjectRoot); + DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath); + BuildNumber=$(BuildNumber); + Configuration=$(Configuration); + + + $(TemplateProjCommmonProperties); + DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath); + VersionMetadata=timestamped; + + + $(TemplateProjCommmonProperties); + DotNetPackageVersionPropsPath=$(GeneratedNoTimestampPackageVersionPropsPath); + BuildNumber=$(BuildNumber)-final; + VersionMetadata=notimestamp; + + + + + + + + + + + + + + + + + + + <_NoTimestampPackages Include="@(ExternalDependency)" /> + <_NoTimestampPackages Include="%(FinalBuildArtifactInfo.PackageId)" Version="%(FinalBuildArtifactInfo.Version)" Condition=" '%(FinalBuildArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> + <_NoTimestampPackages Include="Microsoft.AspNetCore.All" Version="$(PackageVersionNoTimestamp)" /> + + + + + + + + + + + + + + + diff --git a/build/artifacts.props b/build/artifacts.props index 4216072dcc..e9aa8e3a92 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -113,6 +113,7 @@ + @@ -121,6 +122,10 @@ + + + + diff --git a/build/common.props b/build/common.props index 20fe8aea4b..db6fa6bd04 100644 --- a/build/common.props +++ b/build/common.props @@ -1,5 +1,4 @@ - diff --git a/build/dependencies.props b/build/dependencies.props index 6add83592f..5f901f3a7e 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -123,7 +123,7 @@ https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json - 2.0.2-beta-15522 + 2.0.2-rc1-15526 @@ -270,7 +270,7 @@ KRB2004 - + XunitRunnerVisualStudioPackageVersion diff --git a/build/repo.targets b/build/repo.targets index 67ca16bf53..4465cf7065 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -2,6 +2,7 @@ + @@ -22,7 +23,7 @@ $(CleanDependsOn);CleanArtifacts;CleanUniverseArtifacts $(RestoreDependsOn);RestoreExternalDependencies $(CompileDependsOn);BuildRepositories - $(PackageDependsOn);BuildAllMetapackage;SplitPackages + $(PackageDependsOn);BuildAllMetapackage;BuildTemplates;SplitPackages $(VerifyDependsOn);VerifyCoherentVersions @@ -44,6 +45,7 @@ <_LineupPackages Include="@(ExternalDependency)" /> <_LineupPackages Include="%(ArtifactInfo.PackageId)" Version="%(ArtifactInfo.Version)" Condition=" '%(ArtifactInfo.ArtifactType)' == 'NuGetPackage' " /> + <_LineupPackages Include="Microsoft.AspNetCore.All" Version="$(PackageVersion)" /> <_LineupSources Include="$(_DependencyPackagesDirectory)" Condition="'$(_DependencyPackagesDirectory)' != '' AND Exists('$(_DependencyPackagesDirectory)')" /> <_LineupSources Include="$(BuildDir)" /> diff --git a/build/tasks/GeneratePackageVersionPropsFile.cs b/build/tasks/GeneratePackageVersionPropsFile.cs index 43a12b6bde..c007fd321f 100644 --- a/build/tasks/GeneratePackageVersionPropsFile.cs +++ b/build/tasks/GeneratePackageVersionPropsFile.cs @@ -32,6 +32,8 @@ namespace RepoTasks var root = new XElement("Project", props); var doc = new XDocument(root); + props.Add(new XElement("MSBuildAllProjects", "$(MSBuildAllProjects);$(MSBuildThisFileFullPath)")); + var varNames = new HashSet(); var versionElements = new List(); foreach (var pkg in Packages) diff --git a/build/tasks/GenerateRestoreSourcesPropsFile.cs b/build/tasks/GenerateRestoreSourcesPropsFile.cs index 253420dfae..2d27ba4e6a 100644 --- a/build/tasks/GenerateRestoreSourcesPropsFile.cs +++ b/build/tasks/GenerateRestoreSourcesPropsFile.cs @@ -26,7 +26,10 @@ namespace RepoTasks Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); var sources = new XElement("DotNetRestoreSources"); - var doc = new XDocument(new XElement("Project", new XElement("PropertyGroup", sources))); + var propertyGroup = new XElement("PropertyGroup", sources); + var doc = new XDocument(new XElement("Project", propertyGroup)); + + propertyGroup.Add(new XElement("MSBuildAllProjects", "$(MSBuildAllProjects);$(MSBuildThisFileFullPath)")); var sb = new StringBuilder(); diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 2a51b344b6..7f216dfb7a 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-beta-15522 -commithash:f26cb086ebd8b5a187381bb9a431794f11d69025 +version:2.0.2-rc1-15526 +commithash:436afc3dc08f5e278431db807866cc5f032f4d7b diff --git a/modules/Templating b/modules/Templating new file mode 160000 index 0000000000..a4ba5a1cb1 --- /dev/null +++ b/modules/Templating @@ -0,0 +1 @@ +Subproject commit a4ba5a1cb1c61d3a4bead34915baccf438a8d1b3 diff --git a/src/Directory.Build.props b/src/Directory.Build.props new file mode 100644 index 0000000000..7d57abfc71 --- /dev/null +++ b/src/Directory.Build.props @@ -0,0 +1,4 @@ + + + + diff --git a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj index 67dda6b16a..d7155f1cc3 100644 --- a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj +++ b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj @@ -1,6 +1,5 @@ - @@ -15,6 +14,7 @@ false + false netcoreapp2.0 aspnetcore Microsoft.AspNetCore.All diff --git a/version.props b/version.props index 7e4c6138cc..9cccb9ea3e 100644 --- a/version.props +++ b/version.props @@ -3,7 +3,9 @@ 2.0.3 rtm $(VersionPrefix) + $(VersionPrefix) $(VersionPrefix)-$(VersionSuffix)-final + $(VersionPrefix)-$(VersionSuffix)-final $(VersionSuffix)-$(BuildNumber) From 91d4be47b9843c3464243c6325e6bbe0d47a9fe5 Mon Sep 17 00:00:00 2001 From: John Luo Date: Thu, 12 Oct 2017 12:54:37 -0700 Subject: [PATCH 083/115] Produce separate reference package for runtime store generation --- build/RuntimeStore.targets | 40 +++- build/artifacts.props | 194 +++++++++--------- build/dependencies.props | 84 ++++---- build/tasks/AddMetapackageReferences.cs | 6 +- build/tasks/AddRSReferences.cs | 85 ++++++++ build/tasks/RepoTasks.tasks | 1 + build/tasks/ResolveHostingStartupPackages.cs | 2 +- .../templates/RS.Manifest/RS.Manifest.csproj | 2 +- .../RS.References/RS.References.csproj | 18 ++ 9 files changed, 288 insertions(+), 144 deletions(-) create mode 100644 build/tasks/AddRSReferences.cs create mode 100644 build/tools/templates/RS.References/RS.References.csproj diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index a6f91b12c0..1706cba3d3 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -33,7 +33,7 @@ - + - + + + + $(_WorkRoot)RS.References\ + + + + + + + + + + + + + + + + + + + + + + + @@ -91,7 +125,7 @@ - + <__ComposeStoreProps /> <_ComposeStoreProps>$(_ComposeStoreProps);$(_RsManifestProps) diff --git a/build/artifacts.props b/build/artifacts.props index e9aa8e3a92..fe06653938 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -2,6 +2,8 @@ false + false + false false false @@ -10,52 +12,52 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - + + + + + + + - - + + - + - - - + + + - - - - - - - + + + + + + + @@ -65,88 +67,88 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - - - - - - + + + + + + - - - + + + - - - - + + + + - - - + + + - + - - - + + + - + - - + + - + - - - + + + - - - - - - - - + + + + + + + + - + diff --git a/build/dependencies.props b/build/dependencies.props index 5f901f3a7e..d3d5ad054c 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -14,6 +14,10 @@ false false + + false + + false false @@ -293,46 +297,46 @@ not building again in this patch. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/tasks/AddMetapackageReferences.cs b/build/tasks/AddMetapackageReferences.cs index fab06c38ba..5488dec22e 100644 --- a/build/tasks/AddMetapackageReferences.cs +++ b/build/tasks/AddMetapackageReferences.cs @@ -27,8 +27,8 @@ namespace RepoTasks public override bool Execute() { // Parse input - var metapackageArtifacts = PackageArtifacts.Where(p => p.GetMetadata("Metapackage") != "false"); - var externalArtifacts = ExternalDependencies.Where(p => p.GetMetadata("Metapackage") != "false"); + var metapackageArtifacts = PackageArtifacts.Where(p => p.GetMetadata("Metapackage") == "true"); + var externalArtifacts = ExternalDependencies.Where(p => p.GetMetadata("Metapackage") == "true"); var buildArtifacts = BuildArtifacts.Select(ArtifactInfo.Parse) .OfType() .Where(p => !p.IsSymbolsArtifact); @@ -41,7 +41,7 @@ namespace RepoTasks // Items var itemGroupElement = xmlDoc.CreateElement("ItemGroup"); - Log.LogMessage(MessageImportance.High, $"Runtime store will include the following packages"); + Log.LogMessage(MessageImportance.High, $"Metapackage will include the following packages"); foreach (var package in metapackageArtifacts) { diff --git a/build/tasks/AddRSReferences.cs b/build/tasks/AddRSReferences.cs new file mode 100644 index 0000000000..1cbc504661 --- /dev/null +++ b/build/tasks/AddRSReferences.cs @@ -0,0 +1,85 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Xml; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class AddRSReferences : Task + { + [Required] + public string ReferencePackagePath { get; set; } + + [Required] + public ITaskItem[] BuildArtifacts { get; set; } + + [Required] + public ITaskItem[] PackageArtifacts { get; set; } + + [Required] + public ITaskItem[] ExternalDependencies { get; set; } + + public override bool Execute() + { + // Parse input + var runtimeStoreArtifacts = PackageArtifacts.Where(p => p.GetMetadata("RuntimeStore") == "true"); + var externalArtifacts = ExternalDependencies.Where(p => p.GetMetadata("RuntimeStore") == "true"); + var buildArtifacts = BuildArtifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact); + + var xmlDoc = new XmlDocument(); + xmlDoc.Load(ReferencePackagePath); + + // Project + var projectElement = xmlDoc.FirstChild; + + // Items + var itemGroupElement = xmlDoc.CreateElement("ItemGroup"); + Log.LogMessage(MessageImportance.High, $"Runtime store will include the following packages"); + + foreach (var package in runtimeStoreArtifacts) + { + var packageName = package.ItemSpec; + var packageVersion = buildArtifacts + .Single(p => string.Equals(p.PackageInfo.Id, packageName, StringComparison.OrdinalIgnoreCase)) + .PackageInfo.Version.ToString(); + Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + packageReferenceElement.SetAttribute("PrivateAssets", "None"); + + itemGroupElement.AppendChild(packageReferenceElement); + } + + foreach (var package in externalArtifacts) + { + var packageName = package.ItemSpec; + var packageVersion = package.GetMetadata("Version"); + Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); + + var packageReferenceElement = xmlDoc.CreateElement("PackageReference"); + packageReferenceElement.SetAttribute("Include", packageName); + packageReferenceElement.SetAttribute("Version", packageVersion); + packageReferenceElement.SetAttribute("PrivateAssets", "None"); + + itemGroupElement.AppendChild(packageReferenceElement); + } + + projectElement.AppendChild(itemGroupElement); + + // Save updated file + xmlDoc.AppendChild(projectElement); + xmlDoc.Save(ReferencePackagePath); + + return true; + } + } +} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 0d8e3fca8c..b390901779 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -11,6 +11,7 @@ + diff --git a/build/tasks/ResolveHostingStartupPackages.cs b/build/tasks/ResolveHostingStartupPackages.cs index 0d5b0838f4..8bfc27399c 100644 --- a/build/tasks/ResolveHostingStartupPackages.cs +++ b/build/tasks/ResolveHostingStartupPackages.cs @@ -21,7 +21,7 @@ namespace RepoTasks public override bool Execute() { // Parse input - var hostingStartupArtifacts = PackageArtifacts.Where(p => p.GetMetadata("Metapackage") == "hostingstartup"); + var hostingStartupArtifacts = PackageArtifacts.Where(p => p.GetMetadata("HostingStartup") == "true"); HostingStartupArtifacts = BuildArtifacts.Where(p => hostingStartupArtifacts.Any(h => h.GetMetadata("Identity") == p.GetMetadata("PackageId"))).ToArray(); return true; diff --git a/build/tools/templates/RS.Manifest/RS.Manifest.csproj b/build/tools/templates/RS.Manifest/RS.Manifest.csproj index 514fd38948..e8c8261383 100644 --- a/build/tools/templates/RS.Manifest/RS.Manifest.csproj +++ b/build/tools/templates/RS.Manifest/RS.Manifest.csproj @@ -19,7 +19,7 @@ - + diff --git a/build/tools/templates/RS.References/RS.References.csproj b/build/tools/templates/RS.References/RS.References.csproj new file mode 100644 index 0000000000..91977d96d5 --- /dev/null +++ b/build/tools/templates/RS.References/RS.References.csproj @@ -0,0 +1,18 @@ + + + + + + $(DotNetRestoreSources) + + $(RestoreSources); + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; + + + $(RestoreSources); + https://api.nuget.org/v3/index.json; + + netcoreapp2.0 + + + From 77c846201dbac47869984fd5759944f3a7dc8a2d Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 13 Oct 2017 11:22:14 -0700 Subject: [PATCH 084/115] Fix runtime store build --- build/RuntimeStore.targets | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 1706cba3d3..5ffce60009 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -80,12 +80,12 @@ + Properties="Configuration=$(Configuration);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath);_Target=Restore" /> + Properties="Configuration=$(Configuration);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath)" /> @@ -96,9 +96,6 @@ - - - <_RuntimeStoreRestoreSources Include="$(_DependencyBuildDirectory)" Condition="Exists($(_DependencyBuildDirectory))" /> <_RuntimeStoreRestoreSources Include="$(_DependencyMirrorDirectory)" Condition="Exists($(_DependencyMirrorDirectory))" /> @@ -125,7 +122,10 @@ - + + + + <__ComposeStoreProps /> <_ComposeStoreProps>$(_ComposeStoreProps);$(_RsManifestProps) From 286c0a7e3b6ef2113e420644739b43133e33ca0d Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 13 Oct 2017 13:15:13 -0700 Subject: [PATCH 085/115] Update submodules to latest build tools and fix bug in razor tests --- modules/Antiforgery | 2 +- modules/AzureIntegration | 2 +- modules/BasicMiddleware | 2 +- modules/BrowserLink | 2 +- modules/CORS | 2 +- modules/Caching | 2 +- modules/Common | 2 +- modules/Configuration | 2 +- modules/DataProtection | 2 +- modules/DependencyInjection | 2 +- modules/Diagnostics | 2 +- modules/DotNetTools | 2 +- modules/EntityFrameworkCore | 2 +- modules/EventNotification | 2 +- modules/FileSystem | 2 +- modules/Hosting | 2 +- modules/HtmlAbstractions | 2 +- modules/HttpAbstractions | 2 +- modules/HttpSysServer | 2 +- modules/IISIntegration | 2 +- modules/Identity | 2 +- modules/JavaScriptServices | 2 +- modules/JsonPatch | 2 +- modules/KestrelHttpServer | 2 +- modules/Localization | 2 +- modules/Logging | 2 +- modules/MetaPackages | 2 +- modules/Microsoft.Data.Sqlite | 2 +- modules/Mvc | 2 +- modules/MvcPrecompilation | 2 +- modules/Options | 2 +- modules/Proxy | 2 +- modules/Razor | 2 +- modules/ResponseCaching | 2 +- modules/Routing | 2 +- modules/Scaffolding | 2 +- modules/Security | 2 +- modules/ServerTests | 2 +- modules/Session | 2 +- modules/StaticFiles | 2 +- modules/Templating | 2 +- modules/Testing | 2 +- modules/WebSockets | 2 +- 43 files changed, 43 insertions(+), 43 deletions(-) diff --git a/modules/Antiforgery b/modules/Antiforgery index a45c919344..06b53886bd 160000 --- a/modules/Antiforgery +++ b/modules/Antiforgery @@ -1 +1 @@ -Subproject commit a45c919344bd40ee719e41b205d7418728a3c7fa +Subproject commit 06b53886bd9bfafa3e96da22d61edcbad8db3842 diff --git a/modules/AzureIntegration b/modules/AzureIntegration index 7f86cd58fd..714ea279ca 160000 --- a/modules/AzureIntegration +++ b/modules/AzureIntegration @@ -1 +1 @@ -Subproject commit 7f86cd58fd91ff8bb8c4286eda5444188d002002 +Subproject commit 714ea279ca648cd2fce0a302af9d79c708b0e954 diff --git a/modules/BasicMiddleware b/modules/BasicMiddleware index 47e4b45852..8319d8eb0f 160000 --- a/modules/BasicMiddleware +++ b/modules/BasicMiddleware @@ -1 +1 @@ -Subproject commit 47e4b458528c2d05c8b0004de1718b5134687211 +Subproject commit 8319d8eb0f2711e97ce86c9d98ec7a71d4710a21 diff --git a/modules/BrowserLink b/modules/BrowserLink index 0c496c3652..d35de0ae6b 160000 --- a/modules/BrowserLink +++ b/modules/BrowserLink @@ -1 +1 @@ -Subproject commit 0c496c3652faf6f8091394f31ffc03e8f3199ffa +Subproject commit d35de0ae6ba7f5a670387192ef0a5c5fef94a073 diff --git a/modules/CORS b/modules/CORS index a95ab767fd..5a617d8cf1 160000 --- a/modules/CORS +++ b/modules/CORS @@ -1 +1 @@ -Subproject commit a95ab767fda039777a81fc712f6986cde15bff8d +Subproject commit 5a617d8cf152ed6cad54f93cecb8dc4dcdc47224 diff --git a/modules/Caching b/modules/Caching index 19dc192591..f3541e9187 160000 --- a/modules/Caching +++ b/modules/Caching @@ -1 +1 @@ -Subproject commit 19dc192591c77f998c0856213ab7ae088997d043 +Subproject commit f3541e918714cdad4c24968292470b9531095b92 diff --git a/modules/Common b/modules/Common index 7309bc455b..007f386436 160000 --- a/modules/Common +++ b/modules/Common @@ -1 +1 @@ -Subproject commit 7309bc455b15eb9cf5634fd281a227b744d29b68 +Subproject commit 007f3864365947058b7ab95d605a44892b06eed9 diff --git a/modules/Configuration b/modules/Configuration index ebc12e335e..99777e88f9 160000 --- a/modules/Configuration +++ b/modules/Configuration @@ -1 +1 @@ -Subproject commit ebc12e335e194893f9e1818ac32078f02ebc6b4a +Subproject commit 99777e88f936170830708fc51482fc23acc2dfec diff --git a/modules/DataProtection b/modules/DataProtection index f5004b5ad8..7cc58239c4 160000 --- a/modules/DataProtection +++ b/modules/DataProtection @@ -1 +1 @@ -Subproject commit f5004b5ad8c8d802d584d9a186cd96e362e66857 +Subproject commit 7cc58239c477bcc91349f9bd217723ca46ab2da9 diff --git a/modules/DependencyInjection b/modules/DependencyInjection index b5e7f41cd9..cd2269792e 160000 --- a/modules/DependencyInjection +++ b/modules/DependencyInjection @@ -1 +1 @@ -Subproject commit b5e7f41cd9a2ac2650f3168d09de491fa10f2336 +Subproject commit cd2269792e5ec2ced8a7fe475236fde836663aa1 diff --git a/modules/Diagnostics b/modules/Diagnostics index 0372c762cf..111b6a4821 160000 --- a/modules/Diagnostics +++ b/modules/Diagnostics @@ -1 +1 @@ -Subproject commit 0372c762cf5df2084e9b9cd2ac299a63d707afc6 +Subproject commit 111b6a4821f3d23b33a720a45b91a79ed720fab5 diff --git a/modules/DotNetTools b/modules/DotNetTools index 72add92b43..8586b2d47d 160000 --- a/modules/DotNetTools +++ b/modules/DotNetTools @@ -1 +1 @@ -Subproject commit 72add92b43df8327d14f38b25013ad035d8b71a1 +Subproject commit 8586b2d47d9d3e54cb349bcb156bc4bb20426e5b diff --git a/modules/EntityFrameworkCore b/modules/EntityFrameworkCore index 81d6c37d36..98f4a94b89 160000 --- a/modules/EntityFrameworkCore +++ b/modules/EntityFrameworkCore @@ -1 +1 @@ -Subproject commit 81d6c37d360701a30c2c0bbb8a4492f4308be35a +Subproject commit 98f4a94b89bec705bc9377e2609c5c0807d01417 diff --git a/modules/EventNotification b/modules/EventNotification index bd62d77546..b6ed5ce753 160000 --- a/modules/EventNotification +++ b/modules/EventNotification @@ -1 +1 @@ -Subproject commit bd62d775466ff5f7286ba847681451f3ecaa3b0a +Subproject commit b6ed5ce7534a7ead28cbe4bc6dc0458f53b0d4f7 diff --git a/modules/FileSystem b/modules/FileSystem index befc6c1a86..73a4b45edb 160000 --- a/modules/FileSystem +++ b/modules/FileSystem @@ -1 +1 @@ -Subproject commit befc6c1a86f36bcc66ed4ab6562b9904b56a4802 +Subproject commit 73a4b45edbb14b6f3f5063400ab67f83230d5414 diff --git a/modules/Hosting b/modules/Hosting index befe353db8..86df76440b 160000 --- a/modules/Hosting +++ b/modules/Hosting @@ -1 +1 @@ -Subproject commit befe353db8a376c1a2c6b58c62f08d315d90e0b2 +Subproject commit 86df76440b450daa3972d5b317026b58f718b803 diff --git a/modules/HtmlAbstractions b/modules/HtmlAbstractions index dc04ba8ee7..dd02e6829e 160000 --- a/modules/HtmlAbstractions +++ b/modules/HtmlAbstractions @@ -1 +1 @@ -Subproject commit dc04ba8ee7137289afe4c64d8b1b7177ceb2e780 +Subproject commit dd02e6829ef4210795c17d198a97465e9b1875ea diff --git a/modules/HttpAbstractions b/modules/HttpAbstractions index 4afe4480b4..9cfc7502c1 160000 --- a/modules/HttpAbstractions +++ b/modules/HttpAbstractions @@ -1 +1 @@ -Subproject commit 4afe4480b4909141160ded50487b575cc8fd4cd8 +Subproject commit 9cfc7502c15bf0f2240121d7d68b515fe16a8c95 diff --git a/modules/HttpSysServer b/modules/HttpSysServer index 7ea2905820..88862aeb68 160000 --- a/modules/HttpSysServer +++ b/modules/HttpSysServer @@ -1 +1 @@ -Subproject commit 7ea2905820aac58488466ad4671d9397566c3173 +Subproject commit 88862aeb6831567d182e406ecffb646bc327d44a diff --git a/modules/IISIntegration b/modules/IISIntegration index f8a04f2d4e..f270358b27 160000 --- a/modules/IISIntegration +++ b/modules/IISIntegration @@ -1 +1 @@ -Subproject commit f8a04f2d4e7a0d401e7fed445262d57fcc64b81e +Subproject commit f270358b27174f0f6a6afae44715f8176bd9d486 diff --git a/modules/Identity b/modules/Identity index 8a880adf72..98fa107c05 160000 --- a/modules/Identity +++ b/modules/Identity @@ -1 +1 @@ -Subproject commit 8a880adf7299777d4ef025fa78c23be072fb8a08 +Subproject commit 98fa107c0505a671b68a5b13d11b72ebc6168cf6 diff --git a/modules/JavaScriptServices b/modules/JavaScriptServices index 86e94d7812..64389a9bbe 160000 --- a/modules/JavaScriptServices +++ b/modules/JavaScriptServices @@ -1 +1 @@ -Subproject commit 86e94d7812d0c8790ebcb32a6ae6935b85648d9c +Subproject commit 64389a9bbeda7378c80b4c302700ddcb78d4f0aa diff --git a/modules/JsonPatch b/modules/JsonPatch index bfd5a994cb..8580d8beb5 160000 --- a/modules/JsonPatch +++ b/modules/JsonPatch @@ -1 +1 @@ -Subproject commit bfd5a994cb68dd090694d38b5695958665bcc876 +Subproject commit 8580d8beb5c9ca94e09455f2dac1d1587767478e diff --git a/modules/KestrelHttpServer b/modules/KestrelHttpServer index 39b56770a9..be4e401aa6 160000 --- a/modules/KestrelHttpServer +++ b/modules/KestrelHttpServer @@ -1 +1 @@ -Subproject commit 39b56770a97cdc4943cb2d652329b3ca9afc84a5 +Subproject commit be4e401aa69369b454e448d701c9c3fb0e48113b diff --git a/modules/Localization b/modules/Localization index 6be6b8fe1d..4ee17ae30b 160000 --- a/modules/Localization +++ b/modules/Localization @@ -1 +1 @@ -Subproject commit 6be6b8fe1d44cbc04e85d06ae69e4eef94091f28 +Subproject commit 4ee17ae30bfbd464e11ceb28a5e4ee00c6141725 diff --git a/modules/Logging b/modules/Logging index 527a2809bf..6f6781e3fe 160000 --- a/modules/Logging +++ b/modules/Logging @@ -1 +1 @@ -Subproject commit 527a2809bf4739a3502fdc87626411fd04f483a8 +Subproject commit 6f6781e3fe98a7d8005f7b8e13e9e302c4067119 diff --git a/modules/MetaPackages b/modules/MetaPackages index 3d5b69063f..62790d278f 160000 --- a/modules/MetaPackages +++ b/modules/MetaPackages @@ -1 +1 @@ -Subproject commit 3d5b69063fa85a4c504f7d9208e260b8c33bd683 +Subproject commit 62790d278fe79b90d51e288c3ff3fe2313acc7bb diff --git a/modules/Microsoft.Data.Sqlite b/modules/Microsoft.Data.Sqlite index e911180fea..1ef4e2bf1b 160000 --- a/modules/Microsoft.Data.Sqlite +++ b/modules/Microsoft.Data.Sqlite @@ -1 +1 @@ -Subproject commit e911180feacb80daddeb374e28169fb5d85e62aa +Subproject commit 1ef4e2bf1b6fd14655dd8df918344b50add48ce3 diff --git a/modules/Mvc b/modules/Mvc index 3bfb023679..f8789f5d5c 160000 --- a/modules/Mvc +++ b/modules/Mvc @@ -1 +1 @@ -Subproject commit 3bfb023679e73b19b0a513002ccffd42d820eecb +Subproject commit f8789f5d5c4d5869490a05b8f7250b6151f1673e diff --git a/modules/MvcPrecompilation b/modules/MvcPrecompilation index 9d07f205f2..bc58d8495a 160000 --- a/modules/MvcPrecompilation +++ b/modules/MvcPrecompilation @@ -1 +1 @@ -Subproject commit 9d07f205f2bda99f7b62c0d65ad70043b22e7a22 +Subproject commit bc58d8495a431d3de606afc52c2987d1ebf1e6ad diff --git a/modules/Options b/modules/Options index 35c1a7a57b..0ea1fec422 160000 --- a/modules/Options +++ b/modules/Options @@ -1 +1 @@ -Subproject commit 35c1a7a57b1d71ab3feb40db353b4505c4d53793 +Subproject commit 0ea1fec42203b95785d6beade2e8ea6a93cdf47b diff --git a/modules/Proxy b/modules/Proxy index 2875d8e4a1..f0bec28f34 160000 --- a/modules/Proxy +++ b/modules/Proxy @@ -1 +1 @@ -Subproject commit 2875d8e4a14f9375d409ee42cd5d9a2d49363295 +Subproject commit f0bec28f34e41824019c5eb87de67c73173e6a26 diff --git a/modules/Razor b/modules/Razor index e45618eb1b..a1671c9b9f 160000 --- a/modules/Razor +++ b/modules/Razor @@ -1 +1 @@ -Subproject commit e45618eb1b0d5b37091dc38f169c24b4f546cdaf +Subproject commit a1671c9b9ff1dc574501f5036d9cf14a316d12df diff --git a/modules/ResponseCaching b/modules/ResponseCaching index bf08e66cee..b0362dab9c 160000 --- a/modules/ResponseCaching +++ b/modules/ResponseCaching @@ -1 +1 @@ -Subproject commit bf08e66ceeabd7a8ef8eef9fe3d8492b1b6c23ba +Subproject commit b0362dab9cd4c5bf8de364694d4880ab2c948be5 diff --git a/modules/Routing b/modules/Routing index d104dd8f5f..763aa1447e 160000 --- a/modules/Routing +++ b/modules/Routing @@ -1 +1 @@ -Subproject commit d104dd8f5fc9658214a49a3ceb2a7b1ad448ef54 +Subproject commit 763aa1447e340c4d6607b1542183817d5f08e257 diff --git a/modules/Scaffolding b/modules/Scaffolding index 2f754921f5..5ff6bc6399 160000 --- a/modules/Scaffolding +++ b/modules/Scaffolding @@ -1 +1 @@ -Subproject commit 2f754921f5b58450fe901a179cc9b313ce083a1b +Subproject commit 5ff6bc6399e373535e8e91924a439180d794f1c0 diff --git a/modules/Security b/modules/Security index 30b629edfd..b9d90b0376 160000 --- a/modules/Security +++ b/modules/Security @@ -1 +1 @@ -Subproject commit 30b629edfdc20cdc2d11df01657b94f1e0b9772c +Subproject commit b9d90b03760dab2610be32ca561c702ca3e10a6e diff --git a/modules/ServerTests b/modules/ServerTests index 369b64b1b6..9533c2c643 160000 --- a/modules/ServerTests +++ b/modules/ServerTests @@ -1 +1 @@ -Subproject commit 369b64b1b6f2a816669e4f9929211b0780485ac3 +Subproject commit 9533c2c6435fb86137de0914a93e76a04567eef5 diff --git a/modules/Session b/modules/Session index c2b35d75a3..6cbe68fb3c 160000 --- a/modules/Session +++ b/modules/Session @@ -1 +1 @@ -Subproject commit c2b35d75a3c0358e7773697252552ab396b0120b +Subproject commit 6cbe68fb3c40bbf5e9c9ca2e87b0e4d51d3964f1 diff --git a/modules/StaticFiles b/modules/StaticFiles index 3368112e15..f83ec642e7 160000 --- a/modules/StaticFiles +++ b/modules/StaticFiles @@ -1 +1 @@ -Subproject commit 3368112e15f76f688555968ea16bf4f766dc3ed8 +Subproject commit f83ec642e7421f6ae133d4ef459a20729c9c16fb diff --git a/modules/Templating b/modules/Templating index a4ba5a1cb1..4a897d69bc 160000 --- a/modules/Templating +++ b/modules/Templating @@ -1 +1 @@ -Subproject commit a4ba5a1cb1c61d3a4bead34915baccf438a8d1b3 +Subproject commit 4a897d69bcfab85b1eb9cc186117230cbcb17f2f diff --git a/modules/Testing b/modules/Testing index abd5da6ccf..e270986c14 160000 --- a/modules/Testing +++ b/modules/Testing @@ -1 +1 @@ -Subproject commit abd5da6ccf89ff70f52df7060d7f8dbb39dc5779 +Subproject commit e270986c142758ab529d6635394c76482f9c5416 diff --git a/modules/WebSockets b/modules/WebSockets index b956ad3143..5a9214e335 160000 --- a/modules/WebSockets +++ b/modules/WebSockets @@ -1 +1 @@ -Subproject commit b956ad3143ef4d7144f4f55a9d783679fe60a044 +Subproject commit 5a9214e3355be3d4b8eec873e3d51e5737e061f0 From 5ef64d476456a4dbea484697188632b3c3a8d29c Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 13 Oct 2017 13:47:58 -0700 Subject: [PATCH 086/115] Fix build --- build/RuntimeStore.targets | 11 ++++++----- build/tools/templates/RS.Manifest/RS.Manifest.csproj | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 5ffce60009..483499044c 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -58,7 +58,7 @@ - + @@ -92,7 +92,7 @@ - + @@ -124,7 +124,8 @@ - + + <__ComposeStoreProps /> @@ -136,7 +137,7 @@ - @@ -218,7 +219,7 @@ - diff --git a/build/tools/templates/RS.Manifest/RS.Manifest.csproj b/build/tools/templates/RS.Manifest/RS.Manifest.csproj index e8c8261383..abb2d76d90 100644 --- a/build/tools/templates/RS.Manifest/RS.Manifest.csproj +++ b/build/tools/templates/RS.Manifest/RS.Manifest.csproj @@ -1,6 +1,5 @@ - From a4e87d1dd8b7dad321103ce4795f0c0a92fd9f0d Mon Sep 17 00:00:00 2001 From: = Date: Fri, 13 Oct 2017 01:19:41 -0700 Subject: [PATCH 087/115] Build runtime store installers - hosting tar.gz archives - rpm installers for generic and red hat - deb installers for debian.8, ubuntu 14.04, ubuntu 16.04, ubuntu 16.10 Create hosting targz archives --- build/RuntimeStoreInstaller.targets | 417 ++++++++++++++++++ build/repo.targets | 1 + build/tasks/AddArchiveReferences.cs | 8 +- build/tasks/GetTimestampFreeVersion.cs | 26 ++ build/tasks/ReplaceInFile.cs | 33 ++ build/tasks/RepoTasks.tasks | 2 + build/tasks/Utilities/VersionUtilities.cs | 10 +- build/tools/docker/debian.8/Dockerfile | 51 +++ build/tools/docker/rhel.7/Dockerfile | 27 ++ build/tools/docker/ubuntu.14.04/Dockerfile | 52 +++ build/tools/docker/ubuntu.16.04/Dockerfile | 48 ++ build/tools/docker/ubuntu.16.10/Dockerfile | 52 +++ .../dotnet-deb-tool-consumer.csproj | 12 + build/tools/packaging/changelog | 2 + .../packaging/hosting_debian_config.json | 35 ++ .../tools/packaging/store_debian_config.json | 34 ++ 16 files changed, 803 insertions(+), 7 deletions(-) create mode 100644 build/RuntimeStoreInstaller.targets create mode 100644 build/tasks/GetTimestampFreeVersion.cs create mode 100644 build/tasks/ReplaceInFile.cs create mode 100644 build/tools/docker/debian.8/Dockerfile create mode 100644 build/tools/docker/rhel.7/Dockerfile create mode 100644 build/tools/docker/ubuntu.14.04/Dockerfile create mode 100644 build/tools/docker/ubuntu.16.04/Dockerfile create mode 100644 build/tools/docker/ubuntu.16.10/Dockerfile create mode 100644 build/tools/dotnet-deb-tool-consumer/dotnet-deb-tool-consumer.csproj create mode 100644 build/tools/packaging/changelog create mode 100644 build/tools/packaging/hosting_debian_config.json create mode 100644 build/tools/packaging/store_debian_config.json diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets new file mode 100644 index 0000000000..625c4cf8e1 --- /dev/null +++ b/build/RuntimeStoreInstaller.targets @@ -0,0 +1,417 @@ + + + <_TemplatesDirectory>$(MSBuildThisFileDirectory)tools\templates\ + <_DockerDir>$(MSBuildThisFileDirectory)tools\docker\ + <_PackagingDir>$(MSBuildThisFileDirectory)tools\packaging\ + <_DebToolDir>$(MSBuildThisFileDirectory)tools\dotnet-deb-tool-consumer\ + <_TimestampRSSource>$(RepositoryRoot).deps\Signed\Store\ + <_TimestampFreeRSSource>$(RepositoryRoot).deps\Signed\Store-TimestampFree\ + <_CoreInstallerSource>$(RepositoryRoot).deps\CoreInstallers\ + <_WorkRoot>$(RepositoryRoot).w\ + <_WorkLayoutDir>$(_WorkRoot).l\ + <_WorkOutputDir>$(_WorkRoot).o\ + <_DockerRootDirectory>/opt/code/ + + aspnetcore-store + dotnet-hosting + $(_PackagingDir)store_debian_config.json + $(_PackagingDir)hosting_debian_config.json + + + + + + + + + + + + + + + + + + + + 2.0.3-servicing-25808-01 + http://dotnetcli.azureedge.net + $(CoreFeedPrefix)/dotnet/Runtime/$(SharedFrameworkVersion)/dotnet-runtime-$(SharedFrameworkVersion)-linux-x64.tar.gz + + + + + @(_TimestampFreeVersion) + + + + <_TimestampRSArchive Include="$(_TimestampRSSource)aspnetcore-store-$(Version)-linux-x64.tar.gz"/> + <_TimestampFreeRSArchive Include="$(_TimestampFreeRSSource)aspnetcore-store-$(TimestampFreeVersion)-linux-x64.tar.gz"/> + + + + + + + + @(_TimestampRSArchive) + @(_TimestampFreeRSArchive) + + + + + + + + + + + + + + + + + + + + + + + <_RuntimeArchive Include="$(_CoreInstallerSource)dotnet-runtime-$(SharedFrameworkVersion)-linux-x64.tar.gz" /> + + + + @(_RuntimeArchive) + $(HostingInstallerName)-$(Version)-linux-x64.tar.gz + $(HostingInstallerName)-$(TimestampFreeVersion)-linux-x64.tar.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(MaintainerName) <$(MaintainerEmail)> + + + + + + + + + rhel.7 + .NET Foundation + /opt/rh/rh-dotnet20/root/usr/lib64/dotnet/ + /usr/share/dotnet/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(Version) + + + $(SharedFrameworkVersion) + + + $(TimestampFreeVersion) + + + $(SharedFrameworkVersion) + + + + + @(HostingBundleDependencies->' -d "%(Identity) >= %(Version)"', ' ') + @(TimestampFreeHostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') + @(RHStoreDirectories->' --directories "%(FullPath)"', ' ') + @(GenericStoreDirectories->' --directories "%(FullPath)"', ' ') + + Image=$(Image);RPMVendor=$(RPMVendor) + RPMFileSuffix=rhel.7-x64.rpm;RPMInstallRoot=$(GenericInstallerInstallRoot) + RPMFileSuffix=rhel.rh.7-x64.rpm;RPMInstallRoot=$(RHInstallerInstallRoot) + + MaintainerName=@(_RSMaintainerName);MaintainerEmail=@(_RSMaintainerEmail) + $(CommonRSArguments);RPMInstallerPrefix=$(RSInstallerName);RPMRevision=@(_RSPackageRevision) + $(CommonRSArguments);RPMSummary=@(_RSInstallerSummary);RPMDescription=@(_RSDescription) + $(CommonRSArguments);RPMLicense=@(_RSLicense);RPMHomepage=@(_RSHomepage) + + $(CommonArguments);MaintainerName=@(_HostingMaintainerName);MaintainerEmail=@(_HostingMaintainerEmail) + $(CommonHostingArguments);RPMInstallerPrefix=$(HostingInstallerName);RPMRevision=@(_HostingRevision) + $(CommonHostingArguments);RPMSummary=@(_HostingSummary);RPMDescription=@(_HostingDescription) + $(CommonHostingArguments);RPMLicense=@(_HostingLicense);RPMHomepage=@(_HostingHomepage) + + $(CommonArguments);$(CommonGenericArguments);$(CommonRSArguments) + $(TimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(Version);RPMArguments=$(GenericRSArguments) + + $(CommonArguments);$(CommonGenericArguments);$(CommonRSArguments) + $(TimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(GenericRSArguments) + + $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) + $(TimestampHostingArguments);RPMVersion=$(Version);RPMArguments=$(HostingArguments) + + $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) + $(TimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingArguments) + + $(CommonArguments);$(CommonRHArguments);$(CommonRSArguments) + $(RHTimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(Version);RPMArguments=$(RHRSArguments) + + $(CommonArguments);$(CommonRHArguments);$(CommonRSArguments) + $(RHTimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(RHRSArguments) + + $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) + $(RHTimestampHostingArguments);RPMVersion=$(Version);RPMArguments=$(HostingArguments) + + $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) + $(RHTimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingArguments) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Image=$(Image);DebConfig=$(RSDebConfigFile);DebPrefix=$(RSInstallerName) + Image=$(Image);DebConfig=$(HostingDebConfigFile);DotnetVersion=$(SharedFrameworkVersion);DebPrefix=$(HostingInstallerName) + + + + + + + + + + + + + + + + + Version=$(Version);TimestampFreeVersion=$(TimestampFreeVersion) + + + + + + + + + + + + + + + + diff --git a/build/repo.targets b/build/repo.targets index 4465cf7065..c4042084bc 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -1,6 +1,7 @@ + diff --git a/build/tasks/AddArchiveReferences.cs b/build/tasks/AddArchiveReferences.cs index 8bc770b694..5da49ee059 100644 --- a/build/tasks/AddArchiveReferences.cs +++ b/build/tasks/AddArchiveReferences.cs @@ -63,9 +63,7 @@ namespace RepoTasks if (RemoveTimestamp) { - var version = new NuGetVersion(packageVersion); - var updatedVersion = new NuGetVersion(version.Version, VersionUtilities.GetNoTimestampReleaseLabel(version.Release)); - packageVersion = updatedVersion.ToNormalizedString(); + packageVersion = VersionUtilities.GetTimestampFreeVersion(packageVersion); } Log.LogMessage(MessageImportance.High, $" - Package: {packageName} Version: {packageVersion}"); @@ -99,9 +97,7 @@ namespace RepoTasks if (RemoveTimestamp) { - var version = new NuGetVersion(packageVersion); - var updatedVersion = new NuGetVersion(version.Version, VersionUtilities.GetNoTimestampReleaseLabel(version.Release)); - packageVersion = updatedVersion.ToNormalizedString(); + packageVersion = VersionUtilities.GetTimestampFreeVersion(packageVersion); } Log.LogMessage(MessageImportance.High, $" - Tool: {packageName} Version: {packageVersion}"); diff --git a/build/tasks/GetTimestampFreeVersion.cs b/build/tasks/GetTimestampFreeVersion.cs new file mode 100644 index 0000000000..f1df5bc439 --- /dev/null +++ b/build/tasks/GetTimestampFreeVersion.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class GetTimestampFreeVersion : Task + { + [Required] + public string TimestampVersion { get; set; } + + [Output] + public string TimestampFreeVersion { get; set; } + + public override bool Execute() + { + TimestampFreeVersion = VersionUtilities.GetTimestampFreeVersion(TimestampVersion); + + return true; + } + } +} diff --git a/build/tasks/ReplaceInFile.cs b/build/tasks/ReplaceInFile.cs new file mode 100644 index 0000000000..6833277787 --- /dev/null +++ b/build/tasks/ReplaceInFile.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + public class ReplaceInFile : Task + { + [Required] + public string Filename { get; set; } + + [Required] + public ITaskItem[] Items { get; set; } + + public override bool Execute() + { + var fileText = File.ReadAllText(Filename); + + foreach (var item in Items) + { + fileText = fileText.Replace(item.ItemSpec, item.GetMetadata("Replacement")); + } + + File.WriteAllText(Filename, fileText); + + return true; + } + } +} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index b390901779..e03bdc266a 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -17,4 +17,6 @@ + + diff --git a/build/tasks/Utilities/VersionUtilities.cs b/build/tasks/Utilities/VersionUtilities.cs index 9bcd94956c..042ecae378 100644 --- a/build/tasks/Utilities/VersionUtilities.cs +++ b/build/tasks/Utilities/VersionUtilities.cs @@ -2,12 +2,20 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using NuGet.Versioning; namespace RepoTasks.Utilities { public class VersionUtilities { - public static string GetNoTimestampReleaseLabel(string releaseLabel) + public static string GetTimestampFreeVersion(string packageVersion) + { + var version = new NuGetVersion(packageVersion); + var updatedVersion = new NuGetVersion(version.Version, GetTimestampFreeReleaseLabel(version.Release)); + return updatedVersion.ToNormalizedString(); + } + + public static string GetTimestampFreeReleaseLabel(string releaseLabel) { if (releaseLabel.StartsWith("rtm-", StringComparison.OrdinalIgnoreCase)) { diff --git a/build/tools/docker/debian.8/Dockerfile b/build/tools/docker/debian.8/Dockerfile new file mode 100644 index 0000000000..0c032b9d7f --- /dev/null +++ b/build/tools/docker/debian.8/Dockerfile @@ -0,0 +1,51 @@ +# +# Copyright (c) .NET Foundation. All rights reserved. +# Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM debian:jessie + +# Misc Dependencies for build +RUN apt-get update && \ + apt-get -qqy install \ + curl \ + unzip \ + gettext \ + sudo \ + libunwind8 \ + libkrb5-3 \ + libicu52 \ + liblttng-ust0 \ + libssl1.0.0 \ + zlib1g \ + libuuid1 \ + debhelper \ + build-essential \ + devscripts \ + git \ + cmake \ + clang-3.5 \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +# Setup User to match Host User, and give superuser permissions +ARG USER_ID=0 +RUN useradd -m code_executor -u ${USER_ID} -g sudo +RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# With the User Change, we need to change permissions on these directories +RUN chmod -R a+rwx /usr/local +RUN chmod -R a+rwx /home +RUN chmod -R 755 /usr/lib/sudo + +# Set user to the one we just created +USER ${USER_ID} + +# Set working directory +WORKDIR /opt/code diff --git a/build/tools/docker/rhel.7/Dockerfile b/build/tools/docker/rhel.7/Dockerfile new file mode 100644 index 0000000000..73247d2809 --- /dev/null +++ b/build/tools/docker/rhel.7/Dockerfile @@ -0,0 +1,27 @@ +# +# Copyright (c) .NET Foundation. All rights reserved. +# Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM microsoft/dotnet-buildtools-prereqs:rhel-7-rpmpkg-c982313-20174116044113 + +# Install from sudo main package TODO This package needs to be mirrored +RUN yum install -y https://www.sudo.ws/sudo/dist/packages/RHEL/7/sudo-1.8.20-3.el7.x86_64.rpm \ + && yum clean all + +# Setup User to match Host User, and give superuser permissions +ARG USER_ID=0 +RUN useradd -m code_executor -u ${USER_ID} -g root +RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# With the User Change, we need to change permssions on these directories +RUN chmod -R a+rwx /usr/local +RUN chmod -R a+rwx /home +RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo + +# Set user to the one we just created +USER ${USER_ID} + +# Set working directory +WORKDIR /opt/code diff --git a/build/tools/docker/ubuntu.14.04/Dockerfile b/build/tools/docker/ubuntu.14.04/Dockerfile new file mode 100644 index 0000000000..f039506971 --- /dev/null +++ b/build/tools/docker/ubuntu.14.04/Dockerfile @@ -0,0 +1,52 @@ +# +# Copyright (c) .NET Foundation. All rights reserved. +# Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM ubuntu:14.04 + +# Misc Dependencies for build +RUN apt-get update && \ + apt-get -qqy install \ + curl \ + unzip \ + gettext \ + sudo \ + libunwind8 \ + libkrb5-3 \ + libicu52 \ + liblttng-ust0 \ + libssl1.0.0 \ + zlib1g \ + libuuid1 \ + debhelper \ + build-essential \ + devscripts \ + git \ + cmake \ + clang-3.5 \ + lldb-3.6 \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +# Setup User to match Host User, and give superuser permissions +ARG USER_ID=0 +RUN useradd -m code_executor -u ${USER_ID} -g sudo +RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# With the User Change, we need to change permissions on these directories +RUN chmod -R a+rwx /usr/local +RUN chmod -R a+rwx /home +RUN chmod -R 755 /usr/lib/sudo + +# Set user to the one we just created +USER ${USER_ID} + +# Set working directory +WORKDIR /opt/code diff --git a/build/tools/docker/ubuntu.16.04/Dockerfile b/build/tools/docker/ubuntu.16.04/Dockerfile new file mode 100644 index 0000000000..79e76f0703 --- /dev/null +++ b/build/tools/docker/ubuntu.16.04/Dockerfile @@ -0,0 +1,48 @@ +# +# Copyright (c) .NET Foundation. All rights reserved. +# Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +# + +FROM ubuntu:16.04 + +# Install the base toolchain we need to build anything (clang, cmake, make and the like) +# this does not include libraries that we need to compile different projects, we'd like +# them in a different layer. +RUN apt-get update && \ + apt-get install -y cmake \ + make \ + llvm-3.5 \ + clang-3.5 \ + git \ + curl \ + tar \ + sudo \ + debhelper \ + build-essential \ + devscripts \ + libunwind8 \ + libkrb5-3 \ + libicu55 \ + liblttng-ust0 \ + libssl1.0.0 \ + zlib1g \ + libuuid1 \ + liblldb-3.6 \ + wget && \ + apt-get clean + +# Setup User to match Host User, and give superuser permissions +ARG USER_ID=0 +RUN useradd -m code_executor -u ${USER_ID} -g sudo +RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# With the User Change, we need to change permissions on these directories +RUN chmod -R a+rwx /usr/local +RUN chmod -R a+rwx /home +RUN chmod -R 755 /usr/lib/sudo + +# Set user to the one we just created +USER ${USER_ID} + +# Set working directory +WORKDIR /opt/code \ No newline at end of file diff --git a/build/tools/docker/ubuntu.16.10/Dockerfile b/build/tools/docker/ubuntu.16.10/Dockerfile new file mode 100644 index 0000000000..c57bd4c3ff --- /dev/null +++ b/build/tools/docker/ubuntu.16.10/Dockerfile @@ -0,0 +1,52 @@ +# +# Copyright (c) .NET Foundation. All rights reserved. +# Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM ubuntu:16.10 + +# Misc Dependencies for build +RUN apt-get update && \ + apt-get -qqy install \ + curl \ + unzip \ + gettext \ + sudo \ + libunwind8 \ + libkrb5-3 \ + libicu57 \ + liblttng-ust0 \ + libssl1.0.0 \ + zlib1g \ + libuuid1 \ + liblldb-3.5 \ + debhelper \ + build-essential \ + devscripts \ + git \ + cmake \ + clang-3.5 \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +# Setup User to match Host User, and give superuser permissions +ARG USER_ID=0 +RUN useradd -m code_executor -u ${USER_ID} -g sudo +RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# With the User Change, we need to change permissions on these directories +RUN chmod -R a+rwx /usr/local +RUN chmod -R a+rwx /home +RUN chmod -R 755 /usr/lib/sudo + +# Set user to the one we just created +USER ${USER_ID} + +# Set working directory +WORKDIR /opt/code \ No newline at end of file diff --git a/build/tools/dotnet-deb-tool-consumer/dotnet-deb-tool-consumer.csproj b/build/tools/dotnet-deb-tool-consumer/dotnet-deb-tool-consumer.csproj new file mode 100644 index 0000000000..659448ee0e --- /dev/null +++ b/build/tools/dotnet-deb-tool-consumer/dotnet-deb-tool-consumer.csproj @@ -0,0 +1,12 @@ + + + + netcoreapp1.0 + $(RestoreSources);https://dotnet.myget.org/F/cli-deps/api/v3/index.json + + + + + + + diff --git a/build/tools/packaging/changelog b/build/tools/packaging/changelog new file mode 100644 index 0000000000..f28d88ed10 --- /dev/null +++ b/build/tools/packaging/changelog @@ -0,0 +1,2 @@ +* DATE MAINTAINER_NAME - PACKAGE_VERSION-PACKAGE_REVISION +- \ No newline at end of file diff --git a/build/tools/packaging/hosting_debian_config.json b/build/tools/packaging/hosting_debian_config.json new file mode 100644 index 0000000000..cfdd83e74a --- /dev/null +++ b/build/tools/packaging/hosting_debian_config.json @@ -0,0 +1,35 @@ +{ + "maintainer_name": "Microsoft", + "maintainer_email": "nugetaspnet@microsoft.com", + + "package_name": "dotnet-hosting", + "install_root": "/usr/share/dotnet", + + "short_description": "Microsoft .NET Core DEB_VERSION Linux Server Hosting", + "long_description": "Microsoft .NET Core DEB_VERSION Linux Server Hosting enables hosting of ASP.NET Core applications and contains the ASP.NET Core Runtime Package Store and .NET Core Runtime. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub (https://github.com/aspnet/home). We happily accept issues and PRs.", + "homepage": "https://www.asp.net/", + + "release":{ + "package_version":"0.0.0.0", + "package_revision":"1", + "urgency" : "low", + "changelog_message" : "" + }, + + "control": { + "priority":"standard", + "section":"devel", + "architecture":"any" + }, + + "copyright": "Microsoft", + "license": { + "type": "Apache-2.0", + "full_text": "Copyright (c) .NET Foundation. All rights reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use\nthese files except in compliance with the License. You may obtain a copy of the\nLicense at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software distributed\nunder the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\nCONDITIONS OF ANY KIND, either express or implied. See the License for the\nspecific language governing permissions and limitations under the License." + }, + + "debian_dependencies": { + "dotnet-runtime-DOTNET_VERSION": {}, + "aspnetcore-store-DEB_VERSION": {} + } +} \ No newline at end of file diff --git a/build/tools/packaging/store_debian_config.json b/build/tools/packaging/store_debian_config.json new file mode 100644 index 0000000000..70dffe42a9 --- /dev/null +++ b/build/tools/packaging/store_debian_config.json @@ -0,0 +1,34 @@ +{ + "maintainer_name": "Microsoft", + "maintainer_email": "nugetaspnet@microsoft.com", + + "package_name": "aspnetcore-store", + "install_root": "/usr/share/dotnet", + + "short_description": "Microsoft ASP.NET Core DEB_VERSION Runtime Package Store", + "long_description": "Runtime package store for Microsoft ASP.NET Core. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub (https://github.com/aspnet/home). We happily accept issues and PRs.", + "homepage": "https://www.asp.net/", + + "release":{ + "package_version":"0.0.0.0", + "package_revision":"1", + "urgency" : "low", + "changelog_message" : "" + }, + + "control": { + "priority":"standard", + "section":"devel", + "architecture":"any" + }, + + "copyright": "Microsoft", + "license": { + "type": "Apache-2.0", + "full_text": "Copyright (c) .NET Foundation. All rights reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use\nthese files except in compliance with the License. You may obtain a copy of the\nLicense at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software distributed\nunder the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\nCONDITIONS OF ANY KIND, either express or implied. See the License for the\nspecific language governing permissions and limitations under the License." + }, + + "debian_dependencies": { + "aspnetcore-store-2.0.0": {} + } +} \ No newline at end of file From 288199ce72ef940125c0d3b6400c5f84e33d7d6a Mon Sep 17 00:00:00 2001 From: = Date: Fri, 13 Oct 2017 17:06:14 -0700 Subject: [PATCH 088/115] Remove temporary pinning --- build/RuntimeStoreInstaller.targets | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 625c4cf8e1..717fd77a90 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -35,8 +35,6 @@ - - 2.0.3-servicing-25808-01 http://dotnetcli.azureedge.net $(CoreFeedPrefix)/dotnet/Runtime/$(SharedFrameworkVersion)/dotnet-runtime-$(SharedFrameworkVersion)-linux-x64.tar.gz From 6e077fb60aa90b9b331804b2af5d44dd2c78053a Mon Sep 17 00:00:00 2001 From: = Date: Fri, 13 Oct 2017 17:46:01 -0700 Subject: [PATCH 089/115] Fix typos and cleanup --- build/RuntimeStoreInstaller.targets | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 717fd77a90..29a431df51 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -230,12 +230,12 @@ - + $(Version) - - + + $(SharedFrameworkVersion) - + $(TimestampFreeVersion) @@ -245,8 +245,8 @@ - @(HostingBundleDependencies->' -d "%(Identity) >= %(Version)"', ' ') - @(TimestampFreeHostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') + @(HostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') + @(TimestampFreeHostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') @(RHStoreDirectories->' --directories "%(FullPath)"', ' ') @(GenericStoreDirectories->' --directories "%(FullPath)"', ' ') @@ -274,7 +274,7 @@ $(TimestampHostingArguments);RPMVersion=$(Version);RPMArguments=$(HostingArguments) $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) - $(TimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingArguments) + $(TimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingFPMArguments) $(CommonArguments);$(CommonRHArguments);$(CommonRSArguments) $(RHTimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(Version);RPMArguments=$(RHRSArguments) @@ -286,7 +286,7 @@ $(RHTimestampHostingArguments);RPMVersion=$(Version);RPMArguments=$(HostingArguments) $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) - $(RHTimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingArguments) + $(RHTimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingFPMArguments) From d28e9fd8bc4a596c7da668c7f40410d94c997734 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Fri, 13 Oct 2017 18:08:10 -0700 Subject: [PATCH 090/115] Update build tools to 2.0.2-rc1-16001 --- build/dependencies.props | 2 +- korebuild-lock.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index d3d5ad054c..5649774045 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -127,7 +127,7 @@ https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json - 2.0.2-rc1-15526 + 2.0.2-rc1-16001 diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 7f216dfb7a..dc4b83bd54 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-rc1-15526 -commithash:436afc3dc08f5e278431db807866cc5f032f4d7b +version:2.0.2-rc1-16001 +commithash:9f162fb6eb17e2e622b7bc5739f26506117a96b8 From fe031e56553086bf1d59354339de0b7fa759da45 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 14 Oct 2017 03:36:35 -0700 Subject: [PATCH 091/115] Move core installer download to its own target --- build/RuntimeStoreInstaller.targets | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 29a431df51..32672d2760 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -61,7 +61,9 @@ @(_TimestampRSArchive) @(_TimestampFreeRSArchive) + + @@ -80,7 +82,7 @@ - + <_RuntimeArchive Include="$(_CoreInstallerSource)dotnet-runtime-$(SharedFrameworkVersion)-linux-x64.tar.gz" /> From c255c7f276e76f896bd7cc693606927bb7f3e1f1 Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Mon, 16 Oct 2017 09:54:09 -0700 Subject: [PATCH 092/115] Put ApplicationInsights.HostingStartup back into metapackage (#606) --- build/artifacts.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/artifacts.props b/build/artifacts.props index fe06653938..0564ae2f0b 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -13,7 +13,7 @@ - + From a636fb0b74bc057e7d0015c7906f6b66af76dcb9 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 16 Oct 2017 10:14:43 -0700 Subject: [PATCH 093/115] Update to build tools 2.0.2-rc1-16006 --- build/dependencies.props | 2 +- korebuild-lock.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index 5649774045..5b4df58450 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -127,7 +127,7 @@ https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json - 2.0.2-rc1-16001 + 2.0.2-rc1-16006 diff --git a/korebuild-lock.txt b/korebuild-lock.txt index dc4b83bd54..fb75ea4933 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-rc1-16001 -commithash:9f162fb6eb17e2e622b7bc5739f26506117a96b8 +version:2.0.2-rc1-16006 +commithash:a0f88f3f21bc337cbd0b3adac482a387d91aca1f From 0585274d325b11e20b9c3818893ef4206c4b3620 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 16 Oct 2017 10:18:23 -0700 Subject: [PATCH 094/115] Update kestrel submodule and remove the Microsoft.AspNetCore.Internal.CoreFxLab.Sources package from external dependencies list (#607) --- build/dependencies.props | 9 --------- modules/KestrelHttpServer | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index 5b4df58450..46f61bc3f3 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -144,15 +144,6 @@ - - - https://dotnet.myget.org/F/aspnetcore-release/api/v3/index.json - - - - - - https://api.nuget.org/v3/index.json diff --git a/modules/KestrelHttpServer b/modules/KestrelHttpServer index be4e401aa6..4afa9c8ca3 160000 --- a/modules/KestrelHttpServer +++ b/modules/KestrelHttpServer @@ -1 +1 @@ -Subproject commit be4e401aa69369b454e448d701c9c3fb0e48113b +Subproject commit 4afa9c8ca3c8aea987afb8e079f423342385f678 From 2f78d058e71e0f210c4e40fe7193c30822b82fb7 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 16 Oct 2017 11:31:15 -0700 Subject: [PATCH 095/115] Update build tools to 2.0.2-rc1-16007 --- build/dependencies.props | 2 +- korebuild-lock.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/dependencies.props b/build/dependencies.props index 46f61bc3f3..e7dd7c268d 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -127,7 +127,7 @@ https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json - 2.0.2-rc1-16006 + 2.0.2-rc1-16007 diff --git a/korebuild-lock.txt b/korebuild-lock.txt index fb75ea4933..6471fca0b4 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-rc1-16006 -commithash:a0f88f3f21bc337cbd0b3adac482a387d91aca1f +version:2.0.2-rc1-16007 +commithash:bccf097cd0fceb185b7bf6aa8981191304cea9a7 From 44520119e3769d8b9f0f3b2e2a0cc2b142cbf5e4 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 16 Oct 2017 11:47:14 -0700 Subject: [PATCH 096/115] Set IsFinalBuild=true on the non-timestamped version of templating packages --- build/Templating.targets | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/Templating.targets b/build/Templating.targets index 31e026120b..d7592be916 100644 --- a/build/Templating.targets +++ b/build/Templating.targets @@ -20,8 +20,7 @@ $(TemplateProjCommmonProperties); DotNetPackageVersionPropsPath=$(GeneratedNoTimestampPackageVersionPropsPath); - BuildNumber=$(BuildNumber)-final; - VersionMetadata=notimestamp; + IsFinalBuild=true; From 83e22851c527e89527f0fc506eba0180f05ba117 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 16 Oct 2017 12:53:36 -0700 Subject: [PATCH 097/115] Remove dummy reference project from the output of the runtime store --- build/tools/templates/RS.References/RS.References.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/tools/templates/RS.References/RS.References.csproj b/build/tools/templates/RS.References/RS.References.csproj index 91977d96d5..bcbed2fd57 100644 --- a/build/tools/templates/RS.References/RS.References.csproj +++ b/build/tools/templates/RS.References/RS.References.csproj @@ -13,6 +13,9 @@ https://api.nuget.org/v3/index.json; netcoreapp2.0 + false + false + false From 1f6db0e3ad030da615e7f96e1d1479031a71f306 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 17 Oct 2017 12:57:56 -0700 Subject: [PATCH 098/115] Add Microsoft.NETCore.Platforms to the list of external dependencies --- build/dependencies.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/dependencies.props b/build/dependencies.props index e7dd7c268d..4700a67867 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -29,10 +29,12 @@ https://dotnet.myget.org/F/dotnet-core/api/v3/index.json 2.0.2-servicing-25728-02 + 2.0.0 + From 4bb84cf6d0d9e2761f4a5094a8f254a8579fba65 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 17 Oct 2017 15:46:53 -0700 Subject: [PATCH 099/115] Ensure the deb/rpm installers use the same dotnet-core version - Remove unnecessary tasks and scripts - Ensure the KOREBUILD_DOTNET_* environment variables are preserved in the docker build context - Other MSBuild cleanup of the targets --- build/PackageArchive.targets | 4 +- build/RuntimeStore.targets | 18 +- build/RuntimeStoreInstaller.targets | 168 +++++++++--------- build/dependencies.props | 1 + build/tasks/GetTimestampFreeVersion.cs | 26 --- build/tasks/RepoTasks.tasks | 1 - .../scripts/GetSharedFrameworkVersion.ps1 | 6 - .../scripts/GetSharedFrameworkVersion.sh | 2 - .../Microsoft.AspNetCore.All.csproj | 2 +- 9 files changed, 88 insertions(+), 140 deletions(-) delete mode 100644 build/tasks/GetTimestampFreeVersion.cs delete mode 100644 build/tools/scripts/GetSharedFrameworkVersion.ps1 delete mode 100755 build/tools/scripts/GetSharedFrameworkVersion.sh diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index 0465a73e55..8ec8c5aa2a 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -4,7 +4,7 @@ <_WorkRoot>$(RepositoryRoot).w\ - + @@ -37,7 +37,7 @@ + Properties="RestorePackagesPath=$(FallbackStagingDir);RuntimeFrameworkVersion=$(MicrosoftNETCoreApp20PackageVersion);DotNetRestoreSources=$(DotNetRestoreSources);AspNetUniverseBuildOffline=true" /> diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index 483499044c..c369b30a0a 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -122,7 +122,7 @@ - + @@ -133,7 +133,7 @@ <_ComposeStoreProps>$(_ComposeStoreProps);ComposeDir=$(_RuntimeStoreOutputDirectory) <_ComposeStoreProps>$(_ComposeStoreProps);ComposeWorkingDir=$(_RuntimeStoreWorkDirectory) <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeIdentifier=$(RuntimeStoreRID) - <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeFrameworkVersion=$(SharedFrameworkVersion) + <_ComposeStoreProps>$(_ComposeStoreProps);RuntimeFrameworkVersion=$(MicrosoftNETCoreApp20PackageVersion) @@ -142,17 +142,7 @@ Properties="$(_ComposeStoreProps)" /> - - - - - - - - - - - + + Properties="DepsOutputPath=$(_DepsOutputDirectory);HostingStartupPackageName=%(HostingStartupArtifacts.PackageId);HostingStartupPackageVersion=%(HostingStartupArtifacts.Version);RuntimeFrameworkVersion=$(MicrosoftNETCoreApp20PackageVersion)" /> diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 32672d2760..3159278722 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -16,57 +16,43 @@ dotnet-hosting $(_PackagingDir)store_debian_config.json $(_PackagingDir)hosting_debian_config.json + + $(KOREBUILD_DOTNET_FEED_UNCACHED) + https://dotnetcli.blob.core.windows.net/dotnet - + - + - - - - - - http://dotnetcli.azureedge.net - $(CoreFeedPrefix)/dotnet/Runtime/$(SharedFrameworkVersion)/dotnet-runtime-$(SharedFrameworkVersion)-linux-x64.tar.gz - - - - - @(_TimestampFreeVersion) + $(CoreFeedPrefix)/dotnet/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz + $(RuntimeTargzLink)$(KOREBUILD_DOTNET_FEED_CREDENTIAL) - - <_TimestampRSArchive Include="$(_TimestampRSSource)aspnetcore-store-$(Version)-linux-x64.tar.gz"/> - <_TimestampFreeRSArchive Include="$(_TimestampFreeRSSource)aspnetcore-store-$(TimestampFreeVersion)-linux-x64.tar.gz"/> - - - - - - - @(_TimestampRSArchive) - @(_TimestampFreeRSArchive) + $(_TimestampRSSource)aspnetcore-store-$(PackageVersion)-linux-x64.tar.gz + $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)-linux-x64.tar.gz + + + - + @@ -83,14 +69,10 @@ - - <_RuntimeArchive Include="$(_CoreInstallerSource)dotnet-runtime-$(SharedFrameworkVersion)-linux-x64.tar.gz" /> - - - @(_RuntimeArchive) + $(_CoreInstallerSource)dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz $(HostingInstallerName)-$(Version)-linux-x64.tar.gz - $(HostingInstallerName)-$(TimestampFreeVersion)-linux-x64.tar.gz + $(HostingInstallerName)-$(PackageVersionNoTimestamp)-linux-x64.tar.gz @@ -151,6 +133,11 @@ @@ -232,17 +219,17 @@ - - $(Version) + + $(PackageVersion) - - $(SharedFrameworkVersion) + + $(MicrosoftNETCoreApp20PackageVersion) - - $(TimestampFreeVersion) + + $(PackageVersionNoTimestamp) - - $(SharedFrameworkVersion) + + $(MicrosoftNETCoreApp20PackageVersion) @@ -251,7 +238,7 @@ @(TimestampFreeHostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') @(RHStoreDirectories->' --directories "%(FullPath)"', ' ') @(GenericStoreDirectories->' --directories "%(FullPath)"', ' ') - + Image=$(Image);RPMVendor=$(RPMVendor) RPMFileSuffix=rhel.7-x64.rpm;RPMInstallRoot=$(GenericInstallerInstallRoot) RPMFileSuffix=rhel.rh.7-x64.rpm;RPMInstallRoot=$(RHInstallerInstallRoot) @@ -260,35 +247,35 @@ $(CommonRSArguments);RPMInstallerPrefix=$(RSInstallerName);RPMRevision=@(_RSPackageRevision) $(CommonRSArguments);RPMSummary=@(_RSInstallerSummary);RPMDescription=@(_RSDescription) $(CommonRSArguments);RPMLicense=@(_RSLicense);RPMHomepage=@(_RSHomepage) - + $(CommonArguments);MaintainerName=@(_HostingMaintainerName);MaintainerEmail=@(_HostingMaintainerEmail) $(CommonHostingArguments);RPMInstallerPrefix=$(HostingInstallerName);RPMRevision=@(_HostingRevision) $(CommonHostingArguments);RPMSummary=@(_HostingSummary);RPMDescription=@(_HostingDescription) $(CommonHostingArguments);RPMLicense=@(_HostingLicense);RPMHomepage=@(_HostingHomepage) $(CommonArguments);$(CommonGenericArguments);$(CommonRSArguments) - $(TimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(Version);RPMArguments=$(GenericRSArguments) - + $(TimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(PackageVersion);RPMArguments=$(GenericRSArguments) + $(CommonArguments);$(CommonGenericArguments);$(CommonRSArguments) - $(TimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(GenericRSArguments) - + $(TimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(GenericRSArguments) + $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) - $(TimestampHostingArguments);RPMVersion=$(Version);RPMArguments=$(HostingArguments) - + $(TimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingArguments) + $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) - $(TimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingFPMArguments) + $(TimestampFreeHostingArguments);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(TimestampFreeHostingFPMArguments) $(CommonArguments);$(CommonRHArguments);$(CommonRSArguments) - $(RHTimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(Version);RPMArguments=$(RHRSArguments) - + $(RHTimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(PackageVersion);RPMArguments=$(RHRSArguments) + $(CommonArguments);$(CommonRHArguments);$(CommonRSArguments) - $(RHTimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(RHRSArguments) - + $(RHTimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(RHRSArguments) + $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) - $(RHTimestampHostingArguments);RPMVersion=$(Version);RPMArguments=$(HostingArguments) - + $(RHTimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingArguments) + $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) - $(RHTimestampFreeHostingArguments);RPMVersion=$(TimestampFreeVersion);RPMArguments=$(TimestampFreeHostingFPMArguments) + $(RHTimestampFreeHostingArguments);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(TimestampFreeHostingFPMArguments) @@ -299,7 +286,7 @@ - + @@ -345,6 +332,11 @@ -e DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true -e INSTALLER_NAME=$(DebPrefix)-$(DebVersion) -e INSTALLER_VERSION=$(DebVersion) + -e 'KOREBUILD_DOTNET_VERSION=$(KOREBUILD_DOTNET_VERSION)' + -e 'KOREBUILD_DOTNET_SHARED_RUNTIME_VERSION=$(KOREBUILD_DOTNET_SHARED_RUNTIME_VERSION)' + -e 'KOREBUILD_DOTNET_FEED_CDN=$(KOREBUILD_DOTNET_FEED_CDN)' + -e 'KOREBUILD_DOTNET_FEED_UNCACHED=$(KOREBUILD_DOTNET_FEED_UNCACHED)' + -e 'KOREBUILD_DOTNET_FEED_CREDENTIAL=$(KOREBUILD_DOTNET_FEED_CREDENTIAL)' docker-image-$(Image) ./build.sh /t:RunDebTool" ContinueOnError="WarnAndContinue" /> @@ -353,9 +345,9 @@ - + - + Image=$(Image);DebConfig=$(RSDebConfigFile);DebPrefix=$(RSInstallerName) - Image=$(Image);DebConfig=$(HostingDebConfigFile);DotnetVersion=$(SharedFrameworkVersion);DebPrefix=$(HostingInstallerName) + Image=$(Image);DebConfig=$(HostingDebConfigFile);DotnetVersion=$(MicrosoftNETCoreApp20PackageVersion);DebPrefix=$(HostingInstallerName) - + - - - + - - + + - Version=$(Version);TimestampFreeVersion=$(TimestampFreeVersion) + Version=$(Version);TimestampFreeVersion=$(PackageVersionNoTimestamp) - - - - + + + + @@ -410,7 +402,7 @@ - diff --git a/build/dependencies.props b/build/dependencies.props index 4700a67867..b58e80713a 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -30,6 +30,7 @@ https://dotnet.myget.org/F/dotnet-core/api/v3/index.json 2.0.2-servicing-25728-02 2.0.0 + $(CoreSetupPackageVersion) diff --git a/build/tasks/GetTimestampFreeVersion.cs b/build/tasks/GetTimestampFreeVersion.cs deleted file mode 100644 index f1df5bc439..0000000000 --- a/build/tasks/GetTimestampFreeVersion.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Linq; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using RepoTasks.Utilities; - -namespace RepoTasks -{ - public class GetTimestampFreeVersion : Task - { - [Required] - public string TimestampVersion { get; set; } - - [Output] - public string TimestampFreeVersion { get; set; } - - public override bool Execute() - { - TimestampFreeVersion = VersionUtilities.GetTimestampFreeVersion(TimestampVersion); - - return true; - } - } -} diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index e03bdc266a..5410effb57 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -17,6 +17,5 @@ - diff --git a/build/tools/scripts/GetSharedFrameworkVersion.ps1 b/build/tools/scripts/GetSharedFrameworkVersion.ps1 deleted file mode 100644 index 37772625fe..0000000000 --- a/build/tools/scripts/GetSharedFrameworkVersion.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -$infoOutput = dotnet --info -$versions = $infoOutput | Select-String -Pattern "version" -$FXVersionRaw = $versions | Select-Object -Last 1 -$FXVersionString = $FXVersionRaw.ToString() -$FXVersion = $FXVersionString.SubString($FXVersionString.IndexOf(':') + 1).Trim() -Write-Host $FXVersion diff --git a/build/tools/scripts/GetSharedFrameworkVersion.sh b/build/tools/scripts/GetSharedFrameworkVersion.sh deleted file mode 100755 index dee9ea2db8..0000000000 --- a/build/tools/scripts/GetSharedFrameworkVersion.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -dotnet --info | grep -i version | tail -1 | cut -f 2 -d ":" | tr -d ' ' \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj index d7155f1cc3..c3d5b59912 100644 --- a/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj +++ b/src/Microsoft.AspNetCore.All/Microsoft.AspNetCore.All.csproj @@ -19,7 +19,7 @@ aspnetcore Microsoft.AspNetCore.All false - $(CoreSetupPackageVersion) + $(MicrosoftNETCoreApp20PackageVersion) From b191fc8feff02299e34f109ecf30d96a5949fda7 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 17 Oct 2017 16:32:12 -0700 Subject: [PATCH 100/115] Workaround issues with MSBuild variable subtitution and bash --- build/RuntimeStoreInstaller.targets | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 3159278722..bbeb93a645 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -32,8 +32,7 @@ - $(CoreFeedPrefix)/dotnet/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz - $(RuntimeTargzLink)$(KOREBUILD_DOTNET_FEED_CREDENTIAL) + $(CoreFeedPrefix)/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz @@ -52,7 +51,12 @@ - + + @@ -133,11 +137,6 @@ + + + From 3a563b15fa74d4fb9d818fac7fcdac3ae07c4396 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 17 Oct 2017 17:43:16 -0700 Subject: [PATCH 101/115] Fix the BuildFallbackArchive targets to restore from the correct locations --- build/PackageArchive.targets | 66 +++++++++----------- build/tools/templates/Archive/Archive.csproj | 4 +- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index 8ec8c5aa2a..0d9af52cb4 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -2,10 +2,28 @@ <_TemplatesDirectory>$(MSBuildThisFileDirectory)tools\templates\ <_WorkRoot>$(RepositoryRoot).w\ + $(RepositoryRoot).deps\Signed\Packages\ + $(RepositoryRoot).deps\Signed\Packages-NoTimeStamp\ + nuGetPackagesArchive-$(PackageVersion) + nuGetPackagesArchive-$(PackageVersionNoTimestamp) + + + + + + + + - + @@ -28,8 +46,18 @@ $(_WorkRoot)obj\$(OutputPackageName) $(RepositoryRoot)artifacts\$(OutputPackageName).lzma + $(_WorkRoot)restoresources.$(OutputPackageName).props + + <_FallbackArchiveRestoreSources Include="$(MetapackageRestoreSource)" Condition="Exists($(MetapackageRestoreSource))" /> + <_FallbackArchiveRestoreSources Include="$(_DependencyMirrorDirectory)" Condition="Exists($(_DependencyMirrorDirectory))" /> + + + + @@ -37,43 +65,9 @@ + Properties="RestorePackagesPath=$(FallbackStagingDir);RuntimeFrameworkVersion=$(MicrosoftNETCoreApp20PackageVersion);DotNetRestoreSourcePropsPath=$(GeneratedFallbackRestoreSourcesPropsPath);AspNetUniverseBuildOffline=true" /> - - - - - $(RepositoryRoot).deps\Signed\Packages\ - $(RepositoryRoot).deps\Signed\Packages-NoTimeStamp\ - - - - - - - - - @(TimestampMetapackageVersion->Metadata('Filename')) - $(TimeStampVersion.TrimStart('Microsoft.AspNetCore.All')) - @(TimestampFreeMetapackageVersion->Metadata('Filename')) - $(TimeStampFreeVersion.TrimStart('Microsoft.AspNetCore.All')) - nuGetPackagesArchive-$(TimestampVersion) - nuGetPackagesArchive-$(TimestampFreeVersion) - - - - - - - - diff --git a/build/tools/templates/Archive/Archive.csproj b/build/tools/templates/Archive/Archive.csproj index baa0ab2068..dec1f27375 100644 --- a/build/tools/templates/Archive/Archive.csproj +++ b/build/tools/templates/Archive/Archive.csproj @@ -1,5 +1,7 @@ + + netcoreapp2.0 false @@ -18,4 +20,4 @@ - \ No newline at end of file + From d5df2acd455c603ee78b21b047c1e4dab5d8388b Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 17 Oct 2017 17:56:31 -0700 Subject: [PATCH 102/115] Put installers into artifacts/installers/ and the lzma into artifacts/lzma/ --- build/PackageArchive.targets | 7 ++++--- build/RuntimeStoreInstaller.targets | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index 0d9af52cb4..e7819859d8 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -45,7 +45,8 @@ $(_WorkRoot)obj\$(OutputPackageName) - $(RepositoryRoot)artifacts\$(OutputPackageName).lzma + $(ArtifactsDir)lzma\ + $(FallbackOutputDir)$(OutputPackageName).lzma $(_WorkRoot)restoresources.$(OutputPackageName).props @@ -59,7 +60,7 @@ OutputPath="$(GeneratedFallbackRestoreSourcesPropsPath)" /> - + - + diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index bbeb93a645..8e8ac07495 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -11,6 +11,7 @@ <_WorkLayoutDir>$(_WorkRoot).l\ <_WorkOutputDir>$(_WorkRoot).o\ <_DockerRootDirectory>/opt/code/ + $(ArtifactsDir)installers\ aspnetcore-store dotnet-hosting @@ -69,7 +70,7 @@ - + @@ -143,7 +144,7 @@ -s dir -t rpm -n $(RPMInstallerPrefix)-$(RPMVersion) - -p $(_DockerRootDirectory)artifacts/$(RPMInstallerPrefix)-$(RPMVersion)-$(RPMFileSuffix) + -p $(_DockerRootDirectory)artifacts/installers/$(RPMInstallerPrefix)-$(RPMVersion)-$(RPMFileSuffix) -v $(RPMVersion) --iteration $(RPMRevision) -a amd64 @@ -355,7 +356,7 @@ Date: Wed, 18 Oct 2017 07:30:44 -0700 Subject: [PATCH 103/115] Set filepath properties in global build context, not the local targets --- build/RuntimeStoreInstaller.targets | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 8e8ac07495..1d5c244277 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -20,6 +20,10 @@ $(KOREBUILD_DOTNET_FEED_UNCACHED) https://dotnetcli.blob.core.windows.net/dotnet + + $(CoreFeedPrefix)/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz + $(_TimestampRSSource)aspnetcore-store-$(PackageVersion)-linux-x64.tar.gz + $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)-linux-x64.tar.gz @@ -32,15 +36,6 @@ - - $(CoreFeedPrefix)/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz - - - - $(_TimestampRSSource)aspnetcore-store-$(PackageVersion)-linux-x64.tar.gz - $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)-linux-x64.tar.gz - - @@ -307,7 +302,7 @@ - + Date: Wed, 18 Oct 2017 07:35:34 -0700 Subject: [PATCH 104/115] Make the outputpath for the installers --- build/RuntimeStoreInstaller.targets | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 1d5c244277..f615133e54 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -29,6 +29,8 @@ + + From 65fcbdb8c814975e9aa0f570ac113ca591d3f278 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 18 Oct 2017 10:14:50 -0700 Subject: [PATCH 105/115] Installer generation updates --- build/RuntimeStoreInstaller.targets | 35 ++++++++++++++++--- .../packaging/hosting_debian_config.json | 2 +- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index f615133e54..8cb848ca85 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -11,7 +11,7 @@ <_WorkLayoutDir>$(_WorkRoot).l\ <_WorkOutputDir>$(_WorkRoot).o\ <_DockerRootDirectory>/opt/code/ - $(ArtifactsDir)installers\ + <_InstallersOutputDir>$(ArtifactsDir)installers\ aspnetcore-store dotnet-hosting @@ -26,10 +26,10 @@ $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)-linux-x64.tar.gz - + - + @@ -67,7 +67,7 @@ - + @@ -127,8 +127,10 @@ + $(MaintainerName) <$(MaintainerEmail)> + $(RPMSummary.Replace('DEB_VERSION','$(RPMVersion)')) @@ -353,7 +355,7 @@ + + + + + + + + + + + + + diff --git a/build/tools/packaging/hosting_debian_config.json b/build/tools/packaging/hosting_debian_config.json index cfdd83e74a..92c825e901 100644 --- a/build/tools/packaging/hosting_debian_config.json +++ b/build/tools/packaging/hosting_debian_config.json @@ -6,7 +6,7 @@ "install_root": "/usr/share/dotnet", "short_description": "Microsoft .NET Core DEB_VERSION Linux Server Hosting", - "long_description": "Microsoft .NET Core DEB_VERSION Linux Server Hosting enables hosting of ASP.NET Core applications and contains the ASP.NET Core Runtime Package Store and .NET Core Runtime. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub (https://github.com/aspnet/home). We happily accept issues and PRs.", + "long_description": "Microsoft .NET Core Linux Server Hosting enables hosting of ASP.NET Core applications and contains the ASP.NET Core Runtime Package Store and .NET Core Runtime. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub (https://github.com/aspnet/home). We happily accept issues and PRs.", "homepage": "https://www.asp.net/", "release":{ From 354f4e123c748b42fb2f86276ec4d3f00aedf849 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 18 Oct 2017 15:06:14 -0700 Subject: [PATCH 106/115] Update Razor to include minor tweak to build script. aspnet/Razor#1728 --- modules/Razor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Razor b/modules/Razor index a1671c9b9f..cff2ef2e0a 160000 --- a/modules/Razor +++ b/modules/Razor @@ -1 +1 @@ -Subproject commit a1671c9b9ff1dc574501f5036d9cf14a316d12df +Subproject commit cff2ef2e0a3a6a24c689505dcbdceec228cd6d20 From 86d6e51ba26386c39fb1863b2bccbb0915238454 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 19 Oct 2017 12:17:18 -0700 Subject: [PATCH 107/115] Fix rpm packages and hosting tar.gz --- build/RuntimeStoreInstaller.targets | 47 ++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 8cb848ca85..a4c1690d66 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -6,7 +6,7 @@ <_DebToolDir>$(MSBuildThisFileDirectory)tools\dotnet-deb-tool-consumer\ <_TimestampRSSource>$(RepositoryRoot).deps\Signed\Store\ <_TimestampFreeRSSource>$(RepositoryRoot).deps\Signed\Store-TimestampFree\ - <_CoreInstallerSource>$(RepositoryRoot).deps\CoreInstallers\ + <_InstallerSource>$(RepositoryRoot).deps\Installers\ <_WorkRoot>$(RepositoryRoot).w\ <_WorkLayoutDir>$(_WorkRoot).l\ <_WorkOutputDir>$(_WorkRoot).o\ @@ -18,9 +18,11 @@ $(_PackagingDir)store_debian_config.json $(_PackagingDir)hosting_debian_config.json + https://dotnetcli.blob.core.windows.net/dotnet $(KOREBUILD_DOTNET_FEED_UNCACHED) - https://dotnetcli.blob.core.windows.net/dotnet + $(PublicCoreFeedPrefix) + $(PublicCoreFeedPrefix)/aspnetcore/store/2.0.0-26452/Build.RS.linux.tar.gz $(CoreFeedPrefix)/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz $(_TimestampRSSource)aspnetcore-store-$(PackageVersion)-linux-x64.tar.gz $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)-linux-x64.tar.gz @@ -46,15 +48,16 @@ Condition="!Exists('$(TimestampFreeRSArchive)')" /> - + - + - + + @@ -64,15 +67,25 @@ - + + + + + + - + + + <_DependentArchives Include="$(_InstallerSource)dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz" /> + <_DependentArchives Include="$(_InstallerSource)Build.RS.linux.tar.gz" /> + + - $(_CoreInstallerSource)dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz + @(_DependentArchives) $(HostingInstallerName)-$(Version)-linux-x64.tar.gz $(HostingInstallerName)-$(PackageVersionNoTimestamp)-linux-x64.tar.gz @@ -81,12 +94,12 @@ + Properties="DependentArchives=$(DependentArchives);RSArchive=$(TimestampRSArchive);HostingArchiveName=$(HostingArchiveName)" /> + Properties="DependentArchives=$(DependentArchives);RSArchive=$(TimestampFreeRSArchive);HostingArchiveName=$(TimestampFreeHostingArchiveName)" /> @@ -218,6 +231,9 @@ + + 2.0.0 + $(PackageVersion) @@ -235,12 +251,13 @@ @(HostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') @(TimestampFreeHostingDependencies->' -d "%(Identity) >= %(Version)"', ' ') - @(RHStoreDirectories->' --directories "%(FullPath)"', ' ') - @(GenericStoreDirectories->' --directories "%(FullPath)"', ' ') + @(RSDependencies->' -d "%(Identity) >= %(Version)"', ' ') + $(RSDependencyArguments) @(RHStoreDirectories->' --directories "%(FullPath)"', ' ') + $(RSDependencyArguments) @(GenericStoreDirectories->' --directories "%(FullPath)"', ' ') Image=$(Image);RPMVendor=$(RPMVendor) RPMFileSuffix=rhel.7-x64.rpm;RPMInstallRoot=$(GenericInstallerInstallRoot) - RPMFileSuffix=rhel.rh.7-x64.rpm;RPMInstallRoot=$(RHInstallerInstallRoot) + RPMFileSuffix=rh.rhel.7-x64.rpm;RPMInstallRoot=$(RHInstallerInstallRoot) MaintainerName=@(_RSMaintainerName);MaintainerEmail=@(_RSMaintainerEmail) $(CommonRSArguments);RPMInstallerPrefix=$(RSInstallerName);RPMRevision=@(_RSPackageRevision) @@ -259,7 +276,7 @@ $(TimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(GenericRSArguments) $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) - $(TimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingArguments) + $(TimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingFPMArguments) $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) $(TimestampFreeHostingArguments);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(TimestampFreeHostingFPMArguments) @@ -271,7 +288,7 @@ $(RHTimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(RHRSArguments) $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) - $(RHTimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingArguments) + $(RHTimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingFPMArguments) $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) $(RHTimestampFreeHostingArguments);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(TimestampFreeHostingFPMArguments) From 4da6565d5ba11a2b29336d73fade95430dfcada9 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 24 Oct 2017 10:35:19 -0700 Subject: [PATCH 108/115] Update modules/Scaffolding to ce3c4802 to fix aspnet/Scaffolding#645 --- modules/Scaffolding | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Scaffolding b/modules/Scaffolding index 5ff6bc6399..ce3c480287 160000 --- a/modules/Scaffolding +++ b/modules/Scaffolding @@ -1 +1 @@ -Subproject commit 5ff6bc6399e373535e8e91924a439180d794f1c0 +Subproject commit ce3c4802873c0a64baf847544e65b3694c099005 From 386483b9ef69cd86db8142f2e3dcfc2f28f0ce7d Mon Sep 17 00:00:00 2001 From: John Luo Date: Tue, 24 Oct 2017 11:34:56 -0700 Subject: [PATCH 109/115] Create timestamp free cumulative archives --- build/RuntimeStoreInstaller.targets | 93 +++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 10 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index a4c1690d66..0868fa6dee 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -22,10 +22,11 @@ $(KOREBUILD_DOTNET_FEED_UNCACHED) $(PublicCoreFeedPrefix) - $(PublicCoreFeedPrefix)/aspnetcore/store/2.0.0-26452/Build.RS.linux.tar.gz + $(PublicCoreFeedPrefix)/aspnetcore/store/2.0.0-26452/Build.RS. $(CoreFeedPrefix)/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz $(_TimestampRSSource)aspnetcore-store-$(PackageVersion)-linux-x64.tar.gz - $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)-linux-x64.tar.gz + $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)- + $(TimestampFreeRSArchivePrefix)linux-x64.tar.gz @@ -44,11 +45,25 @@ Text="Timestamp linux archive not found. Expected it to exist in $(TimestampRSArchive)." Condition="!Exists('$(TimestampRSArchive)')" /> + Text="Non-timestamp linux archive not found. Expected it to exist in $(TimestampFreeLinuxRSArchive)." + Condition="!Exists('$(TimestampFreeLinuxRSArchive)')" /> + + + - + + + + + + - + @@ -78,6 +93,28 @@ + + + + + + + + + + + + + + + + + + + + + + <_DependentArchives Include="$(_InstallerSource)dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz" /> @@ -99,7 +136,43 @@ + Properties="DependentArchives=$(DependentArchives);RSArchive=$(TimestampFreeLinuxRSArchive);HostingArchiveName=$(TimestampFreeHostingArchiveName)" /> + + + + + + + + + + + + + + $(TimestampFreeRSArchivePrefix)linux-x64.tar.gz + + + $(TimestampFreeRSArchivePrefix)osx-x64.tar.gz + + + $(TimestampFreeRSArchivePrefix)win7-x64.zip + + + $(TimestampFreeRSArchivePrefix)win7-x86.zip + + + + + + + @@ -273,7 +346,7 @@ $(TimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(PackageVersion);RPMArguments=$(GenericRSArguments) $(CommonArguments);$(CommonGenericArguments);$(CommonRSArguments) - $(TimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(GenericRSArguments) + $(TimestampFreeRSArguments);RSArchive=$(TimestampFreeLinuxRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(GenericRSArguments) $(CommonArguments);$(CommonGenericArguments);$(CommonHostingArguments) $(TimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingFPMArguments) @@ -285,7 +358,7 @@ $(RHTimestampRSArguments);RSArchive=$(TimestampRSArchive);RPMVersion=$(PackageVersion);RPMArguments=$(RHRSArguments) $(CommonArguments);$(CommonRHArguments);$(CommonRSArguments) - $(RHTimestampFreeRSArguments);RSArchive=$(TimestampFreeRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(RHRSArguments) + $(RHTimestampFreeRSArguments);RSArchive=$(TimestampFreeLinuxRSArchive);RPMVersion=$(PackageVersionNoTimestamp);RPMArguments=$(RHRSArguments) $(CommonArguments);$(CommonRHArguments);$(CommonHostingArguments) $(RHTimestampHostingArguments);RPMVersion=$(PackageVersion);RPMArguments=$(HostingFPMArguments) @@ -395,7 +468,7 @@ + Properties="$(CommonRSArguments);RSArchive=$(TimestampFreeLinuxRSArchive);DebVersion=$(PackageVersionNoTimestamp)" /> Date: Tue, 24 Oct 2017 12:36:12 -0700 Subject: [PATCH 110/115] Fix target dependencies --- build/RuntimeStoreInstaller.targets | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 0868fa6dee..08f6ecf3a8 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -90,7 +90,7 @@ - + @@ -112,7 +112,7 @@ - + @@ -131,15 +131,15 @@ + Properties="DependentArchives=$(DependentArchives);RSArchive=$(TimestampRSArchive);OutputArchiveName=$(HostingArchiveName)" /> + Properties="DependentArchives=$(DependentArchives);RSArchive=$(TimestampFreeLinuxRSArchive);OutputArchiveName=$(TimestampFreeHostingArchiveName)" /> - + @@ -167,12 +167,12 @@ + Properties="DependentArchives=$(_InstallerSource)%(TargzArchives.Identity);RSArchive=%(TargzArchives.RSArchive);OutputArchiveName=%(TargzArchives.Identity)" /> + Properties="DependentArchives=$(_InstallerSource)%(ZipArchives.Identity);RSArchive=%(ZipArchives.RSArchive);OutputArchiveName=%(ZipArchives.Identity)" /> From 8c63660b15545d6c27eed521ae4d414e170da64b Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Tue, 24 Oct 2017 12:41:26 -0700 Subject: [PATCH 111/115] Update EntityFrameworkCore to 88551da89 - react to aspnet/EntityFrameworkCore#10151 --- modules/EntityFrameworkCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/EntityFrameworkCore b/modules/EntityFrameworkCore index 98f4a94b89..88551da89e 160000 --- a/modules/EntityFrameworkCore +++ b/modules/EntityFrameworkCore @@ -1 +1 @@ -Subproject commit 98f4a94b89bec705bc9377e2609c5c0807d01417 +Subproject commit 88551da89e6d26927dc8a80a50f2fd07157a7e22 From be0dce0fd9964d6480aac9ede1b6bb1e3f5646da Mon Sep 17 00:00:00 2001 From: = Date: Thu, 26 Oct 2017 12:14:34 -0700 Subject: [PATCH 112/115] Revert to old lzma name scheme --- build/PackageArchive.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index e7819859d8..7688976518 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -4,8 +4,8 @@ <_WorkRoot>$(RepositoryRoot).w\ $(RepositoryRoot).deps\Signed\Packages\ $(RepositoryRoot).deps\Signed\Packages-NoTimeStamp\ - nuGetPackagesArchive-$(PackageVersion) - nuGetPackagesArchive-$(PackageVersionNoTimestamp) + nuGetPackagesArchive.timestamped + nuGetPackagesArchive.notimestamp From 60d604580c4e6ca944390ad6fc760e1e2624c283 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Thu, 26 Oct 2017 14:49:35 -0700 Subject: [PATCH 113/115] Add identity fix --- modules/Identity | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Identity b/modules/Identity index 98fa107c05..8c47b90677 160000 --- a/modules/Identity +++ b/modules/Identity @@ -1 +1 @@ -Subproject commit 98fa107c0505a671b68a5b13d11b72ebc6168cf6 +Subproject commit 8c47b90677c0f544844151418ba94f24d9f2a094 From 9d4c3adda1fbd5acc185c2e851aaeaabe4e45943 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 26 Oct 2017 16:14:32 -0700 Subject: [PATCH 114/115] Update modules/Templating to fix workaround dotnet/templating#1302 --- modules/Templating | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Templating b/modules/Templating index 4a897d69bc..4984b2da89 160000 --- a/modules/Templating +++ b/modules/Templating @@ -1 +1 @@ -Subproject commit 4a897d69bcfab85b1eb9cc186117230cbcb17f2f +Subproject commit 4984b2da890a662362cb728ac1a3f605ec9448bc From b0de6da24fb726941371f1697afbf21bc7928f95 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 30 Oct 2017 15:50:25 -0700 Subject: [PATCH 115/115] Update the Scaffolding and Templating submodules --- modules/Scaffolding | 2 +- modules/Templating | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Scaffolding b/modules/Scaffolding index ce3c480287..26822d4c87 160000 --- a/modules/Scaffolding +++ b/modules/Scaffolding @@ -1 +1 @@ -Subproject commit ce3c4802873c0a64baf847544e65b3694c099005 +Subproject commit 26822d4c876ee6203014eaf6df398c874c4535ea diff --git a/modules/Templating b/modules/Templating index 4984b2da89..f0991b4e6d 160000 --- a/modules/Templating +++ b/modules/Templating @@ -1 +1 @@ -Subproject commit 4984b2da890a662362cb728ac1a3f605ec9448bc +Subproject commit f0991b4e6daaf3acd0403b303b9b5142521a1572