Update the bootstrapper to install compiled KoreBuild

- This installs KoreBuild as a compiled artifact and introduces the korebuild-lock.txt file
 - In addition, changed from using PackagePublisher directly as a console tool to using the PushNuGetPackages task
This commit is contained in:
Nate McMaster 2017-07-17 10:33:10 -07:00
parent abd2ce9cf2
commit 581bcd7396
6 changed files with 371 additions and 110 deletions

5
.gitignore vendored
View File

@ -1,5 +1,6 @@
bin
obj
.vs/
*.suo
*.user
_ReSharper.*
@ -14,8 +15,8 @@ artifacts
StyleCop.Cache
node_modules
*.snk
.build
.nuget
.r
.deps
global.json
global.json
korebuild-lock.txt

217
build.ps1
View File

@ -1,66 +1,177 @@
$ErrorActionPreference = "Stop"
#!/usr/bin/env powershell
#requires -version 4
function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries)
{
while($true)
{
try
{
Invoke-WebRequest $url -OutFile $downloadLocation
break
}
catch
{
$exceptionMessage = $_.Exception.Message
Write-Host "Failed to download '$url': $exceptionMessage"
if ($retries -gt 0) {
$retries--
Write-Host "Waiting 10 seconds before retrying. Retries left: $retries"
Start-Sleep -Seconds 10
<#
.SYNOPSIS
Build this repository
.DESCRIPTION
Downloads korebuild if required. Then builds the repository.
.PARAMETER Path
The folder to build. Defaults to the folder containing this script.
.PARAMETER Channel
The channel of KoreBuild to download. Overrides the value from the config file.
.PARAMETER DotNetHome
The directory where .NET Core tools will be stored.
.PARAMETER ToolsSource
The base url where build tools can be downloaded. Overrides the value from the config file.
.PARAMETER Update
Updates KoreBuild to the latest version even if a lock file is present.
.PARAMETER ConfigFile
The path to the configuration file that stores values. Defaults to version.xml.
.PARAMETER MSBuildArgs
Arguments to be passed to MSBuild
.NOTES
This function will create a file $PSScriptRoot/korebuild-lock.txt. This lock file can be committed to source, but does not have to be.
When the lockfile is not present, KoreBuild will create one using latest available version from $Channel.
The $ConfigFile is expected to be an XML file. It is optional, and the configuration values in it are optional as well.
.EXAMPLE
Example config file:
```xml
<!-- version.xml -->
<Project>
<PropertyGroup>
<KoreBuildChannel>dev</KoreBuildChannel>
<KoreBuildToolsSource>https://aspnetcore.blob.core.windows.net/buildtools</KoreBuildToolsSource>
</PropertyGroup>
</Project>
```
#>
[CmdletBinding(PositionalBinding = $false)]
param(
[string]$Path = $PSScriptRoot,
[Alias('c')]
[string]$Channel,
[Alias('d')]
[string]$DotNetHome,
[Alias('s')]
[string]$ToolsSource,
[Alias('u')]
[switch]$Update,
[string]$ConfigFile = (Join-Path $PSScriptRoot 'version.xml'),
[Parameter(ValueFromRemainingArguments = $true)]
[string[]]$MSBuildArgs
)
Set-StrictMode -Version 2
$ErrorActionPreference = 'Stop'
#
# Functions
#
function Get-KoreBuild {
$lockFile = Join-Path $Path 'korebuild-lock.txt'
if (!(Test-Path $lockFile) -or $Update) {
Get-RemoteFile "$ToolsSource/korebuild/channels/$Channel/latest.txt" $lockFile
}
$version = Get-Content $lockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1
if (!$version) {
Write-Error "Failed to parse version from $lockFile. Expected a line that begins with 'version:'"
}
$version = $version.TrimStart('version:').Trim()
$korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version)
if (!(Test-Path $korebuildPath)) {
Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version"
New-Item -ItemType Directory -Path $korebuildPath | Out-Null
$remotePath = "$ToolsSource/korebuild/artifacts/$version/korebuild.$version.zip"
try {
$tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip"
Get-RemoteFile $remotePath $tmpfile
if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) {
# Use built-in commands where possible as they are cross-plat compatible
Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath
}
else
{
$exception = $_.Exception
throw $exception
else {
# Fallback to old approach for old installations of PowerShell
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($tmpfile, $korebuildPath)
}
}
catch {
Remove-Item -Recurse -Force $korebuildPath -ErrorAction Ignore
throw
}
finally {
Remove-Item $tmpfile -ErrorAction Ignore
}
}
return $korebuildPath
}
cd $PSScriptRoot
$repoFolder = $PSScriptRoot
$env:REPO_FOLDER = $repoFolder
$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/dev.zip"
if ($env:KOREBUILD_ZIP)
{
$koreBuildZip=$env:KOREBUILD_ZIP
function Join-Paths([string]$path, [string[]]$childPaths) {
$childPaths | ForEach-Object { $path = Join-Path $path $_ }
return $path
}
$buildFolder = ".build"
$buildFile="$buildFolder\KoreBuild.ps1"
if (!(Test-Path $buildFolder)) {
Write-Host "Downloading KoreBuild from $koreBuildZip"
$tempFolder=$env:TEMP + "\KoreBuild-" + [guid]::NewGuid()
New-Item -Path "$tempFolder" -Type directory | Out-Null
$localZipFile="$tempFolder\korebuild.zip"
DownloadWithRetry -url $koreBuildZip -downloadLocation $localZipFile -retries 6
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($localZipFile, $tempFolder)
New-Item -Path "$buildFolder" -Type directory | Out-Null
copy-item "$tempFolder\**\build\*" $buildFolder -Recurse
# Cleanup
if (Test-Path $tempFolder) {
Remove-Item -Recurse -Force $tempFolder
function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
if ($RemotePath -notlike 'http*') {
Copy-Item $RemotePath $LocalPath
return
}
$retries = 10
while ($retries -gt 0) {
$retries -= 1
try {
Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
return
}
catch {
Write-Verbose "Request failed. $retries retries remaining"
}
}
Write-Error "Download failed: '$RemotePath'."
}
&"$buildFile" @args
#
# Main
#
# Load configuration or set defaults
if (Test-Path $ConfigFile) {
[xml] $config = Get-Content $ConfigFile
if (!($Channel)) { [string] $Channel = Select-Xml -Xml $config -XPath '/Project/PropertyGroup/KoreBuildChannel' }
if (!($ToolsSource)) { [string] $ToolsSource = Select-Xml -Xml $config -XPath '/Project/PropertyGroup/KoreBuildToolsSource' }
}
if (!$DotNetHome) {
$DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } `
elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} `
elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}`
else { Join-Path $PSScriptRoot '.dotnet'}
}
if (!$Channel) { $Channel = 'dev' }
if (!$ToolsSource) { $ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' }
# Execute
$korebuildPath = Get-KoreBuild
Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1')
try {
Install-Tools $ToolsSource $DotNetHome
Invoke-RepositoryBuild $Path @MSBuildArgs
}
finally {
Remove-Module 'KoreBuild' -ErrorAction Ignore
}

223
build.sh
View File

@ -1,46 +1,195 @@
#!/usr/bin/env bash
repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $repoFolder
koreBuildZip="https://github.com/aspnet/KoreBuild/archive/dev.zip"
if [ ! -z $KOREBUILD_ZIP ]; then
koreBuildZip=$KOREBUILD_ZIP
fi
set -euo pipefail
buildFolder=".build"
buildFile="$buildFolder/KoreBuild.sh"
#
# variables
#
if test ! -d $buildFolder; then
echo "Downloading KoreBuild from $koreBuildZip"
RESET="\033[0m"
RED="\033[0;31m"
MAGENTA="\033[0;95m"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
[ -z "${DOTNET_HOME:-}"] && DOTNET_HOME="$HOME/.dotnet"
config_file="$DIR/version.xml"
verbose=false
update=false
repo_path="$DIR"
channel=''
tools_source=''
tempFolder="/tmp/KoreBuild-$(uuidgen)"
mkdir $tempFolder
#
# Functions
#
__usage() {
echo "Usage: $(basename ${BASH_SOURCE[0]}) [options] [[--] <MSBUILD_ARG>...]"
echo ""
echo "Arguments:"
echo " <MSBUILD_ARG>... Arguments passed to MSBuild. Variable number of arguments allowed."
echo ""
echo "Options:"
echo " --verbose Show verbose output."
echo " -c|--channel <CHANNEL> The channel of KoreBuild to download. Overrides the value from the config file.."
echo " --config-file <FILE> TThe path to the configuration file that stores values. Defaults to version.xml."
echo " -d|--dotnet-home <DIR> The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet."
echo " --path <PATH> The directory to build. Defaults to the directory containing the script."
echo " -s|--tools-source <URL> The base url where build tools can be downloaded. Overrides the value from the config file."
echo " -u|--update Update to the latest KoreBuild even if the lock file is present."
echo ""
echo "Description:"
echo " This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be."
echo " When the lockfile is not present, KoreBuild will create one using latest available version from \$channel."
localZipFile="$tempFolder/korebuild.zip"
retries=6
until (wget -O $localZipFile $koreBuildZip 2>/dev/null || curl -o $localZipFile --location $koreBuildZip 2>/dev/null)
do
echo "Failed to download '$koreBuildZip'"
if [ "$retries" -le 0 ]; then
exit 1
fi
retries=$((retries - 1))
echo "Waiting 10 seconds before retrying. Retries left: $retries"
sleep 10s
done
unzip -q -d $tempFolder $localZipFile
mkdir $buildFolder
cp -r $tempFolder/**/build/** $buildFolder
chmod +x $buildFile
# Cleanup
if test -d $tempFolder; then
rm -rf $tempFolder
if [[ "${1:-}" != '--no-exit' ]]; then
exit 2
fi
}
get_korebuild() {
local lock_file="$repo_path/korebuild-lock.txt"
if [ ! -f $lock_file ] || [ "$update" = true ]; then
__get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" $lock_file
fi
local version="$(grep 'version:*' -m 1 $lock_file)"
if [[ "$version" == '' ]]; then
__error "Failed to parse version from $lock_file. Expected a line that begins with 'version:'"
return 1
fi
version="$(echo ${version#version:} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version"
{
if [ ! -d "$korebuild_path" ]; then
mkdir -p "$korebuild_path"
local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip"
tmpfile="$(mktemp)"
echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}"
if __get_remote_file $remote_path $tmpfile; then
unzip -q -d "$korebuild_path" $tmpfile
fi
rm $tmpfile || true
fi
source "$korebuild_path/KoreBuild.sh"
} || {
if [ -d "$korebuild_path" ]; then
echo "Cleaning up after failed installation"
rm -rf "$korebuild_path" || true
fi
return 1
}
}
__error() {
echo -e "${RED}$@${RESET}" 1>&2
}
__machine_has() {
hash "$1" > /dev/null 2>&1
return $?
}
__get_remote_file() {
local remote_path=$1
local local_path=$2
if [[ "$remote_path" != 'http'* ]]; then
cp $remote_path $local_path
return 0
fi
failed=false
if __machine_has curl ; then
curl --retry 10 -sSL -f --create-dirs -o $local_path $remote_path || failed=true
elif __machine_has wget; then
wget --tries 10 -O $local_path $remote_path || failed=true
else
failed=true
fi
if [ "$failed" = true ]; then
__error "Download failed: $remote_path" 1>&2
return 1
fi
}
__read_dom () { local IFS=\> ; read -d \< ENTITY CONTENT ;}
#
# main
#
while [[ $# > 0 ]]; do
case $1 in
-\?|-h|--help)
__usage --no-exit
exit 0
;;
-c|--channel|-Channel)
shift
channel=${1:-}
[ -z "$channel" ] && __usage
;;
--config-file|-ConfigFile)
shift
config_file="${1:-}"
[ -z "$config_file" ] && __usage
;;
-d|--dotnet-home|-DotNetHome)
shift
DOTNET_HOME=${1:-}
[ -z "$DOTNET_HOME" ] && __usage
;;
--path|-Path)
shift
repo_path="${1:-}"
[ -z "$repo_path" ] && __usage
;;
-s|--tools-source|-ToolsSource)
shift
tools_source="${1:-}"
[ -z "$tools_source" ] && __usage
;;
-u|--update|-Update)
update=true
;;
--verbose|-Verbose)
verbose=true
;;
--)
shift
break
;;
*)
break
;;
esac
shift
done
if ! __machine_has unzip; then
__error 'Missing required command: unzip'
exit 1
fi
$buildFile -r $repoFolder "$@"
if ! __machine_has curl && ! __machine_has wget; then
__error 'Missing required command. Either wget or curl is required.'
exit 1
fi
if [ -f $config_file ]; then
comment=false
while __read_dom; do
if [ "$comment" = true ]; then [[ $CONTENT == *'-->'* ]] && comment=false ; continue; fi
if [[ $ENTITY == '!--'* ]]; then comment=true; continue; fi
if [ -z "$channel" ] && [[ $ENTITY == "KoreBuildChannel" ]]; then channel=$CONTENT; fi
if [ -z "$tools_source" ] && [[ $ENTITY == "KoreBuildToolsSource" ]]; then tools_source=$CONTENT; fi
done < $config_file
fi
[ -z "$channel" ] && channel='dev'
[ -z "$tools_source" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools'
get_korebuild
install_tools "$tools_source" "$DOTNET_HOME"
invoke_repository_build "$repo_path" $@

View File

@ -42,20 +42,14 @@
<Message Text="============ Building $(RepositoryToBuild) ============" Importance="High" />
<!-- Copy Korebuild folder to individual repos to avoid downloading it again and again and also to
prevent from taking in newer Korebuild changes when Universe is building midway -->
<ItemGroup>
<_KorebuildItems Include="$(RepositoryRoot).build\**\*.*" />
</ItemGroup>
<Message Text="Copying Korebuild from Universe to repository $(BuildRepositoryRoot)"/>
<Copy SourceFiles="@(_KorebuildItems)" DestinationFolder="$(BuildRepositoryRoot).build\%(RecursiveDir)" SkipUnchangedFiles="true" UseHardlinksIfPossible="true" />
<!-- Copy Korebuild lock file to individual repos to align version if the repo doesn't already have one -->
<Message Text="Copying KoreBuild lockfile from Universe to repository $(BuildRepositoryRoot)"/>
<Copy SourceFiles="$(RepositoryRoot)korebuild-lock.txt" DestinationFolder="$(BuildRepositoryRoot)" />
<Exec
Command="./$(_BuildScriptToExecute) $(BuildArguments)"
EnvironmentVariables="KOREBUILD_SKIP_RUNTIME_INSTALL=1"
Command="./$(_BuildScriptToExecute) -Path $(BuildRepositoryRoot) $(BuildArguments)"
IgnoreStandardErrorWarningFormat="true"
WorkingDirectory="$(BuildRepositoryRoot)" />
WorkingDirectory="$(RepositoryRoot)" />
<ItemGroup>
<RepositoryArtifacts Include="$(RepositoryArtifactsBuildDirectory)*" />
@ -75,8 +69,10 @@
<Message Text="Publishing the following packages to the volatile feed: @(RepositoryNupkgs -> '%(Filename)%(Extension)', ', ')"
Condition="'$(PublishPackages)'=='true' AND '@(RepositoryNupkgs)' != ''" />
<Exec
Command="$(DotNetPath) $(PackagePublisherNetCoreApp) -d $(RepositoryArtifactsBuildDirectory) -f $(NuGetPublishVolatileFeed)"
<PushNuGetPackages
Packages="@(RepositoryNupkgs)"
Feed="$(NuGetPublishVolatileFeed)"
ApiKey="$(APIKey)"
Condition="'$(PublishPackages)'=='true' AND '@(RepositoryNupkgs)' != ''" />
<Message Text="============ Done building $(RepositoryToBuild) ============" Importance="High" />

View File

@ -23,10 +23,6 @@
<BuildDependsOn>$(BuildDependsOn);CloneRepositories;BuildRepositories</BuildDependsOn>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="PackagePublisher" Version="2.0.0-*" />
</ItemGroup>
<Import Project="$(_RepositoryListToImport)" />
<Target Name="CleanUniverseArtifacts">
@ -111,7 +107,7 @@
</Target>
<Target Name="BuildRepositories"
DependsOnTargets="_PrepareRepositories;_FindDotNetPath;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;_GenerateBuildGraph;_BuildRepositories" />
DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;_GenerateBuildGraph;_BuildRepositories" />
<Target Name="_PrepareRestoreGraphSpecs" DependsOnTargets="_PrepareRepositories">
<ItemGroup>

8
version.xml Normal file
View File

@ -0,0 +1,8 @@
<!-- This file may be overwritten by automation. -->
<Project>
<PropertyGroup>
<KoreBuildChannel>dev</KoreBuildChannel>
<VersionPrefix>2.1.0</VersionPrefix>
<VersionSuffix>preview1</VersionSuffix>
</PropertyGroup>
</Project>