From c3beb1f9b1497078974525ef8987a35da74376b3 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Thu, 20 Dec 2018 16:52:15 -0800 Subject: [PATCH] build.cmd/sh changes (#5958) Update the build scripts to support building subfolders or subgroups of projects * Add build scripts for ci * Remove obsolete scripts * Add flags like --test and --pack to control running just test or packaging * Add flags like --managed and --native to control building sub-types of projects * Remove KoreBuild bootstrapper flags * Update to extensions 3.0.0-preview.18619.1 (needed to get a fix for aspnet/Extensions#815 to make this change work on MSBuild.exe) --- .azure/pipelines/ci.yml | 32 +- .azure/pipelines/jobs/default-build.yml | 20 +- .azure/pipelines/pr-validation-temp.yml | 6 +- build.cmd | 2 +- build.ps1 | 264 +++++++++++++++ build.sh | 284 +++++++++++++++- build/CodeSign.targets | 2 - build/RepositoryBuild.targets | 36 +- build/SharedFx.targets | 2 +- build/dependencies.props | 116 +++---- build/repo.props | 95 +++--- build/repo.targets | 43 ++- docs/BuildFromSource.md | 33 +- .../scripts/InstallVisualStudio.ps1 | 4 +- eng/scripts/cibuild.cmd | 3 + eng/scripts/cibuild.sh | 7 + .../VsRequirements => eng/scripts}/vs.json | 0 run.cmd => restore.cmd | 2 +- restore.sh | 7 + run.ps1 | 207 ------------ run.sh | 250 -------------- scripts/Tag-TeamCityBuild.ps1 | 37 --- scripts/UpdateBuildTools.ps1 | 99 ------ scripts/UpdateDependencies.ps1 | 80 ----- scripts/UpdateSubmodules.ps1 | 144 -------- scripts/common.psm1 | 314 ------------------ src/Components/build.cmd | 2 +- src/Components/build.sh | 2 +- src/Components/korebuild.json | 14 - src/DataProtection/build.cmd | 4 +- src/DataProtection/build.sh | 2 +- src/Mvc/build.cmd | 2 +- src/Mvc/build.sh | 4 +- ...ft.Extensions.ApiDescription.Design.nuspec | 3 +- src/Razor/build.cmd | 3 + src/{MusicStore => Razor}/build.sh | 2 +- src/Razor/korebuild.json | 14 - src/Servers/Directory.Build.props | 12 - src/Servers/IIS/ResolveIisReferences.targets | 8 +- src/Servers/IIS/build.cmd | 3 + src/Servers/Kestrel/build.cmd | 3 + src/Servers/Kestrel/build.sh | 7 + src/SignalR/build.cmd | 2 +- src/SignalR/build.sh | 2 +- src/SignalR/korebuild.json | 10 - src/SiteExtensions/LoggingAggregate/build.cmd | 2 +- src/SiteExtensions/LoggingAggregate/build.sh | 2 +- 47 files changed, 796 insertions(+), 1396 deletions(-) create mode 100644 build.ps1 rename scripts/install_vs.ps1 => eng/scripts/InstallVisualStudio.ps1 (97%) create mode 100644 eng/scripts/cibuild.cmd create mode 100755 eng/scripts/cibuild.sh rename {scripts/VsRequirements => eng/scripts}/vs.json (100%) rename run.cmd => restore.cmd (71%) create mode 100644 restore.sh delete mode 100644 run.ps1 delete mode 100755 run.sh delete mode 100755 scripts/Tag-TeamCityBuild.ps1 delete mode 100755 scripts/UpdateBuildTools.ps1 delete mode 100755 scripts/UpdateDependencies.ps1 delete mode 100755 scripts/UpdateSubmodules.ps1 delete mode 100644 scripts/common.psm1 delete mode 100644 src/Components/korebuild.json create mode 100644 src/Razor/build.cmd rename src/{MusicStore => Razor}/build.sh (58%) delete mode 100644 src/Razor/korebuild.json delete mode 100644 src/Servers/Directory.Build.props create mode 100644 src/Servers/IIS/build.cmd create mode 100644 src/Servers/Kestrel/build.cmd create mode 100755 src/Servers/Kestrel/build.sh delete mode 100644 src/SignalR/korebuild.json diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 1216abe4e6..da6de64a6b 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -13,6 +13,7 @@ jobs: jobName: Windows_Build jobDisplayName: "Build and test: Windows" agentOs: Windows + buildScript: ./eng/scripts/cibuild.cmd beforeBuild: - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" displayName: Setup IISExpress test certificates and schema @@ -21,37 +22,10 @@ jobs: jobName: MacOs_Build jobDisplayName: "Build and test: macOS" agentOs: macOs + buildScript: ./eng/scripts/cibuild.sh - template: jobs/default-build.yml parameters: jobName: Linux_Build jobDisplayName: "Build and test: Linux" agentOs: Linux -# - template: jobs/iisintegration-job.yml -# parameters: -# TestGroupName: IIS -# SkipIISTests: false -# SkipIISExpressTests: true -# SkipIISForwardsCompatibilityTests: true -# SkipIISBackwardsCompatibilityTests: true -# - template: jobs/iisintegration-job.yml -# parameters: -# TestGroupName: IISExpress -# SkipIISTests: true -# SkipIISExpressTests: false -# SkipIISForwardsCompatibilityTests: true -# SkipIISBackwardsCompatibilityTests: true -# - template: jobs/iisintegration-job.yml -# parameters: -# TestGroupName: IISForwardCompat -# SkipIISTests: true -# SkipIISExpressTests: true -# SkipIISForwardsCompatibilityTests: false -# SkipIISBackwardsCompatibilityTests: true -# - template: jobs/iisintegration-job.yml -# parameters: -# TestGroupName: IISBackCompat -# SkipIISTests: true -# SkipIISExpressTests: true -# SkipIISForwardsCompatibilityTests: true -# SkipIISBackwardsCompatibilityTests: false - + buildScript: ./eng/scripts/cibuild.sh diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 1ed9dd4501..b750a78f7f 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -35,6 +35,8 @@ # This build definition is enabled for code signing. (Only applies to Windows) # buildDirectory: string # Specifies what directory to run build.sh/cmd +# buildScript: string +# Specifies the build script to run. Defaults to build.sh or build.cmd. # # See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details @@ -58,6 +60,7 @@ parameters: publish: true path: 'artifacts/' buildDirectory: '' + buildScript: '' jobs: - job: ${{ coalesce(parameters.jobName, parameters.agentOs) }} @@ -90,6 +93,7 @@ jobs: AgentOsName: ${{ parameters.agentOs }} ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping DOTNET_HOME: $(Agent.WorkFolder)/.dotnet + BuildScript: ${{ parameters.buildScript }} BuildScriptArgs: ${{ parameters.buildArgs }} BuildConfiguration: ${{ parameters.configuration }} BuildDirectory: ${{ parameters.buildDirectory }} @@ -114,12 +118,16 @@ jobs: signType: $(_SignType) zipSources: false - ${{ parameters.beforeBuild }} - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - script: .\$(BuildDirectory)\build.cmd -ci /p:SignType=$(_SignType) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) - displayName: Run build.cmd - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: ./$(BuildDirectory)/build.sh -ci -p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) - displayName: Run build.sh + - ${{ if eq(parameters.buildScript, '') }}: + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - script: .\$(BuildDirectory)\build.cmd -ci /p:SignType=$(_SignType) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) + displayName: Run build.cmd + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: ./$(BuildDirectory)/build.sh -ci -p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) + displayName: Run build.sh + - ${{ if ne(parameters.buildScript, '') }}: + - script: $(BuildScript) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) + displayName: run $(BuildScript) - task: PublishTestResults@2 displayName: Publish test results condition: always() diff --git a/.azure/pipelines/pr-validation-temp.yml b/.azure/pipelines/pr-validation-temp.yml index b190d7c0c4..c6b3feeb1a 100644 --- a/.azure/pipelines/pr-validation-temp.yml +++ b/.azure/pipelines/pr-validation-temp.yml @@ -15,16 +15,16 @@ jobs: jobName: Windows_Build jobDisplayName: "Build only : Windows" agentOs: Windows - buildArgs: '/p:SkipTests=true' + buildArgs: -all -pack - template: jobs/default-build.yml parameters: jobName: macOs_Build jobDisplayName: "Build only : macOS" agentOs: macOS - buildArgs: '/p:SkipTests=true' + buildArgs: --all --pack - template: jobs/default-build.yml parameters: jobName: Linux_Build jobDisplayName: "Build only : Linux" agentOs: Linux - buildArgs: '/p:SkipTests=true' + buildArgs: --all --pack diff --git a/build.cmd b/build.cmd index c0050bda12..b6c8d24864 100644 --- a/build.cmd +++ b/build.cmd @@ -1,2 +1,2 @@ @ECHO OFF -PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0run.ps1' default-build %*; exit $LASTEXITCODE" +PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' %*; exit $LASTEXITCODE" diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 0000000000..82f1241ad6 --- /dev/null +++ b/build.ps1 @@ -0,0 +1,264 @@ +#requires -version 4 + +<# +.SYNOPSIS +Builds this repository. + +.DESCRIPTION +This build script installs required tools and runs an MSBuild command on this repository. +This script can be used to invoke various targets, such as targets to produce packages, +build projects, run tests, and generate code. + +.PARAMETER RepoPath +The folder to build. Defaults to the folder containing this script. This will be removed soon. + +.PARAMETER CI +Sets up CI specific settings and variables. + +.PARAMETER Restore +Run restore on projects. + +.PARAMETER Build +Compile projects. + +.PARAMETER Pack +Produce packages. + +.PARAMETER Test +Run tests. + +.PARAMETER Sign +Run code signing. + +.PARAMETER Projects +A list of projects to build. (Must be an absolute path.) Globbing patterns are supported, such as "$(pwd)/**/*.csproj" + +.PARAMETER All +Build all project types. + +.PARAMETER Managed +Build managed projects (C#, F#, VB). + +.PARAMETER Native +Build native projects (C++). + +.PARAMETER NodeJS +Build NodeJS projects (TypeScript, JS). + +.PARAMETER MSBuildArguments +Additional MSBuild arguments to be passed through. + +.EXAMPLE +Building both native and managed projects. + + build.ps1 -managed -native + +.EXAMPLE +Building a subfolder of code. + + build.ps1 "$(pwd)/src/SomeFolder/**/*.csproj" + +.EXAMPLE +Running tests. + + build.ps1 -test + +.LINK +Online version: https://github.com/aspnet/AspNetCore/blob/master/docs/BuildFromSource.md +#> +[CmdletBinding(PositionalBinding = $false, DefaultParameterSetName='Groups')] +param( + # Bootstrapper options + [Obsolete('This parameter will be removed when we finish https://github.com/aspnet/AspNetCore/issues/4246')] + [string]$RepoRoot = $PSScriptRoot, + + [switch]$CI, + + # Build lifecycle options + [switch]$Restore = $True, # Run tests + [switch]$Build = $True, # Compile + [switch]$Pack, # Produce packages + [switch]$Test, # Run tests + [switch]$Sign, # Code sign + + # Project selection + [Parameter(ParameterSetName = 'All')] + [switch]$All, # Build everything + + # A list of projects which should be built. + [Parameter(ParameterSetName = 'Projects')] + [string]$Projects, + + # Build a specified set of project groups + [Parameter(ParameterSetName = 'Groups')] + [switch]$Managed, + [Parameter(ParameterSetName = 'Groups')] + [switch]$Native, + [Parameter(ParameterSetName = 'Groups')] + [switch]$NodeJS, + + # Other lifecycle targets + [switch]$Help, # Show help + + # Capture the rest + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$MSBuildArguments +) + +Set-StrictMode -Version 2 +$ErrorActionPreference = 'Stop' + +# +# Functions +# + +function Get-KoreBuild { + + if (!(Test-Path $LockFile)) { + Get-RemoteFile "$ToolsSource/korebuild/channels/$Channel/latest.txt" $LockFile + } + + $version = Get-Content $LockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1 + if (!$version) { + Write-Error "Failed to parse version from $LockFile. Expected a line that begins with 'version:'" + } + $version = $version.TrimStart('version:').Trim() + $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version) + + if (!(Test-Path $korebuildPath)) { + Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version" + New-Item -ItemType Directory -Path $korebuildPath | Out-Null + $remotePath = "$ToolsSource/korebuild/artifacts/$version/korebuild.$version.zip" + + try { + $tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip" + Get-RemoteFile $remotePath $tmpfile + if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) { + # Use built-in commands where possible as they are cross-plat compatible + Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath + } + else { + # Fallback to old approach for old installations of PowerShell + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($tmpfile, $korebuildPath) + } + } + catch { + Remove-Item -Recurse -Force $korebuildPath -ErrorAction Ignore + throw + } + finally { + Remove-Item $tmpfile -ErrorAction Ignore + } + } + + return $korebuildPath +} + +function Join-Paths([string]$path, [string[]]$childPaths) { + $childPaths | ForEach-Object { $path = Join-Path $path $_ } + return $path +} + +function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) { + if ($RemotePath -notlike 'http*') { + Copy-Item $RemotePath $LocalPath + return + } + + $retries = 10 + while ($retries -gt 0) { + $retries -= 1 + try { + Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath + return + } + catch { + Write-Verbose "Request failed. $retries retries remaining" + } + } + + Write-Error "Download failed: '$RemotePath'." +} + +# +# Main +# + +# Load configuration or set defaults + +if ($Help) { + Get-Help $PSCommandPath + exit 1 +} + +$RepoRoot = Resolve-Path $RepoRoot +$Channel = 'master' +$ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' +$ConfigFile = Join-Path $PSScriptRoot 'korebuild.json' +$LockFile = Join-Path $PSScriptRoot 'korebuild-lock.txt' + +if (Test-Path $ConfigFile) { + try { + $config = Get-Content -Raw -Encoding UTF8 -Path $ConfigFile | ConvertFrom-Json + if ($config) { + if (!($Channel) -and (Get-Member -Name 'channel' -InputObject $config)) { [string] $Channel = $config.channel } + if (!($ToolsSource) -and (Get-Member -Name 'toolsSource' -InputObject $config)) { [string] $ToolsSource = $config.toolsSource} + } + } catch { + Write-Warning "$ConfigFile could not be read. Its settings will be ignored." + Write-Warning $Error[0] + } +} + +$DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } ` + elseif ($CI) { Join-Path $PSScriptRoot '.dotnet' } ` + elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} ` + elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}` + else { Join-Path $PSScriptRoot '.dotnet'} + +$env:DOTNET_HOME = $DotNetHome + +# Execute + +$korebuildPath = Get-KoreBuild + +# Project selection +if ($All) { + $MSBuildArguments += '/p:BuildAllProjects=true' +} +elseif ($Projects) { + $MSBuildArguments += "/p:Projects=$Projects" +} +else { + # When adding new sub-group build flags, add them to this check + if((-not $Native) -and (-not $Managed) -and (-not $NodeJS)) { + Write-Warning "No default group of projects was specified, so building the 'managed' and 'native' subset of projects. Run ``build.cmd -help`` for more details." + + # This goal of this is to pick a sensible default for `build.cmd` with zero arguments. + # We believe the most common thing our contributors will work on is C#, so if no other build group was picked, build the C# projects. + + $Managed = $true + } + + $MSBuildArguments += "/p:BuildManaged=$Managed" + $MSBuildArguments += "/p:BuildNative=$Native" + $MSBuildArguments += "/p:BuildNodeJS=$NodeJS" +} + +# Target selection +$MSBuildArguments += "/p:_RunRestore=$Restore" +$MSBuildArguments += "/p:_RunBuild=$Build" +$MSBuildArguments += "/p:_RunPack=$Pack" +$MSBuildArguments += "/p:_RunTests=$Test" +$MSBuildArguments += "/p:_RunSign=$Sign" + +Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1') + +try { + Set-KoreBuildSettings -ToolsSource $ToolsSource -DotNetHome $DotNetHome -RepoPath $RepoRoot -ConfigFile $ConfigFile -CI:$CI + Invoke-KoreBuildCommand 'default-build' @MSBuildArguments +} +finally { + Remove-Module 'KoreBuild' -ErrorAction Ignore +} diff --git a/build.sh b/build.sh index 98a4b22765..3f99fddf97 100755 --- a/build.sh +++ b/build.sh @@ -1,8 +1,284 @@ #!/usr/bin/env bash set -euo pipefail -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# Call "sync" between "chmod" and execution to prevent "text file busy" error in Docker (aufs) -chmod +x "$DIR/run.sh"; sync -"$DIR/run.sh" default-build "$@" +# +# variables +# + +RESET="\033[0m" +RED="\033[0;31m" +YELLOW="\033[0;33m" +MAGENTA="\033[0;95m" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +verbose=false +update=false +reinstall=false +repo_path="$DIR" +lockfile_path="$DIR/korebuild-lock.txt" +config_file="$DIR/korebuild.json" +channel='master' +tools_source='https://aspnetcore.blob.core.windows.net/buildtools' +ci=false +run_restore=true +run_build=true +run_pack=false +run_tests=false +build_all=false +build_managed=false +build_nodejs=false +build_projects='' +msbuild_args=() + +# +# Functions +# +__usage() { + echo "Usage: $(basename "${BASH_SOURCE[0]}") [options] [[--] ...] + +Arguments: + ... Arguments passed to the command. Variable number of arguments allowed. + +Options: + --[no-]restore Run restore. + --[no-]build Compile projects + --[no-]pack Produce packages. + --[no-]test Run tests. + + --projects A list of projects to build. (Must be an absolute path.) + Globbing patterns are supported, such as \"$(pwd)/**/*.csproj\". + + --all Build all project types. + --managed Build managed projects (C#, F#, VB). + --nodejs Build NodeJS projects (TypeScript, JS). + + --ci Apply CI specific settings and environment variables. + --verbose Show verbose output. + +Description: + This build script installs required tools and runs an MSBuild command on this repository + This script can be used to invoke various targets, such as targets to produce packages + build projects, run tests, and generate code. +" + + if [[ "${1:-}" != '--no-exit' ]]; then + exit 2 + fi +} + +get_korebuild() { + local version + if [ ! -f "$lockfile_path" ]; then + __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" "$lockfile_path" + fi + version="$(grep 'version:*' -m 1 "$lockfile_path")" + if [[ "$version" == '' ]]; then + __error "Failed to parse version from $lockfile_path. Expected a line that begins with 'version:'" + return 1 + fi + version="$(echo "${version#version:}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version" + + { + if [ ! -d "$korebuild_path" ]; then + mkdir -p "$korebuild_path" + local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip" + tmpfile="$(mktemp)" + echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}" + if __get_remote_file "$remote_path" "$tmpfile"; then + unzip -q -d "$korebuild_path" "$tmpfile" + fi + rm "$tmpfile" || true + fi + + source "$korebuild_path/KoreBuild.sh" + } || { + if [ -d "$korebuild_path" ]; then + echo "Cleaning up after failed installation" + rm -rf "$korebuild_path" || true + fi + return 1 + } +} + +__error() { + echo -e "${RED}error: $*${RESET}" 1>&2 +} + +__warn() { + echo -e "${YELLOW}warning: $*${RESET}" +} + +__machine_has() { + hash "$1" > /dev/null 2>&1 + return $? +} + +__get_remote_file() { + local remote_path=$1 + local local_path=$2 + + if [[ "$remote_path" != 'http'* ]]; then + cp "$remote_path" "$local_path" + return 0 + fi + + local failed=false + if __machine_has wget; then + wget --tries 10 --quiet -O "$local_path" "$remote_path" || failed=true + else + failed=true + fi + + if [ "$failed" = true ] && __machine_has curl; then + failed=false + curl --retry 10 -sSL -f --create-dirs -o "$local_path" "$remote_path" || failed=true + fi + + if [ "$failed" = true ]; then + __error "Download failed: $remote_path" 1>&2 + return 1 + fi +} + +# +# main +# + +while [[ $# -gt 0 ]]; do + case $1 in + -\?|-h|--help) + __usage --no-exit + exit 0 + ;; + --repo-root|-RepoRoot) + shift + __warn '--repo-root is obsolete and will be removed when we finish https://github.com/aspnet/AspNetCore/issues/4246' + repo_path="${1:-}" + [ -z "$repo_path" ] && __error "Missing value for parameter --repo-root" && __usage + ;; + --restore|-[Rr]estore) + run_restore=true + ;; + --no-restore) + run_restore=false + ;; + --build|-[Bb]build) + run_build=true + ;; + --no-build) + run_build=false + ;; + --pack|-[Pp]ack) + run_pack=true + ;; + --no-pack) + run_pack=false + ;; + --test|-[Tt]est) + run_tests=true + ;; + --no-test) + run_tests=false + ;; + --projects|-[Pp]rojects) + shift + build_projects="${1:-}" + [ -z "$build_projects" ] && __error "Missing value for parameter --projects" && __usage + ;; + --all|-[Aa]ll) + build_all=true + ;; + --managed|-[Mm]anaged) + build_managed=true + ;; + --nodejs|-[Nn]ode[Jj][Ss]) + build_nodejs=true + ;; + --native|-[Nn]ative) + __warn 'The C++ projects in this repo only build on Windows. The --native flag will be ignored.' + ;; + --ci|-[Cc][Ii]) + ci=true + if [[ -z "${DOTNET_HOME:-}" ]]; then + DOTNET_HOME="$DIR/.dotnet" + fi + ;; + --verbose|-[Vv]erbose) + verbose=true + ;; + *) + msbuild_args[${#msbuild_args[*]}]="$1" + ;; + esac + shift +done + +if ! __machine_has unzip; then + __error 'Missing required command: unzip' + exit 1 +fi + +if ! __machine_has curl && ! __machine_has wget; then + __error 'Missing required command. Either wget or curl is required.' + exit 1 +fi + +if [ -f "$config_file" ]; then + if __machine_has jq ; then + if jq '.' "$config_file" >/dev/null ; then + config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")" + config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")" + else + __error "$config_file is invalid JSON. Its settings will be ignored." + exit 1 + fi + elif __machine_has python ; then + if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then + config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")" + config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")" + else + __error "$config_file is invalid JSON. Its settings will be ignored." + exit 1 + fi + else + __error 'Missing required command: jq or python. Could not parse the JSON file. Its settings will be ignored.' + exit 1 + fi + + [ ! -z "${config_channel:-}" ] && channel="$config_channel" + [ ! -z "${config_tools_source:-}" ] && tools_source="$config_tools_source" +fi + +[ -z "${DOTNET_HOME:-}" ] && DOTNET_HOME="$HOME/.dotnet" +export DOTNET_HOME="$DOTNET_HOME" + +get_korebuild + +if [ "$build_all" = true ]; then + msbuild_args[${#msbuild_args[*]}]="-p:BuildAllProjects=true" +elif [ ! -z "$build_projects" ]; then + msbuild_args[${#msbuild_args[*]}]="-p:Projects=$build_projects" +else + # When adding new sub-group build flags, add them to this check + if [ "$build_managed" = false ] && [ "$build_nodejs" = false ]; then + # This goal of this is to pick a sensible default for `build.sh` with zero arguments. + # We believe the most common thing our contributors will work on is C#, so if no other build group was picked, build the C# projects. + __warn "No default group of projects was specified, so building the 'managed' subset of projects. Run ``build.sh -help`` for more details." + build_managed=true + fi + + msbuild_args[${#msbuild_args[*]}]="-p:BuildManaged=$build_managed" + msbuild_args[${#msbuild_args[*]}]="-p:BuildNodeJS=$build_nodejs" +fi + +msbuild_args[${#msbuild_args[*]}]="-p:_RunRestore=$run_restore" +msbuild_args[${#msbuild_args[*]}]="-p:_RunBuild=$run_build" +msbuild_args[${#msbuild_args[*]}]="-p:_RunPack=$run_pack" +msbuild_args[${#msbuild_args[*]}]="-p:_RunTests=$run_tests" + +set_korebuildsettings "$tools_source" "$DOTNET_HOME" "$repo_path" "$config_file" "$ci" + +# This incantation avoids unbound variable issues if msbuild_args is empty +# https://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u +invoke_korebuild_command 'default-build' ${msbuild_args[@]+"${msbuild_args[@]}"} diff --git a/build/CodeSign.targets b/build/CodeSign.targets index d7f2ba57de..fecc25ee41 100644 --- a/build/CodeSign.targets +++ b/build/CodeSign.targets @@ -1,8 +1,6 @@ - - true $(CodeSignDependsOn);CollectFileSignInfo diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index c189b786fe..ba064d3451 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -74,7 +74,6 @@ true $(RepositoryBuildArguments) -ci - $(RepositoryBuildArguments) -DotNetHome '$(DOTNET_HOME)' $(RepositoryBuildArguments) /p:AspNetUniverseBuildOffline=true $(RepositoryBuildArguments) /p:IsUniverseBuild=true @@ -87,36 +86,17 @@ $(RepositoryBuildArguments) /p:BuildNumberSuffix=$(BuildNumberSuffix) $(RepositoryBuildArguments) /p:Configuration=$(Configuration) $(RepositoryBuildArguments) /p:IsFinalBuild=$(IsFinalBuild) + + $(RepositoryBuildArguments) /p:DisablePackageReferenceRestrictions=true $(RepositoryBuildArguments) /p:DisableCodeSigning=true $(RepositoryBuildArguments) '/p:DotNetAssetRootAccessTokenSuffix=$(DotNetAssetRootAccessTokenSuffix)' $(RepositoryBuildArguments) '/p:DotNetAssetRootUrl=$(DotNetAssetRootUrl)' $(RepositoryBuildArguments) /p:SkipAspNetCoreRuntimeInstall=true - - $(RepositoryRoot)korebuild-lock.txt - $(BuildRepositoryRoot)korebuild-lock.txt - $(IntermediateDir)$(RepositoryToBuild)-korebuild-lock.txt - $(BuildRepositoryRoot)global.json - $(IntermediateDir)$(RepositoryToBuild)-global.json - - - - - - - - - - - - - - - - + /t:CleanArtifacts /t:Build /p:SkipTests=true $(RepositoryBuildArguments) $(BuildArguments) /bl:$(LogOutputDir)$(RepositoryToBuild).build.binlog @@ -126,7 +106,7 @@ - - @@ -150,7 +128,7 @@ - + /t:Test /p:NoBuild=$(_NoBuildRepos) $(RepositoryBuildArguments) @@ -165,15 +143,13 @@ - - diff --git a/build/SharedFx.targets b/build/SharedFx.targets index c15774aeb4..08f15d775f 100644 --- a/build/SharedFx.targets +++ b/build/SharedFx.targets @@ -2,7 +2,7 @@ $(RepositoryRoot)src\Framework\test\Microsoft.AspNetCore.App.UnitTests.csproj $([MSBuild]::NormalizePath($(UnitTestFxProject))) - $(CodeSignDependsOn);GetSharedFxFilesToSign + $(CodeSignDependsOn);GetSharedFxFilesToSign _BuildSharedFxProjects;TestSharedFx $(BuildSharedFxDependsOn);CodeSign $(IntermediateDir)ar\$(SharedFxRid)\ diff --git a/build/dependencies.props b/build/dependencies.props index 9c5a55666f..b7d6f54fa7 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -36,64 +36,64 @@ 4.6.0-preview.18619.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 - 3.0.0-preview.18617.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 + 3.0.0-preview.18619.1 3.0.0-preview.18604.3 diff --git a/build/repo.props b/build/repo.props index 6fd5255a1e..0cb8c9d666 100644 --- a/build/repo.props +++ b/build/repo.props @@ -11,8 +11,6 @@ false - false - true false $(RepositoryRoot)src\ @@ -29,15 +27,18 @@ $(RepositoryRoot)src\Shared\ + + true + true + true + + + - - - - + + + - - - - - - Platform=x64 - Platform=x86 - - - + $(RepositoryRoot)src\Tools\dotnet-watch\test\TestProjects\**\*.csproj; + $(RepositoryRoot)src\Razor\Razor.Design\test\testassets\**\*.*proj; + " /> + + + + + + + + + + + + + + Platform=x64 + Platform=x86 + + + + + + + + + diff --git a/build/repo.targets b/build/repo.targets index 3592232a23..1527c80d11 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -14,21 +14,40 @@ $(IntermediateDir)dependencies.g.props $(IntermediateDir)sources.g.props + + Prepare + + $(BuildDependsOn);Restore + $(BuildDependsOn);Compile + $(BuildDependsOn);Package + $(BuildDependsOn);Test;Verify + true + true + SetTeamCityBuildNumberToVersion;$(PrepareDependsOn) - $(PrepareDependsOn);VerifyPackageArtifactConfig;VerifyExternalDependencyConfig;PrepareOutputPaths + $(PrepareDependsOn);VerifyPackageArtifactConfig;VerifyExternalDependencyConfig;PrepareOutputPaths + $(CleanDependsOn);CleanArtifacts;CleanRepoArtifacts - $(RestoreDependsOn);RestoreNpm + $(RestoreDependsOn);InstallDotNet;RestoreProjects + $(RestoreDependsOn);RestoreNpm + + + Restore $(CompileDependsOn);BuildProjects;PackSharedSources - $(CompileDependsOn);PackProjects;BuildRepositories;BuildSharedFx - $(PackageDependsOn);PackProjects - $(PackageDependsOn);RemoveSharedFrameworkOnlyRefsFromNuspec - $(PackageDependsOn);CodeSign + $(CompileDependsOn);PackProjects;BuildRepositories;BuildSharedFx + + $(PackageDependsOn);PackProjects + $(PackageDependsOn);RemoveSharedFrameworkOnlyRefsFromNuspec + + + $(TestDependsOn);Compile $(TestDependsOn);TestProjects - $(TestDependsOn);_TestRepositories + $(TestDependsOn);_TestRepositories + $(GetArtifactInfoDependsOn);GetProjectArtifactInfo $(GetArtifactInfoDependsOn);ResolveSharedSourcesPackageInfo - $(GetArtifactInfoDependsOn);ResolveRepoInfo + $(GetArtifactInfoDependsOn);ResolveRepoInfo $(CodeSignDependsOn);RemoveSharedFrameworkOnlyRefsFromNuspec @@ -37,12 +56,8 @@ - - install --no-optional - ci - - - + + diff --git a/docs/BuildFromSource.md b/docs/BuildFromSource.md index e96d350bc1..128f8d4c95 100644 --- a/docs/BuildFromSource.md +++ b/docs/BuildFromSource.md @@ -16,7 +16,7 @@ Building ASP.NET Core on Windows requires: * Windows 7 or higher * At least 10 GB of disk space and a good internet connection (our build scripts download a lot of tools and dependencies) * Visual Studio 2017. - * To install the exact required components, run scripts\install_vs.ps1. This will use VS2017. + * To install the exact required components, run [eng/scripts/InstallVisualStudio.ps1](/eng/scripts/InstallVisualStudio.ps1). This will use VS2017. * Git. * (Optional) some optional components, like the SignalR Java client, may require * NodeJS. LTS version of 10.14.2 or newer recommended @@ -58,7 +58,7 @@ Before opening our .sln files in Visual Studio or VS Code, you need to perform t 1. Executing the following on command-line: ``` - .\build.cmd /p:SkipTests=true /p:_ProjectsOnly=true + .\build.cmd ``` This will download required tools and build the entire repository once. At that point, you should be able to open .sln files to work on the projects you care about. @@ -129,15 +129,27 @@ On macOS/Linux: ./build.sh ``` -### Building a subset of the code +## Running tests on command-line + +Tests are not run by default. Use the `-test` option to run tests in addition to building. + +On Windows: +``` +.\build.cmd -test +``` + +On macOS/Linux: +``` +./build.sh --test +``` + +## Building a subset of the code This repository is large. Look for `build.cmd`/`.sh` scripts in subfolders. These scripts can be used to invoke build and test on a smaller set of projects. -#### Known issue: not every subfolder has a build.cmd script +Furthermore, you can use flags on `build.cmd`/`.sh` to build subsets based on language type, like C++, TypeScript, or C#. Run `build.sh --help` or `build.cmd -help` for details. -We'll be adding more. See https://github.com/aspnet/AspNetCore/issues/4247. - -#### Build properties +## Build properties Additional properties can be added as an argument in the form `/property:$name=$value`, or `/p:$name=$value` for short. For example: ``` @@ -150,8 +162,7 @@ Property | Description -------------------------|------------------------------------------------------------------------------------------------------------- BuildNumberSuffix | (string). A specific build number, typically from a CI counter, which is appended to the pre-release label. Configuration | `Debug` or `Release`. Default = `Debug`. -SkipTests | `true` or `false`. When true, builds without running tests. -NoBuild | `true` or `false`. Runs tests without rebuilding. +SharedFxRID | The runtime identifier of the shared framework. ## Use the result of your build @@ -183,3 +194,7 @@ See ["Artifacts"](./Artifacts.md) for more explanation of the different folders Some features, such as new target frameworks, may require prerelease tooling builds for Visual Studio. These are available in the [Visual Studio Preview](https://www.visualstudio.com/vs/preview/). + +## Resx files + +If you need to make changes to a .resx file, run `dotnet msbuild /t:Resx `. This will update the generated C#. diff --git a/scripts/install_vs.ps1 b/eng/scripts/InstallVisualStudio.ps1 similarity index 97% rename from scripts/install_vs.ps1 rename to eng/scripts/InstallVisualStudio.ps1 index bb2c506fce..918292cef2 100644 --- a/scripts/install_vs.ps1 +++ b/eng/scripts/InstallVisualStudio.ps1 @@ -21,7 +21,7 @@ mkdir $intermedateDir -ErrorAction Ignore | Out-Null $bootstrapper = "$intermedateDir\vs_enterprise1.exe" Invoke-WebRequest -Uri 'https://aka.ms/vs/15/release/vs_enterprise.exe' -OutFile $bootstrapper -$vsJson = "$PSScriptRoot\VsRequirements\vs.json" +$vsJson = "$PSScriptRoot\vs.json" # no backslashes - this breaks the installer $vsInstallPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\Enterprise" $arguments = @( @@ -63,4 +63,4 @@ elseif (($process.ExitCode -eq 5004) -or ($process.ExitCode -eq 1602)) { } elseif ($process.ExitCode -ne 0) { Write-Error "Installation failed on $(hostname) for unknown reason" -} \ No newline at end of file +} diff --git a/eng/scripts/cibuild.cmd b/eng/scripts/cibuild.cmd new file mode 100644 index 0000000000..b1048279a3 --- /dev/null +++ b/eng/scripts/cibuild.cmd @@ -0,0 +1,3 @@ +@ECHO OFF +SET RepoRoot=%~dp0..\.. +%RepoRoot%\build.cmd -ci -all -restore -build -pack -test -sign %* diff --git a/eng/scripts/cibuild.sh b/eng/scripts/cibuild.sh new file mode 100755 index 0000000000..6eab1faa2f --- /dev/null +++ b/eng/scripts/cibuild.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +repo_root="$DIR/../.." +"$repo_root/build.sh" --ci --all --restore --build --pack --test "$@" diff --git a/scripts/VsRequirements/vs.json b/eng/scripts/vs.json similarity index 100% rename from scripts/VsRequirements/vs.json rename to eng/scripts/vs.json diff --git a/run.cmd b/restore.cmd similarity index 71% rename from run.cmd rename to restore.cmd index d52d5c7e68..e6275ff5d1 100644 --- a/run.cmd +++ b/restore.cmd @@ -1,2 +1,2 @@ @ECHO OFF -PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0run.ps1' %*; exit $LASTEXITCODE" +PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' -all -restore:$true -build:$False %*; exit $LASTEXITCODE" diff --git a/restore.sh b/restore.sh new file mode 100644 index 0000000000..0dee0a4a03 --- /dev/null +++ b/restore.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +repo_root="$DIR/../.." +"$repo_root/build.sh" --all --restore --no-build "$@" diff --git a/run.ps1 b/run.ps1 deleted file mode 100644 index aa08c6485b..0000000000 --- a/run.ps1 +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env powershell -#requires -version 4 - -<# -.SYNOPSIS -Executes KoreBuild commands. - -.DESCRIPTION -Downloads korebuild if required. Then executes the KoreBuild command. To see available commands, execute with `-Command help`. - -.PARAMETER Command -The KoreBuild command to run. - -.PARAMETER Path -The folder to build. Defaults to the folder containing this script. - -.PARAMETER LockFile -The path to the korebuild-lock.txt file. Defaults to $Path/korebuild-lock.txt - -.PARAMETER Channel -The channel of KoreBuild to download. Overrides the value from the config file. - -.PARAMETER DotNetHome -The directory where .NET Core tools will be stored. - -.PARAMETER ToolsSource -The base url where build tools can be downloaded. Overrides the value from the config file. - -.PARAMETER Update -Updates KoreBuild to the latest version even if a lock file is present. - -.PARAMETER Reinstall -Re-installs KoreBuild - -.PARAMETER ConfigFile -The path to the configuration file that stores values. Defaults to korebuild.json. - -.PARAMETER CI -Sets up CI specific settings and variables. - -.PARAMETER MSBuildArguments -Additional MSBuild arguments to be passed through. - -.NOTES -This function will create a file $PSScriptRoot/korebuild-lock.txt. This lock file can be committed to source, but does not have to be. -When the lockfile is not present, KoreBuild will create one using latest available version from $Channel. - -The $ConfigFile is expected to be an JSON file. It is optional, and the configuration values in it are optional as well. Any options set -in the file are overridden by command line parameters. - -.EXAMPLE -Example config file: -```json -{ - "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/master/tools/korebuild.schema.json", - "channel": "master", - "toolsSource": "https://aspnetcore.blob.core.windows.net/buildtools" -} -``` -#> -[CmdletBinding(PositionalBinding = $false)] -param( - [Parameter(Mandatory=$true, Position = 0)] - [string]$Command, - [string]$Path = $PSScriptRoot, - [string]$LockFile, - [Alias('c')] - [string]$Channel, - [Alias('d')] - [string]$DotNetHome, - [Alias('s')] - [string]$ToolsSource, - [Alias('u')] - [switch]$Update, - [switch]$Reinstall, - [string]$ConfigFile = $null, - [switch]$CI, - [Parameter(ValueFromRemainingArguments = $true)] - [string[]]$MSBuildArguments -) - -Set-StrictMode -Version 2 -$ErrorActionPreference = 'Stop' - -# -# Functions -# - -function Get-KoreBuild { - - if (!(Test-Path $LockFile) -or $Update) { - Get-RemoteFile "$ToolsSource/korebuild/channels/$Channel/latest.txt" $LockFile - } - - $version = Get-Content $LockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1 - if (!$version) { - Write-Error "Failed to parse version from $LockFile. Expected a line that begins with 'version:'" - } - $version = $version.TrimStart('version:').Trim() - $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version) - - if ($Reinstall -and (Test-Path $korebuildPath)) { - Remove-Item -Force -Recurse $korebuildPath - } - - if (!(Test-Path $korebuildPath)) { - Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version" - New-Item -ItemType Directory -Path $korebuildPath | Out-Null - $remotePath = "$ToolsSource/korebuild/artifacts/$version/korebuild.$version.zip" - - try { - $tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip" - Get-RemoteFile $remotePath $tmpfile - if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) { - # Use built-in commands where possible as they are cross-plat compatible - Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath - } - else { - # Fallback to old approach for old installations of PowerShell - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($tmpfile, $korebuildPath) - } - } - catch { - Remove-Item -Recurse -Force $korebuildPath -ErrorAction Ignore - throw - } - finally { - Remove-Item $tmpfile -ErrorAction Ignore - } - } - - return $korebuildPath -} - -function Join-Paths([string]$path, [string[]]$childPaths) { - $childPaths | ForEach-Object { $path = Join-Path $path $_ } - return $path -} - -function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) { - if ($RemotePath -notlike 'http*') { - Copy-Item $RemotePath $LocalPath - return - } - - $retries = 10 - while ($retries -gt 0) { - $retries -= 1 - try { - Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath - return - } - catch { - Write-Verbose "Request failed. $retries retries remaining" - } - } - - Write-Error "Download failed: '$RemotePath'." -} - -# -# Main -# - -# Load configuration or set defaults - -$Path = Resolve-Path $Path -if (!$ConfigFile) { $ConfigFile = Join-Path $Path 'korebuild.json' } - -if (Test-Path $ConfigFile) { - try { - $config = Get-Content -Raw -Encoding UTF8 -Path $ConfigFile | ConvertFrom-Json - if ($config) { - if (!($Channel) -and (Get-Member -Name 'channel' -InputObject $config)) { [string] $Channel = $config.channel } - if (!($ToolsSource) -and (Get-Member -Name 'toolsSource' -InputObject $config)) { [string] $ToolsSource = $config.toolsSource} - } - } catch { - Write-Warning "$ConfigFile could not be read. Its settings will be ignored." - Write-Warning $Error[0] - } -} - -if (!$DotNetHome) { - $DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } ` - elseif ($CI) { Join-Path $PSScriptRoot '.dotnet' } ` - elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} ` - elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}` - else { Join-Path $PSScriptRoot '.dotnet'} -} - -if (!$LockFile) { $LockFile = Join-Path $Path 'korebuild-lock.txt' } -if (!$Channel) { $Channel = 'master' } -if (!$ToolsSource) { $ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' } - -# Execute - -$korebuildPath = Get-KoreBuild -Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1') - -try { - Set-KoreBuildSettings -ToolsSource $ToolsSource -DotNetHome $DotNetHome -RepoPath $Path -ConfigFile $ConfigFile -CI:$CI - Invoke-KoreBuildCommand $Command @MSBuildArguments -} -finally { - Remove-Module 'KoreBuild' -ErrorAction Ignore -} diff --git a/run.sh b/run.sh deleted file mode 100755 index e23cd4620a..0000000000 --- a/run.sh +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# -# variables -# - -RESET="\033[0m" -RED="\033[0;31m" -YELLOW="\033[0;33m" -MAGENTA="\033[0;95m" -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -verbose=false -update=false -reinstall=false -repo_path="$DIR" -lockfile_path='' -channel='' -tools_source='' -ci=false -msbuild_args=() - -# -# Functions -# -__usage() { - echo "Usage: $(basename "${BASH_SOURCE[0]}") command [options] [[--] ...]" - echo "" - echo "Arguments:" - echo " command The command to be run." - echo " ... Arguments passed to the command. Variable number of arguments allowed." - echo "" - echo "Options:" - echo " --verbose Show verbose output." - echo " -c|--channel The channel of KoreBuild to download. Overrides the value from the config file.." - echo " --config-file The path to the configuration file that stores values. Defaults to korebuild.json." - echo " -d|--dotnet-home The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet." - echo " --path The directory to build. Defaults to the directory containing the script." - echo " --lockfile The path to the korebuild-lock.txt file. Defaults to \$repo_path/korebuild-lock.txt" - echo " -s|--tools-source|-ToolsSource The base url where build tools can be downloaded. Overrides the value from the config file." - echo " -u|--update Update to the latest KoreBuild even if the lock file is present." - echo " --reinstall Reinstall KoreBuild." - echo " --ci Apply CI specific settings and environment variables." - echo "" - echo "Description:" - echo " This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be." - echo " When the lockfile is not present, KoreBuild will create one using latest available version from \$channel." - - if [[ "${1:-}" != '--no-exit' ]]; then - exit 2 - fi -} - -get_korebuild() { - local version - if [ ! -f "$lockfile_path" ] || [ "$update" = true ]; then - __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" "$lockfile_path" - fi - version="$(grep 'version:*' -m 1 "$lockfile_path")" - if [[ "$version" == '' ]]; then - __error "Failed to parse version from $lockfile_path. Expected a line that begins with 'version:'" - return 1 - fi - version="$(echo "${version#version:}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" - local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version" - - if [ "$reinstall" = true ] && [ -d "$korebuild_path" ]; then - rm -rf "$korebuild_path" - fi - - { - if [ ! -d "$korebuild_path" ]; then - mkdir -p "$korebuild_path" - local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip" - tmpfile="$(mktemp)" - echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}" - if __get_remote_file "$remote_path" "$tmpfile"; then - unzip -q -d "$korebuild_path" "$tmpfile" - fi - rm "$tmpfile" || true - fi - - source "$korebuild_path/KoreBuild.sh" - } || { - if [ -d "$korebuild_path" ]; then - echo "Cleaning up after failed installation" - rm -rf "$korebuild_path" || true - fi - return 1 - } -} - -__error() { - echo -e "${RED}error: $*${RESET}" 1>&2 -} - -__warn() { - echo -e "${YELLOW}warning: $*${RESET}" -} - -__machine_has() { - hash "$1" > /dev/null 2>&1 - return $? -} - -__get_remote_file() { - local remote_path=$1 - local local_path=$2 - - if [[ "$remote_path" != 'http'* ]]; then - cp "$remote_path" "$local_path" - return 0 - fi - - local failed=false - if __machine_has wget; then - wget --tries 10 --quiet -O "$local_path" "$remote_path" || failed=true - else - failed=true - fi - - if [ "$failed" = true ] && __machine_has curl; then - failed=false - curl --retry 10 -sSL -f --create-dirs -o "$local_path" "$remote_path" || failed=true - fi - - if [ "$failed" = true ]; then - __error "Download failed: $remote_path" 1>&2 - return 1 - fi -} - -# -# main -# - -command="${1:-}" -shift - -while [[ $# -gt 0 ]]; do - case $1 in - -\?|-h|--help) - __usage --no-exit - exit 0 - ;; - -c|--channel|-Channel) - shift - channel="${1:-}" - [ -z "$channel" ] && __error "Missing value for parameter --channel" && __usage - ;; - --config-file|-ConfigFile) - shift - config_file="${1:-}" - [ -z "$config_file" ] && __error "Missing value for parameter --config-file" && __usage - if [ ! -f "$config_file" ]; then - __error "Invalid value for --config-file. $config_file does not exist." - exit 1 - fi - ;; - -d|--dotnet-home|-DotNetHome) - shift - DOTNET_HOME="${1:-}" - [ -z "$DOTNET_HOME" ] && __error "Missing value for parameter --dotnet-home" && __usage - ;; - --path|-Path) - shift - repo_path="${1:-}" - [ -z "$repo_path" ] && __error "Missing value for parameter --path" && __usage - ;; - --[Ll]ock[Ff]ile) - shift - lockfile_path="${1:-}" - [ -z "$lockfile_path" ] && __error "Missing value for parameter --lockfile" && __usage - ;; - -s|--tools-source|-ToolsSource) - shift - tools_source="${1:-}" - [ -z "$tools_source" ] && __error "Missing value for parameter --tools-source" && __usage - ;; - -u|--update|-Update) - update=true - ;; - --reinstall|-Reinstall) - reinstall=true - ;; - --ci|-[Cc][Ii]) - ci=true - if [[ -z "${DOTNET_HOME:-}" ]]; then - DOTNET_HOME="$DIR/.dotnet" - fi - ;; - --verbose|-Verbose) - verbose=true - ;; - *) - msbuild_args[${#msbuild_args[*]}]="$1" - ;; - esac - shift -done - -if ! __machine_has unzip; then - __error 'Missing required command: unzip' - exit 1 -fi - -if ! __machine_has curl && ! __machine_has wget; then - __error 'Missing required command. Either wget or curl is required.' - exit 1 -fi - -[ -z "${config_file:-}" ] && config_file="$repo_path/korebuild.json" -if [ -f "$config_file" ]; then - if __machine_has jq ; then - if jq '.' "$config_file" >/dev/null ; then - config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")" - config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")" - else - __error "$config_file is invalid JSON. Its settings will be ignored." - exit 1 - fi - elif __machine_has python ; then - if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then - config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")" - config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")" - else - __error "$config_file is invalid JSON. Its settings will be ignored." - exit 1 - fi - else - __error 'Missing required command: jq or python. Could not parse the JSON file. Its settings will be ignored.' - exit 1 - fi - - [ ! -z "${config_channel:-}" ] && channel="$config_channel" - [ ! -z "${config_tools_source:-}" ] && tools_source="$config_tools_source" -fi - -[ -z "${DOTNET_HOME:-}" ] && DOTNET_HOME="$HOME/.dotnet" -[ -z "$lockfile_path" ] && lockfile_path="$repo_path/korebuild-lock.txt" -[ -z "$channel" ] && channel='master' -[ -z "$tools_source" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools' - -get_korebuild -set_korebuildsettings "$tools_source" "$DOTNET_HOME" "$repo_path" "$config_file" "$ci" - -# This incantation avoids unbound variable issues if msbuild_args is empty -# https://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u -invoke_korebuild_command "$command" ${msbuild_args[@]+"${msbuild_args[@]}"} diff --git a/scripts/Tag-TeamCityBuild.ps1 b/scripts/Tag-TeamCityBuild.ps1 deleted file mode 100755 index d624d4f50c..0000000000 --- a/scripts/Tag-TeamCityBuild.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS - Tags the given TeamCity build with the given tag. -.PARAMETER BuildId - The BuildId of the build to be tagged. -.PARAMETER Tag - The tag to put on this build. -#> - -[cmdletbinding(SupportsShouldProcess = $true)] -param( - [Parameter(Mandatory = $true)] - [string]$BuildId, - [Parameter(Mandatory = $true)] - [string]$Tag, - [Parameter(Mandatory = $true)] - [string]$UserName, - [Parameter(Mandatory = $true)] - [string]$Password -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2 - -$authInfo = "${UserName}:$Password" -$authEncoded = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes($authInfo)) -$basicAuthValue = "Basic $authEncoded" - -$headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]" -$headers.Add("Authorization", $basicAuthValue) -$headers.Add("Content-Type", "text/plain") - -$uri = "http://aspnetci/app/rest/builds/$BuildId/tags/" - -Invoke-WebRequest -Uri $uri -Method 'POST' -Headers $headers -Body $Tag -ErrorAction SilentlyContinue diff --git a/scripts/UpdateBuildTools.ps1 b/scripts/UpdateBuildTools.ps1 deleted file mode 100755 index efcda1e03b..0000000000 --- a/scripts/UpdateBuildTools.ps1 +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS - Updates the build tools version and generates a commit message with the list of changes -.PARAMETER RepoRoot - The directory containing the repo -.PARAMETER GitAuthorName - The author name to use in the commit message. (Optional) -.PARAMETER GitAuthorEmail - The author email to use in the commit message. (Optional) -.PARAMETER GitCommitArgs - Additional arguments to pass into git-commit -.PARAMETER NoCommit - Make changes without executing git-commit -.PARAMETER ToolsSource - The location of the build tools -.PARAMETER Force - Specified this to make a commit with any changes -#> -[cmdletbinding(SupportsShouldProcess = $true)] -param( - [string]$RepoRoot, - [string]$GitAuthorName = $null, - [string]$GitAuthorEmail = $null, - [string[]]$GitCommitArgs = @(), - [string]$ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools', - [switch]$NoCommit, - [switch]$Force -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2 - -if (-not $RepoRoot) { - $RepoRoot = Resolve-Path "$PSScriptRoot\.." -} - -Import-Module "$PSScriptRoot/common.psm1" -Scope Local -Force - -function Get-KoreBuildVersion { - $lockFile = "$RepoRoot/korebuild-lock.txt" - if (!(Test-Path $lockFile)) { - return '' - } - $version = Get-Content $lockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1 - if (!$version) { - Write-Error "Failed to parse version from $lockFile. Expected a line that begins with 'version:'" - } - $version = $version.TrimStart('version:').Trim() - return $version -} - -Push-Location $RepoRoot -try { - Assert-Git - - $oldVersion = Get-KoreBuildVersion - - & "$RepoRoot/run.ps1" -Update -ToolsSource $ToolsSource -Command noop | Out-Null - - $newVersion = Get-KoreBuildVersion - - if ($oldVersion -eq $newVersion) { - Write-Host -ForegroundColor Magenta 'No changes to build tools' - exit 0 - } - - Invoke-Block { git add "$RepoRoot/global.json" } - Invoke-Block { git add "$RepoRoot/korebuild-lock.txt" } - Invoke-Block { git add "$RepoRoot/build/dependencies.props" } - - $shortMessage = "Updating BuildTools from $oldVersion to $newVersion" - # add this to the commit message to make it possible to filter commit triggers based on message - $message = "$shortMessage`n`n[auto-updated: buildtools]" - - if (-not $NoCommit -and ($Force -or ($PSCmdlet.ShouldContinue($shortMessage, 'Create a new commit with these changes?')))) { - - $gitConfigArgs = @() - if ($GitAuthorName) { - $gitConfigArgs += '-c', "user.name=$GitAuthorName" - } - - if ($GitAuthorEmail) { - $gitConfigArgs += '-c', "user.email=$GitAuthorEmail" - } - - Invoke-Block { git @gitConfigArgs commit -m $message @GitCommitArgs } - } - else { - # If composing this script with others, return the message that would have been used - return @{ - message = $message - } - } -} -finally { - Pop-Location -} diff --git a/scripts/UpdateDependencies.ps1 b/scripts/UpdateDependencies.ps1 deleted file mode 100755 index 6adaea833e..0000000000 --- a/scripts/UpdateDependencies.ps1 +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env pwsh -c -<# -.PARAMETER BuildXml - The URL or file path to a build.xml file that defines package versions to be used -#> -[CmdletBinding()] -param( - [Parameter(Mandatory = $true)] - $BuildXml, - [switch]$NoCommit, - [string]$GithubUpstreamBranch, - [string]$GithubEmail, - [string]$GithubUsername, - [string]$GithubToken -) - -$ErrorActionPreference = 'Stop' -Import-Module -Scope Local -Force "$PSScriptRoot/common.psm1" -Set-StrictMode -Version 1 -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -if (-not $NoCommit) { - Set-GitHubInfo $GithubToken $GithubUsername $GithubEmail -} - -$depsPath = Resolve-Path "$PSScriptRoot/../build/dependencies.props" -[xml] $dependencies = LoadXml $depsPath - -if ($BuildXml -like 'http*') { - $url = $BuildXml - New-Item -Type Directory "$PSScriptRoot/../obj/" -ErrorAction Ignore - $BuildXml = "$PSScriptRoot/../obj/build.xml" - Write-Verbose "Downloading from $url to $BuildXml" - Invoke-WebRequest -OutFile $BuildXml $url -} - -[xml] $remoteDeps = LoadXml $BuildXml - -$variables = @{} - -foreach ($package in $remoteDeps.SelectNodes('//Package')) { - $packageId = $package.Id - $packageVersion = $package.Version - $varName = PackageIdVarName $packageId - Write-Verbose "Found {id: $packageId, version: $packageVersion, varName: $varName }" - - if ($variables[$varName]) { - if ($variables[$varName].Where( {$_ -eq $packageVersion}, 'First').Count -eq 0) { - $variables[$varName] += $packageVersion - } - } - else { - $variables[$varName] = @($packageVersion) - } -} - -if (-not $NoCommit) { - $currentBranch = Invoke-Block { & git rev-parse --abbrev-ref HEAD } - $destinationBranch = "dotnetbot/UpdateDeps" - Invoke-Block { & git checkout -tb $destinationBranch "origin/$GithubUpstreamBranch" } -} - -try { - $updatedVars = UpdateVersions $variables $dependencies $depsPath - - if ($NoCommit) { - exit 0 - } - - $body = CommitUpdatedVersions $updatedVars $dependencies $depsPath - - if ($body) { - CreatePR "aspnet" $GithubUsername $GithubUpstreamBranch $destinationBranch $body $GithubToken - } -} -finally { - if (-not $NoCommit) { - Invoke-Block { & git checkout $currentBranch } - } -} diff --git a/scripts/UpdateSubmodules.ps1 b/scripts/UpdateSubmodules.ps1 deleted file mode 100755 index 11058328f0..0000000000 --- a/scripts/UpdateSubmodules.ps1 +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env pwsh -c - -<# -.SYNOPSIS - Updates git submodules and generates a commit message with the list of changes -.PARAMETER GitAuthorName - The author name to use in the commit message. (Optional) -.PARAMETER GitAuthorEmail - The author email to use in the commit message. (Optional) -.PARAMETER GitCommitArgs - Additional arguments to pass into git-commit -.PARAMETER NoCommit - Make changes without executing git-commit -.PARAMETER Force - Specified this to make a commit with any changes -.PARAMETER IgnoredRepos - Repos to not update (likely because they are temporarily broken). -#> -[cmdletbinding(SupportsShouldProcess = $true)] -param( - [string]$GitAuthorName = $null, - [string]$GitAuthorEmail = $null, - [string[]]$GitCommitArgs = @(), - [switch]$NoCommit, - [switch]$Force, - [string[]]$IgnoredRepos = @() -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2 - -$RepoRoot = Resolve-Path "$PSScriptRoot\.." -$ModuleDirectory = Join-Path $RepoRoot "modules" - -Import-Module "$PSScriptRoot/common.psm1" -Scope Local -Force - -function Get-GitChanges([string]$Path) { - Write-Verbose "git diff --cached --quiet $Path" - & git diff --cached --quiet $Path | Out-Null - if ($LastExitCode -ne 0) { - return $true - } - Write-Verbose "git diff --quiet $Path" - & git diff --quiet $Path | Out-Null - return $LastExitCode -ne 0 -} - -Push-Location $RepoRoot | Out-Null -try { - Assert-Git - - Write-Host "Checking that submodules are in a clean state first..." - if (Get-GitChanges $ModuleDirectory) { - Write-Error "$RepoRoot/modules is in an unclean state. Reset submodules first by running ``git submodule update``" - exit 1 - } - - $submodules = Get-Submodules $RepoRoot -Verbose:$VerbosePreference - - foreach ($submodule in $submodules) { - $submoduleName = $submodule.module - if ($IgnoredRepos.Contains($submoduleName)) - { - Write-Host "Skipping $submoduleName due to IgnoredRepos." - continue - } - - $submodulePath = $submodule.path - Write-Host "Updating $submodulePath" - - $vcs_name = "BUILD_VCS_NUMBER_" + ($submodule.module -replace '\.','_') - $newCommit = [environment]::GetEnvironmentVariable($vcs_name) - - if (-not $newCommit) { - if ($env:TEAMCITY_PROJECT_NAME) { - throw "TeamCity env variable '$vcs_name' not found. Make sure to configure a VCS root for $submodulePath" - } - Invoke-Block { & git submodule update --remote $submodulePath } - Push-Location $submodulePath | Out-Null - try { - $newCommit = $(git rev-parse HEAD) - } - finally { - Pop-Location | Out-Null - } - } - else { - Push-Location $submodulePath | Out-Null - try { - Invoke-Block { & git checkout $newCommit } - } - finally { - Pop-Location | Out-Null - } - } - - $submodule.newCommit = $newCommit - if ($newCommit -ne $submodule.commit) { - $submodule.changed = $true - Write-Host -ForegroundColor Cyan "`t=> $($submodule.module) updated to $($submodule.newCommit)" - } - else { - Write-Host -ForegroundColor Magenta "`t$($submodule.module) did not change" - } - } - - $changes = $submodules ` - | ? { $_.changed } ` - | % { - Invoke-Block { & git add $_.path } - "$($_.module) => $($_.newCommit)" - } - - if ($changes) { - $shortMessage = "Updating submodule(s) `n`n$( $changes -join "`n" )" - # add this to the commit message to make it possible to filter commit triggers based on message - $message = "$shortMessage`n`n[auto-updated: submodules]" - if (-not $NoCommit -and ($Force -or ($PSCmdlet.ShouldContinue($shortMessage, 'Create a new commit with these changes?')))) { - - $gitConfigArgs = @() - if ($GitAuthorName) { - $gitConfigArgs += '-c',"user.name=$GitAuthorName" - } - - if ($GitAuthorEmail) { - $gitConfigArgs += '-c',"user.email=$GitAuthorEmail" - } - - Invoke-Block { & git @gitConfigArgs commit -m $message @GitCommitArgs } - } - else { - # If composing this script with others, return the message that would have been used - return @{ - message = $message - } - } - } - else { - Write-Host -ForegroundColor Magenta 'No changes detected in git submodules' - } -} -finally { - Pop-Location -} diff --git a/scripts/common.psm1 b/scripts/common.psm1 deleted file mode 100644 index c547cba9a9..0000000000 --- a/scripts/common.psm1 +++ /dev/null @@ -1,314 +0,0 @@ -$ErrorActionPreference = 'Stop' -# Update the default TLS support to 1.2 -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -function Assert-Git { - if (!(Get-Command git -ErrorAction Ignore)) { - Write-Error 'git is required to execute this script' - exit 1 - } -} - -function Invoke-Block([scriptblock]$cmd, [string]$WorkingDir = $null) { - if ($WorkingDir) { - Push-Location $WorkingDir - } - - try { - - $cmd | Out-String | Write-Verbose - & $cmd - - # Need to check both of these cases for errors as they represent different items - # - $?: did the powershell script block throw an error - # - $lastexitcode: did a windows command executed by the script block end in error - if ((-not $?) -or ($lastexitcode -ne 0)) { - if ($error -ne $null) - { - Write-Warning $error[0] - } - throw "Command failed to execute: $cmd" - } - } - finally { - if ($WorkingDir) { - Pop-Location - } - } -} - -function Get-Submodules { - param( - [Parameter(Mandatory = $true)] - [string]$RepoRoot, - [switch]$Shipping - ) - - $moduleConfigFile = Join-Path $RepoRoot ".gitmodules" - $submodules = @() - - [xml] $submoduleConfig = Get-Content "$RepoRoot/build/submodules.props" - $repos = $submoduleConfig.Project.ItemGroup.Repository | % { $_.Include } - - Get-ChildItem "$RepoRoot/modules/*" -Directory ` - | ? { (-not $Shipping) -or $($repos -contains $($_.Name)) -or $_.Name -eq 'Templating' } ` - | % { - Push-Location $_ | Out-Null - Write-Verbose "Attempting to get submodule info for $_" - - if (Test-Path 'version.props') { - [xml] $versionXml = Get-Content 'version.props' - $versionPrefix = $versionXml.Project.PropertyGroup.VersionPrefix | select-object -first 1 - $versionSuffix = $versionXml.Project.PropertyGroup.VersionSuffix | select-object -first 1 - } - else { - $versionPrefix = '' - $versionSuffix = '' - } - - try { - $data = [PSCustomObject] @{ - path = $_ - module = $_.Name - commit = $(git rev-parse HEAD) - newCommit = $null - changed = $false - remote = $(git config remote.origin.url) - branch = $(git config -f $moduleConfigFile --get submodule.modules/$($_.Name).branch ) - versionPrefix = $versionPrefix - versionSuffix = $versionSuffix - } - - $submodules += $data - } - finally { - Pop-Location | Out-Null - } - } - - return $submodules -} - -function SaveXml([xml]$xml, [string]$path) { - Write-Verbose "Saving to $path" - $ErrorActionPreference = 'stop' - - $settings = New-Object System.XML.XmlWriterSettings - $settings.OmitXmlDeclaration = $true - $settings.Encoding = New-Object System.Text.UTF8Encoding( $true ) - $writer = [System.XML.XMLTextWriter]::Create($path, $settings) - $xml.Save($writer) - $writer.Close() -} - -function LoadXml([string]$path) { - Write-Verbose "Reading from $path" - - $ErrorActionPreference = 'stop' - $obj = new-object xml - $obj.PreserveWhitespace = $true - $obj.Load($path) - return $obj -} - -function PackageIdVarName([string]$packageId) { - $canonicalVarName = '' - $upperCaseNext = $true - for ($i = 0; $i -lt $packageId.Length; $i++) { - $ch = $packageId[$i] - if (-not [System.Char]::IsLetterOrDigit(($ch))) { - $upperCaseNext = $true - continue - } - if ($upperCaseNext) { - $ch = [System.Char]::ToUpperInvariant($ch) - $upperCaseNext = $false - } - $canonicalVarName += $ch - } - $canonicalVarName += "PackageVersion" - return $canonicalVarName -} - -function Ensure-Hub() { - $tmpDir = "$PSScriptRoot\tmp" - $zipDir = "$tmpDir\Hub" - $hubLocation = "$zipDir\bin\hub.exe" - - if (-Not (Test-Path $hubLocation) ) { - $source = "https://github.com/github/hub/releases/download/v2.3.0-pre9/hub-windows-amd64-2.3.0-pre9.zip" - $zipLocation = "$tmpDir\hub.zip" - - mkdir -Path $tmpDir -ErrorAction Ignore | Out-Null - - Invoke-WebRequest -OutFile $zipLocation -Uri $source - - Expand-Archive -Path $zipLocation -DestinationPath $zipDir -Force - if (-Not (Test-Path $hubLocation)) { - throw "Hub couldn't be downloaded" - } - } - - return $hubLocation -} - -function CreatePR( - [string]$baseFork, - [string]$headFork, - [string]$baseBranch, - [string]$destinationBranch, - [string]$body, - [string]$gitHubToken) { - $hubLocation = Ensure-Hub - - Invoke-Block { git push -f https://$gitHubToken@github.com/$headFork/AspNetCore.git $destinationBranch } - & $hubLocation pull-request -f -b "${baseFork}:$baseBranch" -h "${headFork}:$destinationBranch" -m $body -} - -function Set-GithubInfo( - [string]$GitHubPassword, - [string]$GitHubUser, - [string]$GitHubEmail) -{ - $Env:GITHUB_TOKEN = $GitHubPassword - $Env:GITHUB_USER = $GitHubUser - $Env:GITHUB_EMAIL = $GitHubEmail -} -function CommitUpdatedVersions( - [hashtable]$updatedVars, - [xml]$dependencies, - [string]$depsPath, - [string]$subject = 'Updating external dependencies') -{ - $count = $updatedVars.Count - if ($count -gt 0) { - & git add build\dependencies.props - - $gitConfigArgs = @() - if ($env:GITHUB_USER) { - $gitConfigArgs += '-c',"user.name=$env:GITHUB_USER" - } - - if ($env:GITHUB_EMAIL) { - $gitConfigArgs += '-c',"user.email=$env:GITHUB_EMAIL" - } - - Invoke-Block { & git @gitConfigArgs commit -m $subject } | Out-Null - - $body = "$subject`n`n" - - $body += "New versions:`n" - - foreach ($var in $updatedVars.GetEnumerator()) { - $body += " $($var.Name)`n" - } - - return $body - } -} - -function UpdateVersions([hashtable]$variables, [xml]$dependencies, [string]$depsPath) { - $updatedVars = @{} - - foreach ($varName in ($variables.Keys | sort)) { - $packageVersions = $variables[$varName] - if ($packageVersions.Length -gt 1) { - Write-Warning "Skipped $varName. Multiple version found. { $($packageVersions -join ', ') }." - continue - } - - $packageVersion = $packageVersions | Select-Object -First 1 - - $depVarNode = $dependencies.SelectSingleNode("//PropertyGroup[`@Label=`"Package Versions: Auto`"]/$varName") - if ($depVarNode -and $depVarNode.InnerText -ne $packageVersion) { - $depVarNode.InnerText = $packageVersion - Write-Host -f DarkGray " Updating $varName to $packageVersion" - $updatedVars[$varName] = $packageVersion - } - elseif ($depVarNode) { - Write-Host -f DarkBlue " Didn't update $varName to $packageVersion because it was $($depVarNode.InnerText)" - } - else { - # This isn't a dependency we use - } - } - - if ($updatedVars.Count -gt 0) { - Write-Host -f Cyan "Updating version variables in $depsPath" - SaveXml $dependencies $depsPath - } - else { - Write-Host -f Green "No changes found" - } - - return $updatedVars -} - -function Get-MSBuildPath { - param( - [switch]$Prerelease, - [string[]]$Requires - ) - - $vsInstallDir = $null - if ($env:VSINSTALLDIR -and (Test-Path $env:VSINSTALLDIR)) { - $vsInstallDir = $env:VSINSTALLDIR - Write-Verbose "Using VSINSTALLDIR=$vsInstallDir" - } - else { - $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" - Write-Verbose "Using vswhere.exe from $vswhere" - - if (-not (Test-Path $vswhere)) { - Write-Error "Missing prerequisite: could not find vswhere" - } - - [string[]] $vswhereArgs = @() - - if ($Prerelease) { - $vswhereArgs += '-prerelease' - } - - if ($Requires) { - foreach ($r in $Requires) { - $vswhereArgs += '-requires', $r - } - } - - $installs = & $vswhere -format json -version '[15.0, 16.0)' -latest -products * @vswhereArgs | ConvertFrom-Json - if (!$installs) { - Write-Error "Missing prerequisite: could not find any installations of Visual Studio" - } - - $vs = $installs | Select-Object -First 1 - $vsInstallDir = $vs.installationPath - Write-Host "Using $($vs.displayName)" - } - - $msbuild = Join-Path $vsInstallDir 'MSBuild/15.0/bin/msbuild.exe' - if (!(Test-Path $msbuild)) { - Write-Error "Missing prerequisite: could not find msbuild.exe" - } - return $msbuild -} - -function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) { - if ($RemotePath -notlike 'http*') { - Copy-Item $RemotePath $LocalPath - return - } - - $retries = 10 - while ($retries -gt 0) { - $retries -= 1 - try { - Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath - return - } - catch { - Write-Verbose "Request failed. $retries retries remaining" - } - } - - Write-Error "Download failed: '$RemotePath'." -} diff --git a/src/Components/build.cmd b/src/Components/build.cmd index f4169ea5e4..3b0c1b533a 100644 --- a/src/Components/build.cmd +++ b/src/Components/build.cmd @@ -1,3 +1,3 @@ @ECHO OFF SET RepoRoot="%~dp0..\.." -%RepoRoot%\build.cmd -LockFile %RepoRoot%\korebuild-lock.txt -Path %~dp0 %* +%RepoRoot%\build.cmd -All -RepoRoot %~dp0 %* diff --git a/src/Components/build.sh b/src/Components/build.sh index d5bb0cf631..1e74b5d6f7 100644 --- a/src/Components/build.sh +++ b/src/Components/build.sh @@ -4,4 +4,4 @@ set -euo pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" repo_root="$DIR/../.." -"$repo_root/build.sh" --path "$DIR" --lockfile "$repo_root/korebuild-lock.txt" "$@" +"$repo_root/build.sh" -All -RepoRoot "$DIR" "$@" diff --git a/src/Components/korebuild.json b/src/Components/korebuild.json deleted file mode 100644 index 1770aaf9fc..0000000000 --- a/src/Components/korebuild.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/master/tools/korebuild.schema.json", - "channel": "master", - "toolsets": { - "visualstudio": { - "required": false, - "includePrerelease": true, - "minVersion": "[15.0.26730.03, 16.0)", - "requiredWorkloads": [ - "Microsoft.VisualStudio.Component.VSSDK" - ] - } - } -} diff --git a/src/DataProtection/build.cmd b/src/DataProtection/build.cmd index f4169ea5e4..033fe6f614 100644 --- a/src/DataProtection/build.cmd +++ b/src/DataProtection/build.cmd @@ -1,3 +1,3 @@ @ECHO OFF -SET RepoRoot="%~dp0..\.." -%RepoRoot%\build.cmd -LockFile %RepoRoot%\korebuild-lock.txt -Path %~dp0 %* +SET RepoRoot=%~dp0..\.. +%RepoRoot%\build.cmd -projects %~dp0\**\*.*proj %* diff --git a/src/DataProtection/build.sh b/src/DataProtection/build.sh index d5bb0cf631..7046bb98a0 100755 --- a/src/DataProtection/build.sh +++ b/src/DataProtection/build.sh @@ -4,4 +4,4 @@ set -euo pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" repo_root="$DIR/../.." -"$repo_root/build.sh" --path "$DIR" --lockfile "$repo_root/korebuild-lock.txt" "$@" +"$repo_root/build.sh" --projects "$DIR/**/*.*proj" "$@" diff --git a/src/Mvc/build.cmd b/src/Mvc/build.cmd index 365624833c..e5b3c66518 100644 --- a/src/Mvc/build.cmd +++ b/src/Mvc/build.cmd @@ -1,3 +1,3 @@ @ECHO OFF SET RepoRoot="%~dp0..\.." -%RepoRoot%\build.cmd -LockFile %RepoRoot%\korebuild-lock.txt -Path %~dp0 %* \ No newline at end of file +%RepoRoot%\build.cmd -All -RepoRoot %~dp0 %* \ No newline at end of file diff --git a/src/Mvc/build.sh b/src/Mvc/build.sh index 5af0f7e1de..40bec9e64b 100644 --- a/src/Mvc/build.sh +++ b/src/Mvc/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail - + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" repo_root="$DIR/../.." -"$repo_root/build.sh" --path "$DIR" --lockfile "$repo_root/korebuild-lock.txt" "$@" \ No newline at end of file +"$repo_root/build.sh" -All -RepoRoot "$DIR" "$@" \ No newline at end of file diff --git a/src/Mvc/src/Microsoft.Extensions.ApiDescription.Design/Microsoft.Extensions.ApiDescription.Design.nuspec b/src/Mvc/src/Microsoft.Extensions.ApiDescription.Design/Microsoft.Extensions.ApiDescription.Design.nuspec index e4b8130e5b..e9867f8691 100644 --- a/src/Mvc/src/Microsoft.Extensions.ApiDescription.Design/Microsoft.Extensions.ApiDescription.Design.nuspec +++ b/src/Mvc/src/Microsoft.Extensions.ApiDescription.Design/Microsoft.Extensions.ApiDescription.Design.nuspec @@ -1,6 +1,6 @@  - + $id$ $authors$ $copyright$ @@ -8,7 +8,6 @@ true $iconUrl$ $licenseUrl$ - 2.8 $owners$ $projectUrl$ diff --git a/src/Razor/build.cmd b/src/Razor/build.cmd new file mode 100644 index 0000000000..033fe6f614 --- /dev/null +++ b/src/Razor/build.cmd @@ -0,0 +1,3 @@ +@ECHO OFF +SET RepoRoot=%~dp0..\.. +%RepoRoot%\build.cmd -projects %~dp0\**\*.*proj %* diff --git a/src/MusicStore/build.sh b/src/Razor/build.sh similarity index 58% rename from src/MusicStore/build.sh rename to src/Razor/build.sh index d5bb0cf631..7046bb98a0 100755 --- a/src/MusicStore/build.sh +++ b/src/Razor/build.sh @@ -4,4 +4,4 @@ set -euo pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" repo_root="$DIR/../.." -"$repo_root/build.sh" --path "$DIR" --lockfile "$repo_root/korebuild-lock.txt" "$@" +"$repo_root/build.sh" --projects "$DIR/**/*.*proj" "$@" diff --git a/src/Razor/korebuild.json b/src/Razor/korebuild.json deleted file mode 100644 index b25cdff3c4..0000000000 --- a/src/Razor/korebuild.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/master/tools/korebuild.schema.json", - "channel": "master", - "toolsets": { - "visualstudio": { - "required": false, - "includePrerelease": true, - "versionRange": "[15.0.26730.03, 16.0)", - "requiredWorkloads": [ - "Microsoft.VisualStudio.Component.VSSDK" - ] - } - } -} diff --git a/src/Servers/Directory.Build.props b/src/Servers/Directory.Build.props deleted file mode 100644 index 36ab489915..0000000000 --- a/src/Servers/Directory.Build.props +++ /dev/null @@ -1,12 +0,0 @@ - - - - - $(RepositoryRoot)obj\$(MSBuildProjectName)\ - $(RepositoryRoot)bin\$(MSBuildProjectName)\ - - - - - - diff --git a/src/Servers/IIS/ResolveIisReferences.targets b/src/Servers/IIS/ResolveIisReferences.targets index 11b7c2a16c..eac3356826 100644 --- a/src/Servers/IIS/ResolveIisReferences.targets +++ b/src/Servers/IIS/ResolveIisReferences.targets @@ -20,7 +20,7 @@ with the right MSBuild incantations to get output copied to the right place. - + @@ -71,6 +71,12 @@ with the right MSBuild incantations to get output copied to the right place. + + + + diff --git a/src/Servers/IIS/build.cmd b/src/Servers/IIS/build.cmd new file mode 100644 index 0000000000..bc82307787 --- /dev/null +++ b/src/Servers/IIS/build.cmd @@ -0,0 +1,3 @@ +@ECHO OFF +SET RepoRoot=%~dp0..\..\.. +%RepoRoot%\build.cmd -projects %~dp0\**\*.*proj %* diff --git a/src/Servers/Kestrel/build.cmd b/src/Servers/Kestrel/build.cmd new file mode 100644 index 0000000000..bc82307787 --- /dev/null +++ b/src/Servers/Kestrel/build.cmd @@ -0,0 +1,3 @@ +@ECHO OFF +SET RepoRoot=%~dp0..\..\.. +%RepoRoot%\build.cmd -projects %~dp0\**\*.*proj %* diff --git a/src/Servers/Kestrel/build.sh b/src/Servers/Kestrel/build.sh new file mode 100755 index 0000000000..6fc2eec307 --- /dev/null +++ b/src/Servers/Kestrel/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +repo_root="$DIR/../.." +"$repo_root/build.sh" --all --projects "$DIR/**/*.*proj" "$@" diff --git a/src/SignalR/build.cmd b/src/SignalR/build.cmd index f4169ea5e4..3918bceb34 100644 --- a/src/SignalR/build.cmd +++ b/src/SignalR/build.cmd @@ -1,3 +1,3 @@ @ECHO OFF SET RepoRoot="%~dp0..\.." -%RepoRoot%\build.cmd -LockFile %RepoRoot%\korebuild-lock.txt -Path %~dp0 %* +%RepoRoot%\build.cmd -RepoRoot %~dp0 %* diff --git a/src/SignalR/build.sh b/src/SignalR/build.sh index d5bb0cf631..1e74b5d6f7 100755 --- a/src/SignalR/build.sh +++ b/src/SignalR/build.sh @@ -4,4 +4,4 @@ set -euo pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" repo_root="$DIR/../.." -"$repo_root/build.sh" --path "$DIR" --lockfile "$repo_root/korebuild-lock.txt" "$@" +"$repo_root/build.sh" -All -RepoRoot "$DIR" "$@" diff --git a/src/SignalR/korebuild.json b/src/SignalR/korebuild.json deleted file mode 100644 index ce2e15a055..0000000000 --- a/src/SignalR/korebuild.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/master/tools/korebuild.schema.json", - "channel": "master", - "toolsets": { - "nodejs": { - "required": true, - "minVersion": "8.0" - } - } -} diff --git a/src/SiteExtensions/LoggingAggregate/build.cmd b/src/SiteExtensions/LoggingAggregate/build.cmd index f4169ea5e4..3b0c1b533a 100644 --- a/src/SiteExtensions/LoggingAggregate/build.cmd +++ b/src/SiteExtensions/LoggingAggregate/build.cmd @@ -1,3 +1,3 @@ @ECHO OFF SET RepoRoot="%~dp0..\.." -%RepoRoot%\build.cmd -LockFile %RepoRoot%\korebuild-lock.txt -Path %~dp0 %* +%RepoRoot%\build.cmd -All -RepoRoot %~dp0 %* diff --git a/src/SiteExtensions/LoggingAggregate/build.sh b/src/SiteExtensions/LoggingAggregate/build.sh index d5bb0cf631..7923286d9d 100644 --- a/src/SiteExtensions/LoggingAggregate/build.sh +++ b/src/SiteExtensions/LoggingAggregate/build.sh @@ -4,4 +4,4 @@ set -euo pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" repo_root="$DIR/../.." -"$repo_root/build.sh" --path "$DIR" --lockfile "$repo_root/korebuild-lock.txt" "$@" +"$repo_root/build.sh" -RepoRoot "$DIR" "$@"