diff --git a/build.ps1 b/build.ps1 index b7081bc1c2..9c8dda267e 100644 --- a/build.ps1 +++ b/build.ps1 @@ -23,6 +23,9 @@ The base url where build tools can be downloaded. Overrides the value from the c .PARAMETER Update Updates KoreBuild to the latest version even if a lock file is present. +.PARAMETER Reinstall +Re-installs KoreBuild + .PARAMETER ConfigFile The path to the configuration file that stores values. Defaults to version.props. @@ -57,6 +60,7 @@ param( [string]$ToolsSource, [Alias('u')] [switch]$Update, + [switch]$Reinstall, [string]$ConfigFile = $null, [Parameter(ValueFromRemainingArguments = $true)] [string[]]$MSBuildArgs @@ -84,6 +88,10 @@ function Get-KoreBuild { $version = $version.TrimStart('version:').Trim() $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version) + if ($Reinstall -and (Test-Path $korebuildPath)) { + Remove-Item -Force -Recurse $korebuildPath + } + if (!(Test-Path $korebuildPath)) { Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version" New-Item -ItemType Directory -Path $korebuildPath | Out-Null @@ -103,11 +111,11 @@ function Get-KoreBuild { } } catch { - remove-item -Recurse -Force $korebuildPath -ErrorAction Ignore + Remove-Item -Recurse -Force $korebuildPath -ErrorAction Ignore throw } finally { - remove-item $tmpfile -ErrorAction Ignore + Remove-Item $tmpfile -ErrorAction Ignore } } diff --git a/build.sh b/build.sh index 14d84a8773..77169b3d91 100755 --- a/build.sh +++ b/build.sh @@ -15,6 +15,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" config_file="$DIR/korebuild.json" verbose=false update=false +reinstall=false repo_path="$DIR" channel='' tools_source='' @@ -36,6 +37,7 @@ __usage() { echo " --path The directory to build. Defaults to the directory containing the script." echo " -s|--tools-source The base url where build tools can be downloaded. Overrides the value from the config file." echo " -u|--update Update to the latest KoreBuild even if the lock file is present." + echo " --reinstall Reinstall KoreBuild." echo "" echo "Description:" echo " This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be." @@ -60,6 +62,10 @@ get_korebuild() { version="$(echo "${version#version:}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version" + if [ "$reinstall" = true ] && [ -d "$korebuild_path" ]; then + rm -rf "$korebuild_path" + fi + { if [ ! -d "$korebuild_path" ]; then mkdir -p "$korebuild_path" @@ -164,6 +170,9 @@ while [[ $# -gt 0 ]]; do -u|--update|-Update) update=true ;; + --reinstall|-[Rr]einstall) + reinstall=true + ;; --verbose|-Verbose) verbose=true ;; diff --git a/build/PackageArchive.targets b/build/PackageArchive.targets index 7688976518..b938f13938 100644 --- a/build/PackageArchive.targets +++ b/build/PackageArchive.targets @@ -34,6 +34,11 @@ + + + + + + Properties="RestorePackagesPath=$(FallbackStagingDir);RuntimeFrameworkVersion=$(LZMAMicrosoftNETCoreApp20PackageVersion);DotNetRestoreSourcePropsPath=$(GeneratedFallbackRestoreSourcesPropsPath);AspNetUniverseBuildOffline=true" /> diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets index 9c5cd93425..e46c3a157b 100644 --- a/build/RepositoryBuild.targets +++ b/build/RepositoryBuild.targets @@ -43,10 +43,11 @@ $(RepositoryBuildArguments) /p:DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath) $(RepositoryBuildArguments) /p:DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath) - $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) + $(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) + $(RepositoryBuildArguments) /p:Configuration=$(Configuration) $(RepositoryBuildArguments) /noconsolelogger '/l:RepoTasks.FlowLogger,$(MSBuildThisFileDirectory)tasks\bin\publish\RepoTasks.dll;Summary;FlowId=$(RepositoryToBuild)' - $(_RepositoryBuildTargets) $(RepositoryBuildArguments) + /t:CleanArtifacts $(_RepositoryBuildTargets) $(RepositoryBuildArguments) $(BuildRepositoryRoot)artifacts $(RepositoryArtifactsRoot)\build\ $(RepositoryArtifactsRoot)\msbuild\ diff --git a/build/RuntimeStore.targets b/build/RuntimeStore.targets index c369b30a0a..5fc3b44466 100644 --- a/build/RuntimeStore.targets +++ b/build/RuntimeStore.targets @@ -28,11 +28,17 @@ + + + + + Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath);AspNetUniverseBuildOffline=true;_Target=Restore" /> + Properties="Configuration=$(Configuration);BuildNumber=$(BuildNumber);DotNetRestoreSourcePropsPath=$(GeneratedRestoreSourcesPropsPath);AspNetUniverseBuildOffline=true" /> @@ -142,18 +148,52 @@ Properties="$(_ComposeStoreProps)" /> + + + $(_WorkRoot)HostingStartup\ + <_HostingStartupProps>DepsOutputPath=$(_DepsOutputDirectory) + <_HostingStartupProps>$(_HostingStartupProps);DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath) + <_HostingStartupProps>$(_HostingStartupProps);RuntimeFrameworkVersion=$(MicrosoftNETCoreApp20PackageVersion) + <_HostingStartupProps>$(_HostingStartupProps);HostingStartupPackageName=$(HostingStartupPackageName) + <_HostingStartupProps>$(_HostingStartupProps);HostingStartupPackageVersion=$(HostingStartupPackageVersion) + + + + + + + + + + + + + + + + + + + + + + PackageArtifacts="@(PackageArtifact)" + ExternalDependencies="@(ExternalDependency)"> - - + @@ -207,6 +247,7 @@ + - - - - - - - - - - - - - - - - - - + + diff --git a/build/RuntimeStoreInstaller.targets b/build/RuntimeStoreInstaller.targets index 33fc33992f..c55fb09c65 100644 --- a/build/RuntimeStoreInstaller.targets +++ b/build/RuntimeStoreInstaller.targets @@ -20,12 +20,12 @@ https://dotnetcli.blob.core.windows.net/dotnet $(KOREBUILD_DOTNET_FEED_UNCACHED) - $(PublicCoreFeedPrefix) + $(PublicCoreFeedPrefix) - $(PublicCoreFeedPrefix)/aspnetcore/store/2.0.0-26452/Build.RS. + $(PublicCoreFeedPrefix)/aspnetcore/store/2.0.3-125/Build.RS. $(CoreFeedPrefix)/Runtime/$(MicrosoftNETCoreApp20PackageVersion)/dotnet-runtime-$(MicrosoftNETCoreApp20PackageVersion)-linux-x64.tar.gz - $(_TimestampRSSource)aspnetcore-store-2.0.3-rtm-125-linux-x64.tar.gz - $(_TimestampFreeRSSource)aspnetcore-store-2.0.3- + $(_TimestampRSSource)aspnetcore-store-$(PackageVersion)-linux-x64.tar.gz + $(_TimestampFreeRSSource)aspnetcore-store-$(PackageVersionNoTimestamp)- $(TimestampFreeRSArchivePrefix)linux-x64.tar.gz @@ -47,7 +47,7 @@ - + Condition="!Exists('$(TimestampFreeRSArchivePrefix)win7-x86.zip')" /> @@ -71,8 +71,8 @@ MSBuild doesn't to the substitution correctly because the string contains %, so we'll let bash do it instead. --> - - + + @@ -140,9 +140,9 @@ - - - + + + @@ -304,17 +304,17 @@ - - 2.0.0 + + $(RuntimeStoreInstallerDependencyVersion) - - 2.0.3-rtm-125 + + $(PackageVersion) $(MicrosoftNETCoreApp20PackageVersion) - - 2.0.3 + + $(PackageVersionNoTimestamp) $(MicrosoftNETCoreApp20PackageVersion) @@ -368,18 +368,18 @@ - + - + - + - + @@ -409,7 +409,7 @@ - + @@ -462,22 +462,22 @@ - + Properties="$(CommonRSArguments);RSArchive=$(TimestampRSArchive);DebVersion=$(Version);RsDepVersion=$(RuntimeStoreInstallerDependencyVersion)" /> + Properties="$(CommonRSArguments);RSArchive=$(TimestampFreeLinuxRSArchive);DebVersion=$(PackageVersionNoTimestamp);RsDepVersion=$(RuntimeStoreInstallerDependencyVersion)" /> + Properties="$(CommonHostingArguments);DebVersion=$(Version)" /> + @@ -491,7 +491,6 @@ - diff --git a/build/Templating.targets b/build/Templating.targets index d7592be916..24dc7eef78 100644 --- a/build/Templating.targets +++ b/build/Templating.targets @@ -11,6 +11,7 @@ DotNetRestoreSourcesPropsPath=$(GeneratedRestoreSourcesPropsPath); BuildNumber=$(BuildNumber); Configuration=$(Configuration); + SkipBillOfMaterials=true; $(TemplateProjCommmonProperties); @@ -26,7 +27,7 @@ @@ -55,7 +56,7 @@ diff --git a/build/artifacts.props b/build/artifacts.props index 0564ae2f0b..f52c802dbe 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -10,67 +10,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -79,7 +18,6 @@ - @@ -90,65 +28,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -158,7 +42,5 @@ - - diff --git a/build/dependencies.props b/build/dependencies.props index d262b69550..3ba89f2c4d 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -25,57 +25,71 @@ + + 1.0.8 + 1.1.5 + 2.0.0 + 2.0.5 + + https://dotnet.myget.org/F/dotnet-core/api/v3/index.json - 2.0.2-servicing-25728-02 - 2.0.0 - $(CoreSetupPackageVersion) - - - - - + + + + + + https://dotnet.myget.org/F/dotnet-core/api/v3/index.json + + + + + KRB2004 RuntimeFrameworkVersion netcoreapp2.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + KRB2004 + MicrosoftNETCoreApp20PackageVersion + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -130,7 +144,7 @@ https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json - 2.0.2-rc1-16007 + $(KoreBuildVersion) @@ -153,6 +167,33 @@ + + + + + KRB2004 + RuntimeFrameworkVersion + netcoreapp1.0 + + + KRB2004 + MicrosoftNETCoreApp10PackageVersion + + + KRB2004 + RuntimeFrameworkVersion + netcoreapp1.1 + + + KRB2004 + MicrosoftNETCoreApp11PackageVersion + + + + KRB2004 + DotNetCliTool_MicrosoftNETCoreApp20PackageVersion + netcoreapp2.0 + @@ -250,7 +291,7 @@ - + @@ -290,6 +331,115 @@ not building again in this patch. --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -338,9 +488,24 @@ not building again in this patch. - https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json + https://dotnet.myget.org/F/aspnet-2-0-2-october2017-patch-public/api/v3/index.json + + + + + + + + + + + + + https://dotnet.myget.org/F/aspnetcore-master/api/v3/index.json + + diff --git a/build/repo.props b/build/repo.props index 180925e696..2e60df02b8 100644 --- a/build/repo.props +++ b/build/repo.props @@ -2,6 +2,8 @@ true + + true diff --git a/build/repo.targets b/build/repo.targets index c4042084bc..788f91ad93 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -20,11 +20,11 @@ $(IntermediateDir)dependencies.props $(IntermediateDir)sources.props - $(PrepareDependsOn);PrepareOutputPath + $(PrepareDependsOn);VerifyPackageArtifactConfig;PrepareOutputPath $(CleanDependsOn);CleanArtifacts;CleanUniverseArtifacts $(RestoreDependsOn);RestoreExternalDependencies $(CompileDependsOn);BuildRepositories - $(PackageDependsOn);BuildAllMetapackage;BuildTemplates;SplitPackages + $(PackageDependsOn);BuildTemplates;SplitPackages $(VerifyDependsOn);VerifyCoherentVersions @@ -106,11 +106,19 @@ + + + + - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 94710b1aa3..550bca775f 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -59,9 +59,12 @@ namespace RepoTasks var factory = new SolutionInfoFactory(Log, BuildEngine5); var props = MSBuildListSplitter.GetNamedProperties(Properties); - Log.LogMessage(MessageImportance.High, $"Beginning cross-repo analysis on {Solutions.Length} solutions. Hang tight..."); + if (!props.TryGetValue("Configuration", out var defaultConfig)) + { + defaultConfig = "Debug"; + } - var solutions = factory.Create(Solutions, props, _cts.Token); + var solutions = factory.Create(Solutions, props, defaultConfig, _cts.Token); Log.LogMessage($"Found {solutions.Count} and {solutions.Sum(p => p.Projects.Count)} projects"); if (_cts.IsCancellationRequested) @@ -137,7 +140,7 @@ namespace RepoTasks continue; } - if (!solution.ShouldBuild) + if (!solution.ShouldBuild && solution.Shipped) { reposThatShouldPatch.Add(Path.GetFileName(Path.GetDirectoryName(solution.FullPath))); } diff --git a/build/tasks/CheckRepoGraph.cs b/build/tasks/CheckRepoGraph.cs new file mode 100644 index 0000000000..5502d7316c --- /dev/null +++ b/build/tasks/CheckRepoGraph.cs @@ -0,0 +1,214 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Text; +using System.Threading; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Frameworks; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using RepoTools.BuildGraph; +using RepoTasks.ProjectModel; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class CheckRepoGraph : Task, ICancelableTask + { + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + + [Required] + public ITaskItem[] Solutions { get; set; } + + [Required] + public ITaskItem[] Artifacts { get; set; } + + [Required] + public ITaskItem[] Repositories { get; set; } + + [Required] + public string Properties { get; set; } + + public void Cancel() + { + _cts.Cancel(); + } + + public override bool Execute() + { + var packageArtifacts = Artifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact) + .ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); + + var factory = new SolutionInfoFactory(Log, BuildEngine5); + var props = MSBuildListSplitter.GetNamedProperties(Properties); + + if (!props.TryGetValue("Configuration", out var defaultConfig)) + { + defaultConfig = "Debug"; + } + + var solutions = factory.Create(Solutions, props, defaultConfig, _cts.Token).OrderBy(f => f.Directory).ToList(); + Log.LogMessage($"Found {solutions.Count} and {solutions.Sum(p => p.Projects.Count)} projects"); + + if (_cts.IsCancellationRequested) + { + return false; + } + + var repoGraph = new AdjacencyMatrix(solutions.Count); + var packageToProjectMap = new Dictionary(); + + for (var i = 0; i < solutions.Count; i++) + { + var sln = repoGraph[i] = solutions[i]; + + foreach (var proj in sln.Projects) + { + if (!proj.IsPackable + || proj.FullPath.Contains("samples") + || proj.FullPath.Contains("tools/Microsoft.VisualStudio.Web.CodeGeneration.Design")) + { + continue; + } + + var id = new PackageIdentity(proj.PackageId, new NuGetVersion(proj.PackageVersion)); + + if (packageToProjectMap.TryGetValue(id, out var otherProj)) + { + Log.LogError($"Both {proj.FullPath} and {otherProj.FullPath} produce {id}"); + continue; + } + + packageToProjectMap.Add(id, proj); + } + + var sharedSrc = Path.Combine(sln.Directory, "shared"); + if (Directory.Exists(sharedSrc)) + { + foreach (var dir in Directory.GetDirectories(sharedSrc, "*.Sources")) + { + var id = GetDirectoryName(dir); + var artifactInfo = packageArtifacts[id]; + var sharedSrcProj = new ProjectInfo(dir, + Array.Empty(), + Array.Empty(), + true, + artifactInfo.PackageInfo.Id, + artifactInfo.PackageInfo.Version.ToNormalizedString()); + sharedSrcProj.SolutionInfo = sln; + var identity = new PackageIdentity(artifactInfo.PackageInfo.Id, artifactInfo.PackageInfo.Version); + packageToProjectMap.Add(identity, sharedSrcProj); + } + } + } + + if (Log.HasLoggedErrors) + { + return false; + } + + for (var i = 0; i < solutions.Count; i++) + { + var src = repoGraph[i]; + + foreach (var proj in src.Projects) + { + if (!proj.IsPackable + || proj.FullPath.Contains("samples")) + { + continue; + } + + foreach (var dep in proj.Frameworks.SelectMany(f => f.Dependencies.Values)) + { + if (packageToProjectMap.TryGetValue(new PackageIdentity(dep.Id, new NuGetVersion(dep.Version)), out var target)) + { + var j = repoGraph.FindIndex(target.SolutionInfo); + repoGraph.SetLink(i, j); + } + } + + foreach (var toolDep in proj.Tools) + { + if (packageToProjectMap.TryGetValue(new PackageIdentity(toolDep.Id, new NuGetVersion(toolDep.Version)), out var target)) + { + var j = repoGraph.FindIndex(target.SolutionInfo); + repoGraph.SetLink(i, j); + } + } + } + } + + var repos = Repositories.ToDictionary(i => i.ItemSpec, i => i, StringComparer.OrdinalIgnoreCase); + + for (var i = 0; i < repoGraph.Count; i++) + { + var src = repoGraph[i]; + var repoName = GetDirectoryName(src.Directory); + var repo = repos[repoName]; + + for (var j = 0; j < repoGraph.Count; j++) + { + if (j == i) continue; + if (repoGraph.HasLink(i, j)) + { + var target = repoGraph[j]; + var targetRepoName = GetDirectoryName(target.Directory); + var targetRepo = repos[targetRepoName]; + + if (src.Shipped && !target.Shipped) + { + Log.LogError($"{repoName} cannot depend on {targetRepoName}. Repos marked as 'Shipped' cannot depend on repos that are rebuilding. Update the configuration in submodule.props."); + } + } + } + } + + return !Log.HasLoggedErrors; + } + + private static string GetDirectoryName(string path) + => Path.GetFileName(path.TrimEnd(new[] { '\\', '/' })); + + private class AdjacencyMatrix + { + private readonly bool[,] _matrix; + private readonly SolutionInfo[] _items; + + public AdjacencyMatrix(int size) + { + _matrix = new bool[size, size]; + _items = new SolutionInfo[size]; + Count = size; + } + + public SolutionInfo this[int idx] + { + get => _items[idx]; + set => _items[idx] = value; + } + + public int FindIndex(SolutionInfo item) + { + return Array.FindIndex(_items, t => t.Equals(item)); + } + + public int Count { get; } + + public bool HasLink(int source, int target) => _matrix[source, target]; + + public void SetLink(int source, int target) + { + _matrix[source, target] = true; + } + } + } +} diff --git a/build/tasks/ComposeNewStore.cs b/build/tasks/ComposeNewStore.cs index 90f601e0a2..2ad23089d2 100644 --- a/build/tasks/ComposeNewStore.cs +++ b/build/tasks/ComposeNewStore.cs @@ -17,19 +17,15 @@ namespace RepoTasks [Required] public ITaskItem[] NewManifests { get; set; } - [Required] public ITaskItem[] RuntimeStoreFiles { get; set; } - [Required] public ITaskItem[] RuntimeStoreSymbolFiles { get; set; } [Required] public string ManifestDestination { get; set; } - [Required] public string StoreDestination { get; set; } - [Required] public string SymbolsDestination { get; set; } public override bool Execute() @@ -58,45 +54,51 @@ namespace RepoTasks } } - // Insert new runtime store files - foreach (var storeFile in RuntimeStoreFiles) + if (RuntimeStoreFiles != null) { - // format: {bitness}}/{tfm}}/{id}/{version}}/... - var recursiveDir = storeFile.GetMetadata("RecursiveDir"); - var components = recursiveDir.Split(Path.DirectorySeparatorChar); - var id = components[2]; - var version = components[3]; - - if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) + // Insert new runtime store files + foreach (var storeFile in RuntimeStoreFiles) { - var destinationDir = Path.Combine(StoreDestination, recursiveDir); - if (!Directory.Exists(Path.Combine(StoreDestination, recursiveDir))) - { - Directory.CreateDirectory(destinationDir); - } + // format: {bitness}}/{tfm}}/{id}/{version}}/... + var recursiveDir = storeFile.GetMetadata("RecursiveDir"); + var components = recursiveDir.Split(Path.DirectorySeparatorChar); + var id = components[2]; + var version = components[3]; - File.Copy(storeFile.GetMetadata("FullPath"), Path.Combine(destinationDir, $"{storeFile.GetMetadata("Filename")}{storeFile.GetMetadata("Extension")}"), overwrite: true); + if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) + { + var destinationDir = Path.Combine(StoreDestination, recursiveDir); + if (!Directory.Exists(Path.Combine(StoreDestination, recursiveDir))) + { + Directory.CreateDirectory(destinationDir); + } + + File.Copy(storeFile.GetMetadata("FullPath"), Path.Combine(destinationDir, $"{storeFile.GetMetadata("Filename")}{storeFile.GetMetadata("Extension")}"), overwrite: true); + } } } - // Insert new runtime store files - foreach (var symbolFile in RuntimeStoreSymbolFiles) + if (RuntimeStoreSymbolFiles != null) { - // format: {bitness}}/{tfm}}/{id}/{version}}/... - var recursiveDir = symbolFile.GetMetadata("RecursiveDir"); - var components = recursiveDir.Split(Path.DirectorySeparatorChar); - var id = components[2]; - var version = components[3]; - - if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) + // Insert new runtime store symbol files + foreach (var symbolFile in RuntimeStoreSymbolFiles) { - var destinationDir = Path.Combine(SymbolsDestination, recursiveDir); - if (!Directory.Exists(Path.Combine(SymbolsDestination, recursiveDir))) - { - Directory.CreateDirectory(destinationDir); - } + // format: {bitness}}/{tfm}}/{id}/{version}}/... + var recursiveDir = symbolFile.GetMetadata("RecursiveDir"); + var components = recursiveDir.Split(Path.DirectorySeparatorChar); + var id = components[2]; + var version = components[3]; - File.Copy(symbolFile.GetMetadata("FullPath"), Path.Combine(destinationDir, $"{symbolFile.GetMetadata("Filename")}{symbolFile.GetMetadata("Extension")}"), overwrite: true); + if (!existingFiles.TryGetValue(id, out var versions) || !versions.Contains(version)) + { + var destinationDir = Path.Combine(SymbolsDestination, recursiveDir); + if (!Directory.Exists(Path.Combine(SymbolsDestination, recursiveDir))) + { + Directory.CreateDirectory(destinationDir); + } + + File.Copy(symbolFile.GetMetadata("FullPath"), Path.Combine(destinationDir, $"{symbolFile.GetMetadata("Filename")}{symbolFile.GetMetadata("Extension")}"), overwrite: true); + } } } diff --git a/build/tasks/ProjectModel/ProjectInfo.cs b/build/tasks/ProjectModel/ProjectInfo.cs index 1dd4339185..9aed38898b 100644 --- a/build/tasks/ProjectModel/ProjectInfo.cs +++ b/build/tasks/ProjectModel/ProjectInfo.cs @@ -10,7 +10,6 @@ namespace RepoTasks.ProjectModel internal class ProjectInfo { public ProjectInfo(string fullPath, - string projectExtensionsPath, IReadOnlyList frameworks, IReadOnlyList tools, bool isPackable, @@ -28,7 +27,6 @@ namespace RepoTasks.ProjectModel FullPath = fullPath; FileName = Path.GetFileName(fullPath); Directory = Path.GetDirectoryName(FullPath); - ProjectExtensionsPath = projectExtensionsPath ?? Path.Combine(Directory, "obj"); IsPackable = isPackable; PackageId = packageId; PackageVersion = packageVersion; @@ -36,7 +34,6 @@ namespace RepoTasks.ProjectModel public string FullPath { get; } public string FileName { get; } - public string ProjectExtensionsPath { get; } public string Directory { get; } public string PackageId { get; } public string PackageVersion { get; } @@ -44,5 +41,6 @@ namespace RepoTasks.ProjectModel public IReadOnlyList Frameworks { get; } public IReadOnlyList Tools { get; } + public SolutionInfo SolutionInfo { get; internal set; } } } diff --git a/build/tasks/ProjectModel/ProjectInfoFactory.cs b/build/tasks/ProjectModel/ProjectInfoFactory.cs index 5c739f1784..78d5adc5fd 100644 --- a/build/tasks/ProjectModel/ProjectInfoFactory.cs +++ b/build/tasks/ProjectModel/ProjectInfoFactory.cs @@ -27,7 +27,6 @@ namespace RepoTasks.ProjectModel { var project = GetProject(path, projectCollection); var instance = project.CreateProjectInstance(ProjectInstanceSettings.ImmutableWithFastItemLookup); - var projExtPath = instance.GetPropertyValue("MSBuildProjectExtensionsPath"); var targetFrameworks = instance.GetPropertyValue("TargetFrameworks"); var targetFramework = instance.GetPropertyValue("TargetFramework"); @@ -59,11 +58,17 @@ namespace RepoTasks.ProjectModel var tools = GetTools(instance).ToArray(); bool.TryParse(instance.GetPropertyValue("IsPackable"), out var isPackable); + + if (isPackable) + { + // the default packable setting is disabled for projects referencing this package. + isPackable = !frameworks.SelectMany(f => f.Dependencies.Keys).Any(d => d.Equals("Microsoft.NET.Test.Sdk", StringComparison.OrdinalIgnoreCase)); + } + var packageId = instance.GetPropertyValue("PackageId"); var packageVersion = instance.GetPropertyValue("PackageVersion"); return new ProjectInfo(path, - projExtPath, frameworks, tools, isPackable, @@ -88,6 +93,8 @@ namespace RepoTasks.ProjectModel var globalProps = new Dictionary() { ["DesignTimeBuild"] = "true", + // Isolate the project from post-restore side effects + ["ExcludeRestorePackageImports"] = "true", }; var project = new Project(xml, diff --git a/build/tasks/ProjectModel/SolutionInfo.cs b/build/tasks/ProjectModel/SolutionInfo.cs index 5fe51132db..dce2f0bdd3 100644 --- a/build/tasks/ProjectModel/SolutionInfo.cs +++ b/build/tasks/ProjectModel/SolutionInfo.cs @@ -3,12 +3,13 @@ using System; using System.Collections.Generic; +using System.IO; namespace RepoTasks.ProjectModel { internal class SolutionInfo { - public SolutionInfo(string fullPath, string configName, IReadOnlyList projects, bool shouldBuild) + public SolutionInfo(string fullPath, string configName, IReadOnlyList projects, bool shouldBuild, bool shipped) { if (string.IsNullOrEmpty(fullPath)) { @@ -20,15 +21,24 @@ namespace RepoTasks.ProjectModel throw new ArgumentException(nameof(configName)); } + Directory = Path.GetDirectoryName(fullPath); FullPath = fullPath; ConfigName = configName; Projects = projects ?? throw new ArgumentNullException(nameof(projects)); ShouldBuild = shouldBuild; + Shipped = shipped; + + foreach (var proj in Projects) + { + proj.SolutionInfo = this; + } } + public string Directory { get; } public string FullPath { get; } public string ConfigName { get; } public IReadOnlyList Projects { get; } public bool ShouldBuild { get; } + public bool Shipped { get; } } } diff --git a/build/tasks/ProjectModel/SolutionInfoFactory.cs b/build/tasks/ProjectModel/SolutionInfoFactory.cs index 9ea368cc0a..c24475f7c2 100644 --- a/build/tasks/ProjectModel/SolutionInfoFactory.cs +++ b/build/tasks/ProjectModel/SolutionInfoFactory.cs @@ -27,7 +27,7 @@ namespace RepoTasks.ProjectModel _buildEngine = buildEngine; } - public IReadOnlyList Create(IEnumerable solutionItems, IDictionary properties, CancellationToken ct) + public IReadOnlyList Create(IEnumerable solutionItems, IDictionary properties, string defaultConfig, CancellationToken ct) { var timer = Stopwatch.StartNew(); @@ -49,7 +49,7 @@ namespace RepoTasks.ProjectModel if (solutionProps.TryGetValue("Configuration", out var configName)) { - solutionProps["Configuration"] = configName = "Debug"; + solutionProps["Configuration"] = configName = defaultConfig; } var key = $"SlnInfo:{solutionFile}:{configName}"; @@ -85,12 +85,14 @@ namespace RepoTasks.ProjectModel } bool.TryParse(solution.GetMetadata("Build"), out var shouldBuild); + bool.TryParse(solution.GetMetadata("Shipped"), out var shipped); var solutionInfo = new SolutionInfo( solutionFile, configName, projects.ToArray(), - shouldBuild); + shouldBuild, + shipped); _buildEngine.RegisterTaskObject(key, solutionInfo, RegisteredTaskObjectLifetime.Build, allowEarlyCollection: true); diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 5410effb57..569b766901 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -4,6 +4,7 @@ + diff --git a/build/tasks/ResolveHostingStartupPackages.cs b/build/tasks/ResolveHostingStartupPackages.cs index 8bfc27399c..5a97097db4 100644 --- a/build/tasks/ResolveHostingStartupPackages.cs +++ b/build/tasks/ResolveHostingStartupPackages.cs @@ -15,6 +15,9 @@ namespace RepoTasks [Required] public ITaskItem[] PackageArtifacts { get; set; } + [Required] + public ITaskItem[] ExternalDependencies { get; set; } + [Output] public ITaskItem[] HostingStartupArtifacts { get; set; } @@ -22,7 +25,18 @@ namespace RepoTasks { // Parse input var hostingStartupArtifacts = PackageArtifacts.Where(p => p.GetMetadata("HostingStartup") == "true"); - HostingStartupArtifacts = BuildArtifacts.Where(p => hostingStartupArtifacts.Any(h => h.GetMetadata("Identity") == p.GetMetadata("PackageId"))).ToArray(); + var externalHostingStartupArtifacts = ExternalDependencies.Where(p => p.GetMetadata("HostingStartup") == "true"); + + var hostingStartups = BuildArtifacts.Where(p => hostingStartupArtifacts.Any(h => h.GetMetadata("Identity") == p.GetMetadata("PackageId"))); + + foreach (var externalHostingStartup in externalHostingStartupArtifacts) + { + // The parameters PackageId and Version are required for output. For external dependencies, the identity is the pacakge id. + externalHostingStartup.SetMetadata("PackageId", externalHostingStartup.GetMetadata("Identity")); + hostingStartups = hostingStartups.Append(externalHostingStartup); + } + + HostingStartupArtifacts = hostingStartups.ToArray(); return true; } diff --git a/build/tools/packaging/hosting_debian_config.json b/build/tools/packaging/hosting_debian_config.json index 8159b4e25f..92c825e901 100644 --- a/build/tools/packaging/hosting_debian_config.json +++ b/build/tools/packaging/hosting_debian_config.json @@ -30,6 +30,6 @@ "debian_dependencies": { "dotnet-runtime-DOTNET_VERSION": {}, - "aspnetcore-store-STORE_VERSION": {} + "aspnetcore-store-DEB_VERSION": {} } } \ No newline at end of file diff --git a/build/tools/packaging/store_debian_config.json b/build/tools/packaging/store_debian_config.json index 70dffe42a9..880cba3a38 100644 --- a/build/tools/packaging/store_debian_config.json +++ b/build/tools/packaging/store_debian_config.json @@ -29,6 +29,6 @@ }, "debian_dependencies": { - "aspnetcore-store-2.0.0": {} + "aspnetcore-store-RS_DEP_VERSION": {} } } \ No newline at end of file diff --git a/build/tools/templates/HostingStartup/HostingStartup.csproj b/build/tools/templates/HostingStartup/HostingStartup.csproj index dbb96336d5..12cdf93eb1 100644 --- a/build/tools/templates/HostingStartup/HostingStartup.csproj +++ b/build/tools/templates/HostingStartup/HostingStartup.csproj @@ -1,8 +1,19 @@  + + netcoreapp2.0 Exe + $(DotNetRestoreSources) + + $(RestoreSources); + https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json; + + + $(RestoreSources); + https://api.nuget.org/v3/index.json; + @@ -15,6 +26,6 @@ $(DepsOutputPath)\$(HostingStartupPackageName)\shared\Microsoft.NETCore.App\$(DepsRuntimeFrameworkVersion)\$(HostingStartupPackageName).deps.json - + diff --git a/korebuild-lock.txt b/korebuild-lock.txt index 6471fca0b4..1e32745c4f 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.0.2-rc1-16007 -commithash:bccf097cd0fceb185b7bf6aa8981191304cea9a7 +version:2.0.3-rtm-10005 +commithash:767fa0dcd1cca6b0a722b7b6a3919f698fbd1325 diff --git a/korebuild.json b/korebuild.json index 6bbc5eeb9c..6ac34cd945 100644 --- a/korebuild.json +++ b/korebuild.json @@ -1,4 +1,4 @@ { - "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/rel/2.0.2/tools/korebuild.schema.json", - "channel": "rel/2.0.2" + "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/release/2.0.0/tools/korebuild.schema.json", + "channel": "release/2.0.0" } diff --git a/modules/HttpSysServer b/modules/HttpSysServer index 88862aeb68..1b5ba87bbb 160000 --- a/modules/HttpSysServer +++ b/modules/HttpSysServer @@ -1 +1 @@ -Subproject commit 88862aeb6831567d182e406ecffb646bc327d44a +Subproject commit 1b5ba87bbb80d798bebdf73711a09a846205c19b diff --git a/modules/Identity b/modules/Identity index 8c47b90677..becf1df9c5 160000 --- a/modules/Identity +++ b/modules/Identity @@ -1 +1 @@ -Subproject commit 8c47b90677c0f544844151418ba94f24d9f2a094 +Subproject commit becf1df9c5a3b908e28ec3c1272b072d8801e3dc diff --git a/modules/JavaScriptServices b/modules/JavaScriptServices index 64389a9bbe..d5a664e481 160000 --- a/modules/JavaScriptServices +++ b/modules/JavaScriptServices @@ -1 +1 @@ -Subproject commit 64389a9bbeda7378c80b4c302700ddcb78d4f0aa +Subproject commit d5a664e4817a395cbafec942387162b5afeba7bf diff --git a/modules/Mvc b/modules/Mvc index f8789f5d5c..67f48064ce 160000 --- a/modules/Mvc +++ b/modules/Mvc @@ -1 +1 @@ -Subproject commit f8789f5d5c4d5869490a05b8f7250b6151f1673e +Subproject commit 67f48064ced9ac9c2917d369ffc7d1866d03c84d diff --git a/modules/MvcPrecompilation b/modules/MvcPrecompilation index bc58d8495a..f510e70340 160000 --- a/modules/MvcPrecompilation +++ b/modules/MvcPrecompilation @@ -1 +1 @@ -Subproject commit bc58d8495a431d3de606afc52c2987d1ebf1e6ad +Subproject commit f510e7034000e346f6771197d30aaf791ce2bf48 diff --git a/modules/Scaffolding b/modules/Scaffolding index 26822d4c87..069ca26129 160000 --- a/modules/Scaffolding +++ b/modules/Scaffolding @@ -1 +1 @@ -Subproject commit 26822d4c876ee6203014eaf6df398c874c4535ea +Subproject commit 069ca2612999a49e2b19099f21d8196f422c82de diff --git a/modules/Templating b/modules/Templating index 18feba377f..e3674db32e 160000 --- a/modules/Templating +++ b/modules/Templating @@ -1 +1 @@ -Subproject commit 18feba377f4d420a591c8320ca9170160f32060e +Subproject commit e3674db32e34f4b43671e978eb9902b3b11fb214 diff --git a/scripts/PatchVersionPrefix.ps1 b/scripts/PatchVersionPrefix.ps1 new file mode 100644 index 0000000000..17303de53a --- /dev/null +++ b/scripts/PatchVersionPrefix.ps1 @@ -0,0 +1,66 @@ +<# +.SYNOPSIS + Updates the version.props file in repos to a newer patch version +.PARAMETER Repos + A list of the repositories that should be patched +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string[]]$Repos +) + +$ErrorActionPreference = 'Stop' + +function SaveXml($xml, [string]$path) { + Write-Verbose "Saving to $path" + $ErrorActionPreference = 'stop' + + $settings = New-Object System.XML.XmlWriterSettings + $settings.OmitXmlDeclaration = $true + $settings.Encoding = New-Object System.Text.UTF8Encoding( $true ) + $writer = [System.XML.XMLTextWriter]::Create($path, $settings) + $xml.Save($writer) + $writer.Close() +} + +function LoadXml([string]$path) { + Write-Verbose "Reading to $path" + + $ErrorActionPreference = 'stop' + $obj = new-object xml + $obj.PreserveWhitespace = $true + $obj.Load($path) + return $obj +} + +function BumpPatch([System.Xml.XmlNode]$node) { + if (-not $node) { + return + } + [version] $version = $node.InnerText + $node.InnerText = "{0}.{1}.{2}" -f $version.Major, $version.Minor, ($version.Build + 1) +} + +foreach ($repo in $Repos) { + $path = "$PSScriptRoot/../modules/$repo/version.props" + if (-not (Test-Path $path)) { + Write-Warning "$path does not exist" + continue + } + $path = Resolve-Path $path + Write-Verbose "$path" + [xml] $xml = LoadXml $path + + $suffix = $xml.SelectSingleNode('/Project/PropertyGroup/VersionSuffix') + if (-not $suffix) { + write-error "$path does not have VersionSuffix" + } + + $versionPrefix = $xml.SelectSingleNode('/Project/PropertyGroup/VersionPrefix') + $epxVersionPrefix = $xml.SelectSingleNode('/Project/PropertyGroup/ExperimentalProjectVersionPrefix') + BumpPatch $epxVersionPrefix + BumpPatch $versionPrefix + SaveXml $xml $path +} + diff --git a/scripts/UpdateBuildTools.ps1 b/scripts/UpdateBuildTools.ps1 new file mode 100644 index 0000000000..6f2e4ad3c5 --- /dev/null +++ b/scripts/UpdateBuildTools.ps1 @@ -0,0 +1,96 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + Updates the build tools version and generates a commit message with the list of changes +.PARAMETER RepoRoot + The directory containing the repo +.PARAMETER GitAuthorName + The author name to use in the commit message. (Optional) +.PARAMETER GitAuthorEmail + The author email to use in the commit message. (Optional) +.PARAMETER GitCommitArgs + Additional arguments to pass into git-commit +.PARAMETER NoCommit + Make changes without executing git-commit +.PARAMETER Force + Specified this to make a commit with any changes +#> +[cmdletbinding(SupportsShouldProcess = $true)] +param( + [string]$RepoRoot, + [string]$GitAuthorName = $null, + [string]$GitAuthorEmail = $null, + [string[]]$GitCommitArgs = @(), + [switch]$NoCommit, + [switch]$Force +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2 + +if (-not $RepoRoot) { + $RepoRoot = Resolve-Path "$PSScriptRoot\.." +} + +Import-Module "$PSScriptRoot/common.psm1" -Scope Local -Force + +function Get-KoreBuildVersion { + $lockFile = "$RepoRoot/korebuild-lock.txt" + if (!(Test-Path $lockFile)) { + return '' + } + $version = Get-Content $lockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1 + if (!$version) { + Write-Error "Failed to parse version from $lockFile. Expected a line that begins with 'version:'" + } + $version = $version.TrimStart('version:').Trim() + return $version +} + +Push-Location $RepoRoot +try { + Assert-Git + + $oldVersion = Get-KoreBuildVersion + + # Executes a command that no-ops. The only thing we really need is the updated version of korebuild-lock.txt + & "$RepoRoot/build.ps1" -Update '-t:Noop' | Out-Null + + $newVersion = Get-KoreBuildVersion + + if ($oldVersion -eq $newVersion) { + Write-Host -ForegroundColor Magenta 'No changes to build tools' + exit 0 + } + + Invoke-Block { git add "$RepoRoot/korebuild-lock.txt" } + Invoke-Block { git add "$RepoRoot/build/dependencies.props" } + + $shortMessage = "Updating BuildTools from $oldVersion to $newVersion" + # add this to the commit message to make it possible to filter commit triggers based on message + $message = "$shortMessage`n`n[auto-updated: buildtools]" + + if (-not $NoCommit -and ($Force -or ($PSCmdlet.ShouldContinue($shortMessage, 'Create a new commit with these changes?')))) { + + $gitConfigArgs = @() + if ($GitAuthorName) { + $gitConfigArgs += '-c', "user.name=$GitAuthorName" + } + + if ($GitAuthorEmail) { + $gitConfigArgs += '-c', "user.email=$GitAuthorEmail" + } + + Invoke-Block { git @gitConfigArgs commit -m $message @GitCommitArgs } + } + else { + # If composing this script with others, return the message that would have been used + return @{ + message = $message + } + } +} +finally { + Pop-Location +} diff --git a/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.3.xml b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.3.xml new file mode 100644 index 0000000000..da9ec336b4 --- /dev/null +++ b/src/Microsoft.AspNetCore.All/build/aspnetcore-store-2.0.3.xml @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/version.props b/version.props index ca922dba5c..1b4a62e7b2 100644 --- a/version.props +++ b/version.props @@ -1,6 +1,7 @@ - 2.0.4 + 2.0.3 + 2.0.5 rtm $(VersionPrefix) $(VersionPrefix)