diff --git a/.gitignore b/.gitignore index d5950872a2..1d308dafc6 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ node_modules global.json msbuild.ProjectImports.zip .env +scripts/tmp/ diff --git a/.gitmodules b/.gitmodules index 7b902cbf09..f46e4e64ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -193,4 +193,4 @@ [submodule "modules/WebHooks"] path = modules/WebHooks url = https://github.com/aspnet/WebHooks.git - branch = release/2.1 \ No newline at end of file + branch = release/2.1 diff --git a/README.md b/README.md index c9a3f295c6..ad20827e0b 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Microsoft.AspNetCore | [![][metapackage-myget-badge]][metapackage-m [win-x86-zip]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-win-x86.zip [win-x86-exe]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-win-x86.exe [linux-x64-tar]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-linux-x64.tar.gz +[linux-arm-tar]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-linux-arm.tar.gz [osx-x64-tar]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-osx-x64.tar.gz [debian-x64-deb]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-x64.deb [redhat-x64-rpm]: https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/master/aspnetcore-runtime-latest-x64.rpm @@ -77,6 +78,7 @@ Windows (x64) | [Installer (exe)][win-x64-exe]
[Archive (zip)][win-x6 Windows (x86) | [Installer (exe)][win-x86-exe]
[Archive (zip)][win-x86-zip] | [Installer (exe)][win-x86-exe-rel-21]
[Archive (zip)][win-x86-zip-rel-21] macOS (x64) | [Archive (tar.gz)][osx-x64-tar] | [Archive (tar.gz)][osx-x64-tar-rel-21] Linux (x64) | [Archive (tar.gz)][linux-x64-tar] | [Archive (tar.gz)][linux-x64-tar-rel-21] +Linux (arm32) | [Archive (tar.gz)][linux-arm-tar] | (not available yet) Debian/Ubuntu (x64) | [Installer (deb)][debian-x64-deb] | [Installer (deb)][debian-x64-deb-rel-21] RedHat/Fedora (x64) | [Installer (rpm)][redhat-x64-rpm] | [Installer (rpm)][redhat-x64-rpm-rel-21] Alpine Linux 3.6 (x64)| [Archive (tar.gz)][alpine.3.6-x64-tar] | [Archive (tar.gz)][alpine.3.6-x64-tar-rel-21] diff --git a/build/GenerateCode.targets b/build/GenerateCode.targets new file mode 100644 index 0000000000..069e358ae6 --- /dev/null +++ b/build/GenerateCode.targets @@ -0,0 +1,17 @@ + + + + + + + + + diff --git a/build/Publish.targets b/build/Publish.targets index 7d97b9b81d..cc47d871c1 100644 --- a/build/Publish.targets +++ b/build/Publish.targets @@ -151,7 +151,7 @@ - + @@ -174,21 +174,22 @@ Condition="'$(PublishToMyget)' == 'true'"> + - - - - + - - - + + $(PublishMyGetNpmRegistryUrl.Replace("https:", "")):_authToken @@ -196,9 +197,11 @@ - + @@ -262,6 +265,7 @@ AccountKey="$(PublishBlobFeedKey)" ItemsToPush="@(PackageToPublishToTransport)" Overwrite="$(PushToBlobFeed_Overwrite)" + ManifestBranch="$(BuildBranch)" ManifestBuildId="$(Version)" ManifestBuildData="ProductVersion=$(PackageVersion);UniverseCommitHash=$(CommitHash)" ManifestCommit="$(CommitHash)" @@ -274,6 +278,7 @@ ItemsToPush="@(FilesToPublishToTransport)" PublishFlatContainer="true" Overwrite="$(PushToBlobFeed_Overwrite)" + ManifestBranch="$(BuildBranch)" ManifestBuildId="$(Version)" ManifestBuildData="ProductVersion=$(PackageVersion);UniverseCommitHash=$(CommitHash)" ManifestCommit="$(CommitHash)" diff --git a/build/SharedFx.props b/build/SharedFx.props index 12c542ac23..ab47b3f4b1 100644 --- a/build/SharedFx.props +++ b/build/SharedFx.props @@ -56,9 +56,10 @@ - - + + + diff --git a/build/SharedFx.targets b/build/SharedFx.targets index fd696721b7..0f2ae1fab4 100644 --- a/build/SharedFx.targets +++ b/build/SharedFx.targets @@ -25,7 +25,7 @@ - <_SharedFrameworkSymbolsPackage Include="@(SharedFrameworkName)" Condition="'%(AllSharedFxRIDs.CrossgenSymbolsSupported)' != 'false'"> + <_SharedFrameworkSymbolsPackage Include="@(SharedFrameworkName)" Condition="'%(AllSharedFxRIDs.CrossgenSymbols)' != 'false' AND '%(AllSharedFxRIDs.Crossgen)' != 'false'"> %(AllSharedFxRIDs.Identity) <_SharedFrameworkSymbolsPackage Update="@(_SharedFrameworkSymbolsPackage)" PackageId="runtime.%(Rid).%(Identity)" /> @@ -117,6 +117,10 @@ true + false + true + false + true @@ -366,11 +370,16 @@ - + + diff --git a/build/artifacts.props b/build/artifacts.props index b445ab8b08..6f0a97feb9 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -28,12 +28,13 @@ - + + @@ -156,19 +157,18 @@ + - + - - - + @@ -255,7 +255,7 @@ - + diff --git a/build/dependencies.props b/build/dependencies.props index bdd5712da5..204ba32114 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -1,90 +1,35 @@  - 4.5.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 + 4.5.0-preview3-26413-02 + 2.1.0-preview3-26413-05 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 4.5.0-preview2-26406-04 + 2.1.0-preview3-26413-05 + 4.5.0-preview3-26413-02 4.7.0-preview4.5065 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 2.1.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 1.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 1.2.0-preview2-26406-04 - 1.2.0-preview2-26406-04 - 1.2.0-preview2-26406-04 - 1.2.0-preview2-26406-04 - 1.2.0-preview2-26406-04 - 1.2.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 1.6.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.9.0-preview2-26406-04 - 4.5.0-preview2-26406-04 - 4.5.0-preview2-26406-04 + 4.5.0-preview3-26413-02 + 1.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 1.6.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.9.0-preview3-26413-02 + 4.5.0-preview3-26413-02 + 4.5.0-preview3-26413-02 @@ -96,20 +41,17 @@ $(KoreBuildVersion) $(KoreBuildVersion) - $(KoreBuildVersion) 0.9.9 0.10.13 - 2.4.337 4.2.1 2.1.0-prerelease-02430-04 2.0.0 8.7.0 4.2.1 - 1.0.5 3.1.0 1.10.0 2.1.1 @@ -127,13 +69,6 @@ 2.6.1 2.6.1 2.6.1 - 2.6.1 - 2.6.1 - 2.6.1 - 2.6.1 - 2.6.1 - 2.6.1 - 2.6.1 1.7.0 0.2.0-beta-62606-02 1.0.0-rc3-003121 @@ -146,15 +81,12 @@ 2.0.0 1.0.1 15.6.1 - 3.0.1 3.0.1 3.0.1 3.0.1 15.0.26606 - 15.6.161-preview 15.6.161-preview 15.6.161-preview - 2.6.1 7.10.6070 15.3.224 2.0.6142705 @@ -166,23 +98,19 @@ 8.0.50727 9.0.30729 7.10.6071 - 15.6.161-preview - 15.6.161-preview 15.6.161-preview - 15.6.161-preview - 7.0.0 1.4.0 1.3.8 1.0.1 4.7.49 - 1.0.0-rc + 1.7.3.4 1.6.1 2.0.1 1.0.1 11.0.2 12.2.1100 - 5.8.0 - 1.0.0-preview2 + 5.9.0 + 1.0.2-rc1 2.2.0 1.4.0 3.2.0 @@ -194,22 +122,18 @@ 1.0.0 5.2.0 3.1.1 - 6.1.7601.17515 4.3.2 3.1.1 4.3.0 4.3.0 + 1.3.7 9.0.1 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 - 2.7.0-beta3-62512-06 - 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 - 2.7.0-beta3-62512-06 - 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 2.7.0-beta3-62512-06 @@ -220,6 +144,7 @@ 2.3.1 2.3.1 2.3.1 + 2.3.1 2.3.1 2.4.0-beta.1.build3945 diff --git a/build/docker/alpine.Dockerfile b/build/docker/alpine.Dockerfile index 1f80997905..bbe9f714dc 100644 --- a/build/docker/alpine.Dockerfile +++ b/build/docker/alpine.Dockerfile @@ -1,4 +1,4 @@ -FROM microsoft/dotnet:2.1-runtime-deps-alpine +FROM microsoft/dotnet:2.1.0-preview1-runtime-deps-alpine ARG USER ARG USER_ID ARG GROUP_ID diff --git a/build/external-dependencies.props b/build/external-dependencies.props index 4d86192b53..10a465b964 100644 --- a/build/external-dependencies.props +++ b/build/external-dependencies.props @@ -19,10 +19,8 @@ - - @@ -30,7 +28,6 @@ - @@ -42,8 +39,6 @@ - - @@ -51,26 +46,15 @@ - - - - - - - - - - - @@ -82,20 +66,13 @@ - - - - - - - @@ -109,17 +86,13 @@ - - - - - + @@ -137,36 +110,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -176,55 +119,35 @@ - - - - - - - - - - - - - - - - - - - - - + @@ -232,6 +155,7 @@ + diff --git a/build/repo.props b/build/repo.props index 74acfe357e..9e2f4b7571 100644 --- a/build/repo.props +++ b/build/repo.props @@ -22,6 +22,7 @@ + @@ -30,6 +31,7 @@ + diff --git a/build/repo.targets b/build/repo.targets index 27523f5fcc..f81f495946 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -6,6 +6,7 @@ + diff --git a/build/tasks/AnalyzeBuildGraph.cs b/build/tasks/AnalyzeBuildGraph.cs index 076706eef0..d695e172ff 100644 --- a/build/tasks/AnalyzeBuildGraph.cs +++ b/build/tasks/AnalyzeBuildGraph.cs @@ -93,15 +93,19 @@ namespace RepoTasks { // ensure versions cascade var buildPackageMap = packages.ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); - var dependencyMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var dependencyMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (var dep in Dependencies) { if (!dependencyMap.TryGetValue(dep.ItemSpec, out var versions)) { - dependencyMap[dep.ItemSpec] = versions = new List(); + dependencyMap[dep.ItemSpec] = versions = new List(); } - versions.Add(dep.GetMetadata("Version")); + versions.Add(new ExternalDependency + { + PackageId = dep.ItemSpec, + Version = dep.GetMetadata("Version"), + }); } var inconsistentVersions = new List(); @@ -115,15 +119,21 @@ namespace RepoTasks { if (!buildPackageMap.TryGetValue(dependency.Key, out var package)) { + var idx = -1; // This dependency is not one of the packages that will be compiled by this run of Universe. if (!dependencyMap.TryGetValue(dependency.Key, out var externalVersions) - || !externalVersions.Contains(dependency.Value.Version)) + || (idx = externalVersions.FindIndex(0, externalVersions.Count, i => i.Version == dependency.Value.Version)) < 0) { Log.LogKoreBuildError( project.FullPath, KoreBuildErrors.UndefinedExternalDependency, message: $"Undefined external dependency on {dependency.Key}/{dependency.Value.Version}"); } + + if (idx >= 0) + { + externalVersions[idx].IsReferenced = true; + } continue; } @@ -174,6 +184,15 @@ namespace RepoTasks Log.LogError("Package versions are inconsistent. See build log for details."); } + foreach (var versions in dependencyMap.Values) + { + foreach (var item in versions.Where(i => !i.IsReferenced)) + { + // See https://github.com/aspnet/Universe/wiki/Build-warning-and-error-codes#potentially-unused-external-dependency for details + Log.LogMessage(MessageImportance.Normal, $"Potentially unused external dependency: {item.PackageId}/{item.Version}. See https://github.com/aspnet/Universe/wiki/Build-warning-and-error-codes for details."); + } + } + foreach (var repo in reposThatShouldPatch) { Log.LogError($"{repo} should not be a 'ShippedRepository'. Version changes in other repositories mean it should be patched to perserve cascading version upgrades."); @@ -211,13 +230,13 @@ namespace RepoTasks projectGroup.Add(new Project(proj.PackageId) - { - Repository = repo, - PackageReferences = new HashSet(proj + { + Repository = repo, + PackageReferences = new HashSet(proj .Frameworks .SelectMany(f => f.Dependencies.Keys) .Concat(proj.Tools.Select(t => t.Id)), StringComparer.OrdinalIgnoreCase), - }); + }); } foreach (var packageId in packages.Keys) diff --git a/build/tasks/BuildGraph/ExternalDependency.cs b/build/tasks/BuildGraph/ExternalDependency.cs new file mode 100644 index 0000000000..5bdc67d027 --- /dev/null +++ b/build/tasks/BuildGraph/ExternalDependency.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace RepoTools.BuildGraph +{ + internal class ExternalDependency + { + public string PackageId { get; set; } + public string Version { get; set; } + public bool IsReferenced { get; set; } + } +} diff --git a/build/tasks/CodeGen/DirectedGraphXml.cs b/build/tasks/CodeGen/DirectedGraphXml.cs new file mode 100644 index 0000000000..e90d66ca6b --- /dev/null +++ b/build/tasks/CodeGen/DirectedGraphXml.cs @@ -0,0 +1,42 @@ + +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Xml.Linq; + +namespace RepoTasks.CodeGen +{ + class DirectedGraphXml + { + private readonly XNamespace _ns = "http://schemas.microsoft.com/vs/2009/dgml"; + private readonly XDocument _doc; + private readonly XElement _nodes; + private readonly XElement _links; + + public DirectedGraphXml() + { + _doc = new XDocument(new XElement(_ns + "DirectedGraph")); + _nodes = new XElement(_ns + "Nodes"); + _links = new XElement(_ns + "Links"); + _doc.Root.Add(_nodes); + _doc.Root.Add(_links); + } + + public void AddNode(string id) + { + _nodes.Add(new XElement(_ns + "Node", new XAttribute("Id", id), new XAttribute("Label", id))); + } + + public void AddLink(string source, string target) + { + _links.Add(new XElement(_ns + "Link", + new XAttribute("Source", source), + new XAttribute("Target", target))); + } + + public void Save(string path) + { + _doc.Save(path); + } + } +} diff --git a/build/tasks/CodeGen/GenerateSubmoduleGraph.cs b/build/tasks/CodeGen/GenerateSubmoduleGraph.cs new file mode 100644 index 0000000000..7940b5cba2 --- /dev/null +++ b/build/tasks/CodeGen/GenerateSubmoduleGraph.cs @@ -0,0 +1,226 @@ + +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Text; +using System.Threading; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Frameworks; +using NuGet.Versioning; +using RepoTools.BuildGraph; +using RepoTasks.ProjectModel; +using RepoTasks.Utilities; +using RepoTasks.CodeGen; +using NuGet.Packaging.Core; + +namespace RepoTasks +{ + public class GenerateSubmoduleGraph : Task, ICancelableTask + { + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + + /// + /// Repositories that we are building new versions of. + /// + [Required] + public ITaskItem[] Solutions { get; set; } + + [Required] + public ITaskItem[] Artifacts { get; set; } + + [Required] + public ITaskItem[] Repositories { get; set; } + + [Required] + public string RepositoryRoot { get; set; } + + [Required] + public string Properties { get; set; } + + public void Cancel() + { + _cts.Cancel(); + } + + public override bool Execute() + { + var packageArtifacts = Artifacts.Select(ArtifactInfo.Parse) + .OfType() + .Where(p => !p.IsSymbolsArtifact) + .ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase); + + var factory = new SolutionInfoFactory(Log, BuildEngine5); + var props = MSBuildListSplitter.GetNamedProperties(Properties); + + Log.LogMessage(MessageImportance.High, $"Beginning cross-repo analysis on {Solutions.Length} solutions. Hang tight..."); + + if (!props.TryGetValue("Configuration", out var defaultConfig)) + { + defaultConfig = "Debug"; + } + + var solutions = factory.Create(Solutions, props, defaultConfig, _cts.Token).OrderBy(f => f.Directory).ToList(); + Log.LogMessage($"Found {solutions.Count} and {solutions.Sum(p => p.Projects.Count)} projects"); + + if (_cts.IsCancellationRequested) + { + return false; + } + + return GenerateGraph(packageArtifacts, solutions); + } + + private bool GenerateGraph(IDictionary packageArtifacts, IReadOnlyList solutions) + { + var repoGraph = new AdjacencyMatrix(solutions.Count); + var packageToProjectMap = new Dictionary(); + + for (var i = 0; i < solutions.Count; i++) + { + var sln = repoGraph[i] = solutions[i]; + + foreach (var proj in sln.Projects) + { + if (!proj.IsPackable || proj.FullPath.Contains("samples")) + { + continue; + } + + var id = new PackageIdentity(proj.PackageId, new NuGetVersion(proj.PackageVersion)); + + if (packageToProjectMap.TryGetValue(id, out var otherProj)) + { + Log.LogError($"Both {proj.FullPath} and {otherProj.FullPath} produce {id}"); + continue; + } + + packageToProjectMap.Add(id, proj); + } + + var sharedSrc = Path.Combine(sln.Directory, "shared"); + if (Directory.Exists(sharedSrc)) + { + foreach (var dir in Directory.GetDirectories(sharedSrc, "*.Sources")) + { + var id = Path.GetFileName(dir); + var artifactInfo = packageArtifacts[id]; + var sharedSrcProj = new ProjectInfo(dir, + Array.Empty(), + Array.Empty(), + true, + artifactInfo.PackageInfo.Id, + artifactInfo.PackageInfo.Version.ToNormalizedString()); + sharedSrcProj.SolutionInfo = sln; + var identity = new PackageIdentity(artifactInfo.PackageInfo.Id, artifactInfo.PackageInfo.Version); + packageToProjectMap.Add(identity, sharedSrcProj); + } + } + } + + if (Log.HasLoggedErrors) + { + return false; + } + + for (var i = 0; i < solutions.Count; i++) + { + var sln = repoGraph[i]; + + var deps = from proj in sln.Projects + from tfm in proj.Frameworks + from dep in tfm.Dependencies.Values + select dep; + + foreach (var dep in deps) + { + if (packageToProjectMap.TryGetValue(new PackageIdentity(dep.Id, new NuGetVersion(dep.Version)), out var target)) + { + var j = repoGraph.FindIndex(target.SolutionInfo); + repoGraph.SetLink(i, j); + } + } + + var toolDeps = from proj in sln.Projects + from tool in proj.Tools + select tool; + + foreach (var toolDep in toolDeps) + { + if (packageToProjectMap.TryGetValue(new PackageIdentity(toolDep.Id, new NuGetVersion(toolDep.Version)), out var target)) + { + var j = repoGraph.FindIndex(target.SolutionInfo); + repoGraph.SetLink(i, j); + } + } + } + + CreateDgml(repoGraph); + return !Log.HasLoggedErrors; + } + + + private void CreateDgml(AdjacencyMatrix repoGraph) + { + var dgml = new DirectedGraphXml(); + + for (var i = 0; i < repoGraph.Count; i++) + { + var node = repoGraph[i]; + var nodeName = Path.GetFileName(node.Directory); + dgml.AddNode(nodeName); + + for (var j = 0; j < repoGraph.Count; j++) + { + if (j == i) continue; + if (repoGraph.HasLink(i, j)) + { + var target = repoGraph[j]; + var targetName = Path.GetFileName(target.Directory); + dgml.AddLink(nodeName, targetName); + } + } + } + + dgml.Save(Path.Combine(RepositoryRoot, "modules", "SubmoduleGraph.dgml")); + } + + private class AdjacencyMatrix + { + private readonly bool[,] _matrix; + private readonly SolutionInfo[] _items; + + public AdjacencyMatrix(int size) + { + _matrix = new bool[size, size]; + _items = new SolutionInfo[size]; + Count = size; + } + + public SolutionInfo this[int idx] + { + get => _items[idx]; + set => _items[idx] = value; + } + + public int FindIndex(SolutionInfo item) + { + return Array.FindIndex(_items, t => t.Equals(item)); + } + + public int Count { get; } + + public bool HasLink(int source, int target) => _matrix[source, target]; + + public void SetLink(int source, int target) + { + _matrix[source, target] = true; + } + } + } +} diff --git a/build/tasks/CodeGen/RepositoryProject.cs b/build/tasks/CodeGen/RepositoryProject.cs new file mode 100644 index 0000000000..1cb3b76391 --- /dev/null +++ b/build/tasks/CodeGen/RepositoryProject.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Construction; +using Microsoft.Build.Evaluation; + +namespace RepoTasks.CodeGen +{ + class RepositoryProject + { + private readonly ProjectRootElement _doc; + + public RepositoryProject(string repositoryRoot) + { + _doc = ProjectRootElement.Create(NewProjectFileOptions.None); + var import = _doc.CreateImportElement(@"$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props"); + var propGroup = _doc.AddPropertyGroup(); + if (repositoryRoot[repositoryRoot.Length - 1] != '\\') + { + repositoryRoot += '\\'; + } + propGroup.AddProperty("RepositoryRoot", repositoryRoot); + _doc.AddItemGroup(); + _doc.PrependChild(import); + _doc.AddImport(@"$(MSBuildToolsPath)\Microsoft.Common.targets"); + } + + public void AddProjectReference(string path) + { + _doc.AddItem("ProjectReference", path); + } + + public void AddProperty(string name, string value) + { + _doc.AddProperty(name, value); + } + + public void Save(string filePath) + { + _doc.Save(filePath, Encoding.UTF8); + } + } +} diff --git a/build/tasks/ProjectModel/ProjectInfo.cs b/build/tasks/ProjectModel/ProjectInfo.cs index 1dd4339185..4f4b7a3a9c 100644 --- a/build/tasks/ProjectModel/ProjectInfo.cs +++ b/build/tasks/ProjectModel/ProjectInfo.cs @@ -10,7 +10,6 @@ namespace RepoTasks.ProjectModel internal class ProjectInfo { public ProjectInfo(string fullPath, - string projectExtensionsPath, IReadOnlyList frameworks, IReadOnlyList tools, bool isPackable, @@ -28,7 +27,6 @@ namespace RepoTasks.ProjectModel FullPath = fullPath; FileName = Path.GetFileName(fullPath); Directory = Path.GetDirectoryName(FullPath); - ProjectExtensionsPath = projectExtensionsPath ?? Path.Combine(Directory, "obj"); IsPackable = isPackable; PackageId = packageId; PackageVersion = packageVersion; @@ -36,12 +34,13 @@ namespace RepoTasks.ProjectModel public string FullPath { get; } public string FileName { get; } - public string ProjectExtensionsPath { get; } public string Directory { get; } public string PackageId { get; } public string PackageVersion { get; } public bool IsPackable { get; } + public SolutionInfo SolutionInfo { get; set; } + public IReadOnlyList Frameworks { get; } public IReadOnlyList Tools { get; } } diff --git a/build/tasks/ProjectModel/ProjectInfoFactory.cs b/build/tasks/ProjectModel/ProjectInfoFactory.cs index 2acf90be1c..42c0bd3fee 100644 --- a/build/tasks/ProjectModel/ProjectInfoFactory.cs +++ b/build/tasks/ProjectModel/ProjectInfoFactory.cs @@ -27,7 +27,6 @@ namespace RepoTasks.ProjectModel { var project = GetProject(path, projectCollection); var instance = project.CreateProjectInstance(ProjectInstanceSettings.ImmutableWithFastItemLookup); - var projExtPath = instance.GetPropertyValue("MSBuildProjectExtensionsPath"); var targetFrameworks = instance.GetPropertyValue("TargetFrameworks"); var targetFramework = instance.GetPropertyValue("TargetFramework"); @@ -63,7 +62,6 @@ namespace RepoTasks.ProjectModel var packageVersion = instance.GetPropertyValue("PackageVersion"); return new ProjectInfo(path, - projExtPath, frameworks, tools, isPackable, diff --git a/build/tasks/ProjectModel/SolutionInfo.cs b/build/tasks/ProjectModel/SolutionInfo.cs index 8b9710081e..21af5e5291 100644 --- a/build/tasks/ProjectModel/SolutionInfo.cs +++ b/build/tasks/ProjectModel/SolutionInfo.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO; namespace RepoTasks.ProjectModel { @@ -21,13 +22,20 @@ namespace RepoTasks.ProjectModel } FullPath = fullPath; + Directory = Path.GetDirectoryName(fullPath); ConfigName = configName; Projects = projects ?? throw new ArgumentNullException(nameof(projects)); ShouldBuild = shouldBuild; Shipped = shipped; + + foreach (var proj in Projects) + { + proj.SolutionInfo = this; + } } public string FullPath { get; } + public string Directory { get; } public string ConfigName { get; } public IReadOnlyList Projects { get; } public bool ShouldBuild { get; } diff --git a/build/tasks/RepoTasks.csproj b/build/tasks/RepoTasks.csproj index fbfd560a7c..7c95128fdd 100644 --- a/build/tasks/RepoTasks.csproj +++ b/build/tasks/RepoTasks.csproj @@ -4,7 +4,7 @@ - netstandard2.0 + netcoreapp2.0 diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 7e014a92a3..6f2032d491 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -9,6 +9,7 @@ + diff --git a/build/tasks/tasks.sln b/build/tasks/tasks.sln new file mode 100644 index 0000000000..917ff4ff83 --- /dev/null +++ b/build/tasks/tasks.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RepoTasks", "RepoTasks.csproj", "{A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x64.ActiveCfg = Debug|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x64.Build.0 = Debug|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x86.ActiveCfg = Debug|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x86.Build.0 = Debug|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|Any CPU.Build.0 = Release|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x64.ActiveCfg = Release|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x64.Build.0 = Release|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x86.ActiveCfg = Release|Any CPU + {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/korebuild-lock.txt b/korebuild-lock.txt index b48af92327..a6327f8728 100644 --- a/korebuild-lock.txt +++ b/korebuild-lock.txt @@ -1,2 +1,2 @@ -version:2.1.0-preview2-15769 -commithash:6481a3a15bebd69f28edfce53f655b1bef6048a8 +version:2.1.0-rc1-15770 +commithash:a2abdc6fa0e8a1a465e830100950f1ecc698ab28 diff --git a/modules/SubmoduleGraph.dgml b/modules/SubmoduleGraph.dgml new file mode 100644 index 0000000000..cdcd0a92cd --- /dev/null +++ b/modules/SubmoduleGraph.dgml @@ -0,0 +1,421 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/GenerateTags.ps1 b/scripts/GenerateTags.ps1 index 958ab36378..5fc2ed9ee7 100755 --- a/scripts/GenerateTags.ps1 +++ b/scripts/GenerateTags.ps1 @@ -6,12 +6,15 @@ to the value in version.props .PARAMETER Push Push the tag to origin +.PARAMETER OutFile + When specified, generate a .csv with repo names and tags .PARAMETER WhatIf Dry run #> [cmdletbinding(PositionalBinding = $false, SupportsShouldProcess = $true)] param( - [switch]$Push + [switch]$Push, + [string]$OutFile ) $ErrorActionPreference = 'Stop' @@ -57,6 +60,7 @@ function New-GitTag { function Get-PackageVersion([string]$repoRoot) { $buildScript = if (-not $IsCoreCLR -or $IsWindows) { 'build.ps1' } else { 'build.sh' } $inspectTarget = "/p:CustomAfterKoreBuildTargets=$PSScriptRoot/GetPackageVersion.targets" + Write-Verbose "Running `"$repoRoot/$buildScript`" $inspectTarget /v:m /p:IsFinalBuild=true /t:Noop /t:GetPackageVersion" # Add the /t:Noop target which may be used by the bootstrapper to skip unimportant initialization $output = & "$repoRoot/$buildScript" $inspectTarget /v:m /p:IsFinalBuild=true /t:Noop /t:GetPackageVersion $output | out-string | Write-Verbose @@ -64,7 +68,7 @@ function Get-PackageVersion([string]$repoRoot) { throw "$buildScript failed on $repoRoot. Exit code $LASTEXITCODE" } $packageVersion = $output | where-object { $_ -like '*PackageVersion=*' } | select-object -first 1 - $packageVersion = $packageVersion -replace 'PackageVersion=','' + $packageVersion = $packageVersion -replace 'PackageVersion=', '' if ($packageVersion) { $packageVersion = $packageVersion.Trim() } if (-not $packageVersion) { throw "Could not determine final package version for $repoRoot" @@ -80,9 +84,16 @@ if (-not $PSCmdlet.ShouldContinue("Continue?", "This will apply tags to all subm exit 1 } + $universeTag = Get-PackageVersion $repoRoot New-GitTag $repoRoot $universeTag -WhatIf:$WhatIfPreference +$tags = @([pscustomobject] @{ + repo = $(git config remote.origin.url) + tag = $universeTag + commit = $(git rev-parse HEAD) + }) + Get-Submodules $repoRoot | ForEach-Object { $modPath = $_.path $module = $_.module @@ -96,6 +107,11 @@ Get-Submodules $repoRoot | ForEach-Object { if ($tag -ne $universeTag) { Write-Warning "${module}: version ($tag) does not match universe ($universeTag)" } + $tags += [pscustomobject] @{ + repo = $_.remote + tag = $tag + commit = $_.commit + } } catch { Write-Warning "${module}: Could not automatically determine tag for $modPath. Skipping" @@ -104,3 +120,9 @@ Get-Submodules $repoRoot | ForEach-Object { New-GitTag $_.path $tag -WhatIf:$WhatIfPreference } + +$tags | Format-Table + +if ($OutFile) { + $tags | Select-Object -Property * | Export-Csv -Path $OutFile -WhatIf:$false -NoTypeInformation +} diff --git a/scripts/UpdateBuildTools.ps1 b/scripts/UpdateBuildTools.ps1 index 63d05cc4c2..9e2ac5541d 100755 --- a/scripts/UpdateBuildTools.ps1 +++ b/scripts/UpdateBuildTools.ps1 @@ -13,6 +13,8 @@ Additional arguments to pass into git-commit .PARAMETER NoCommit Make changes without executing git-commit +.PARAMETER ToolsSource + The location of the build tools .PARAMETER Force Specified this to make a commit with any changes #> @@ -22,6 +24,7 @@ param( [string]$GitAuthorName = $null, [string]$GitAuthorEmail = $null, [string[]]$GitCommitArgs = @(), + [string]$ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools', [switch]$NoCommit, [switch]$Force ) @@ -54,7 +57,7 @@ try { $oldVersion = Get-KoreBuildVersion - & "$RepoRoot/run.ps1" -Update -Command noop | Out-Null + & "$RepoRoot/run.ps1" -Update -ToolsSource $ToolsSource -Command noop | Out-Null $newVersion = Get-KoreBuildVersion diff --git a/scripts/UpdateDependencies.ps1 b/scripts/UpdateDependencies.ps1 index 4447719d6f..fa733f24bb 100755 --- a/scripts/UpdateDependencies.ps1 +++ b/scripts/UpdateDependencies.ps1 @@ -7,12 +7,23 @@ param( [Parameter(Mandatory = $true)] $BuildXml, + [switch] + $NoCommit, [string[]]$ConfigVars = @() ) $ErrorActionPreference = 'Stop' Import-Module -Scope Local -Force "$PSScriptRoot/common.psm1" Set-StrictMode -Version 1 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +if (-not $NoCommit) { + $GitHubEmail = $ConfigVars["GithubEmail"] + $GitHubUsername = $ConfigVars["GithubUsername"] + $GitHubPassword = $ConfigVars["GithubToken"] + + Set-GitHubInfo $GitHubPassword $GitHubUsername $GitHubEmail +} $depsPath = Resolve-Path "$PSScriptRoot/../build/dependencies.props" [xml] $dependencies = LoadXml $depsPath @@ -20,13 +31,12 @@ $depsPath = Resolve-Path "$PSScriptRoot/../build/dependencies.props" if ($BuildXml -like 'http*') { $url = $BuildXml New-Item -Type Directory "$PSScriptRoot/../obj/" -ErrorAction Ignore - $localXml = "$PSScriptRoot/../obj/build.xml" + $BuildXml = "$PSScriptRoot/../obj/build.xml" Write-Verbose "Downloading from $url to $BuildXml" - Invoke-WebRequest -OutFile $localXml $url + Invoke-WebRequest -OutFile $BuildXml $url } -[xml] $remoteDeps = LoadXml $localXml -$count = 0 +[xml] $remoteDeps = LoadXml $BuildXml $variables = @{} @@ -46,50 +56,15 @@ foreach ($package in $remoteDeps.SelectNodes('//Package')) { } } -$updatedVars = @{} +$updatedVars = UpdateVersions $variables $dependencies $depsPath -foreach ($varName in ($variables.Keys | sort)) { - $packageVersions = $variables[$varName] - if ($packageVersions.Length -gt 1) { - Write-Warning "Skipped $varName. Multiple version found. { $($packageVersions -join ', ') }." - continue - } +if (-not $NoCommit) { + $body = CommitUpdatedVersions $updatedVars $dependencies $depsPath + $destinationBranch = "dotnetbot/UpdateDeps" - $packageVersion = $packageVersions | Select-Object -First 1 + $baseBranch = $ConfigVars["GithubUpstreamBranch"] - $depVarNode = $dependencies.SelectSingleNode("//PropertyGroup[`@Label=`"Package Versions: Auto`"]/$varName") - if ($depVarNode -and $depVarNode.InnerText -ne $packageVersion) { - $depVarNode.InnerText = $packageVersion - $count++ - Write-Host -f DarkGray " Updating $varName to $packageVersion" - $updatedVars[$varName] = $packageVersion + if ($body) { + CreatePR $baseBranch $destinationBranch $body $GitHubPassword } } - -if ($count -gt 0) { - Write-Host -f Cyan "Updating $count version variables in $depsPath" - SaveXml $dependencies $depsPath - - # Ensure dotnet is installed - & "$PSScriptRoot\..\run.ps1" install-tools - - $ProjectPath = "$PSScriptRoot\update-dependencies\update-dependencies.csproj" - - $ConfigVars += "--BuildXml" - $ConfigVars += $BuildXml - - $ConfigVars += "--UpdatedVersions" - $varString = "" - foreach ($updatedVar in $updatedVars.GetEnumerator()) { - $varString += "$($updatedVar.Name)=$($updatedVar.Value)+" - } - $ConfigVars += $varString - - # Restore and run the app - Write-Host "Invoking App $ProjectPath..." - Invoke-Expression "dotnet run -p `"$ProjectPath`" $ConfigVars" - if ($LASTEXITCODE -ne 0) { throw "Build failed" } -} -else { - Write-Host -f Green "No changes found" -} diff --git a/scripts/UpdateDependenciesCoreFx.ps1 b/scripts/UpdateDependenciesCoreFx.ps1 new file mode 100644 index 0000000000..28db0aa6ec --- /dev/null +++ b/scripts/UpdateDependenciesCoreFx.ps1 @@ -0,0 +1,122 @@ + +[CmdletBinding()] +param( + [string]$GitHubEmail, + [string]$GitHubUsername, + [string]$GitHubPassword +) + +$ErrorActionPreference = 'Stop' +Import-Module -Scope Local -Force "$PSScriptRoot/common.psm1" +Set-StrictMode -Version 1 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +$githubRaw = "https://raw.githubusercontent.com" +$versionsRepo = "dotnet/versions" +$versionsBranch = "master" + +$coreSetupRepo = "dotnet/core-setup" +$coreFxRepo = "dotnet/corefx" + +$coreSetupVersions = "$githubRaw/$versionsRepo/$versionsBranch/build-info/$coreSetupRepo/master/Latest_Packages.txt" + +$tempDir = "$PSScriptRoot/../obj" +$localCoreSetupVersions = "$tempDir/coresetup.packages" +Write-Host "Downloading $coreSetupVersions to $localCoreSetupVersions" +Invoke-WebRequest -OutFile $localCoreSetupVersions -Uri $coreSetupVersions + +$msNetCoreAppPackageVersion = $null +$msNetCoreAppPackageName = "Microsoft.NETCore.App" + +Set-GitHubInfo $GitHubPassword $GitHubUsername $GitHubEmail + +$variables = @{} + +foreach ($line in Get-Content $localCoreSetupVersions) { + if ($line.StartsWith("$msNetCoreAppPackageName ")) { + $msNetCoreAppPackageVersion = $line.Trim("$msNetCoreAppPackageName ") + } + $parts = $line.Split(' ') + $packageName = $parts[0] + + $varName = "$packageName" + "PackageVersion" + $varName = $varName.Replace('.', '') + + $packageVersion = $parts[1] + if ($variables[$varName]) { + if ($variables[$varName].Where( {$_ -eq $packageVersion}, 'First').Count -eq 0) { + $variables[$varName] += $packageVersion + } + } + else { + $variables[$varName] = @($packageVersion) + } +} + +if (!$msNetCoreAppPackageVersion) { + Throw "$msNetCoreAppPackageName was not in $coreSetupVersions" +} + +$coreAppDownloadLink = "https://dotnet.myget.org/F/dotnet-core/api/v2/package/$msNetCoreAppPackageName/$msNetCoreAppPackageVersion" +$netCoreAppNupkg = "$tempDir/microsoft.netcore.app.zip" +Invoke-WebRequest -OutFile $netCoreAppNupkg -Uri $coreAppDownloadLink +$expandedNetCoreApp = "$tempDir/microsoft.netcore.app/" +Expand-Archive -Path $netCoreAppNupkg -DestinationPath $expandedNetCoreApp -Force +$versionsTxt = "$expandedNetCoreApp/$msNetCoreAppPackageName.versions.txt" + +$versionsCoreFxCommit = $null +foreach ($line in Get-Content $versionsTxt) { + if ($line.StartsWith("dotnet/versions/corefx")) { + $versionsCoreFxCommit = $line.Split(' ')[1] + break + } +} + +if (!$versionsCoreFxCommit) { + Throw "no 'dotnet/versions/corefx' in versions.txt of Microsoft.NETCore.App" +} + +$coreFxVersionsUrl = "$githubRaw/$versionsRepo/$versionsCoreFxCommit/build-info/$coreFxRepo/$versionsBranch/Latest_Packages.txt" +$localCoreFxVersions = "$tempDir/$corefx.packages" +Invoke-WebRequest -OutFile $localCoreFxVersions -Uri $coreFxVersionsUrl + +foreach ($line in Get-Content $localCoreFxVersions) { + $parts = $line.Split(' ') + + $packageName = $parts[0] + + $varName = "$packageName" + "PackageVersion" + $varName = $varName.Replace('.', '') + $packageVersion = $parts[1] + if ($variables[$varName]) { + if ($variables[$varName].Where( {$_ -eq $packageVersion}, 'First').Count -eq 0) { + $variables[$varName] += $packageVersion + } + } + else { + $variables[$varName] = @($packageVersion) + } +} + +$depsPath = Resolve-Path "$PSScriptRoot/../build/dependencies.props" +Write-Host "Loading deps from $depsPath" +[xml] $dependencies = LoadXml $depsPath + +$remote = "origin" +$baseBranch = "release/2.1" + +$currentBranch = Invoke-Block { & git rev-parse --abbrev-ref HEAD } +$destinationBranch = "rybrande/UpgradeDepsTest" + +Invoke-Block { & git checkout -tb $destinationBranch "$remote/$baseBranch" } +try { + $updatedVars = UpdateVersions $variables $dependencies $depsPath + $body = CommitUpdatedVersions $updatedVars $dependencies $depsPath + + if ($body) { + CreatePR $baseBranch $destinationBranch $body $GitHubPassword + } +} +finally { + Invoke-Block { & git checkout $currentBranch } +} diff --git a/scripts/common.psm1 b/scripts/common.psm1 index 87205a92d5..b621d48193 100644 --- a/scripts/common.psm1 +++ b/scripts/common.psm1 @@ -1,3 +1,5 @@ +$ErrorActionPreference = 'Stop' + function Assert-Git { if (!(Get-Command git -ErrorAction Ignore)) { Write-Error 'git is required to execute this script' @@ -54,6 +56,7 @@ function Get-Submodules { commit = $(git rev-parse HEAD) newCommit = $null changed = $false + remote = $(git config remote.origin.url) branch = $(git config -f $moduleConfigFile --get submodule.modules/$($_.Name).branch ) versionPrefix = $versionPrefix versionSuffix = $versionSuffix @@ -109,3 +112,105 @@ function PackageIdVarName([string]$packageId) { $canonicalVarName += "PackageVersion" return $canonicalVarName } + +function Ensure-Hub() { + $tmpDir = "$PSScriptRoot\tmp" + $zipDir = "$tmpDir\Hub" + $hubLocation = "$zipDir\bin\hub.exe" + + if (-Not (Test-Path $hubLocation) ) { + $source = "https://github.com/github/hub/releases/download/v2.3.0-pre9/hub-windows-amd64-2.3.0-pre9.zip" + $zipLocation = "$tmpDir\hub.zip" + if(-not (Test-Path $zipLocation)) { + New-Item -ItemType directory -Path $tmpDir + } + + Invoke-WebRequest -OutFile $zipLocation -Uri $source + + Expand-Archive -Path $zipLocation -DestinationPath $zipDir -Force + if (-Not (Test-Path $hubLocation)) { + throw "Hub couldn't be downloaded" + } + } + + return $hubLocation +} + +function CreatePR([string]$baseBranch, [string]$destinationBranch, [string]$body, [string]$gitHubToken) { + $hubLocation = Ensure-Hub + + Invoke-Block { git push -f https://$gitHubToken@github.com/aspnet/Universe.git $destinationBranch } + & $hubLocation pull-request -f -b $baseBranch -h $destinationBranch -m $body +} + +function Set-GithubInfo( + [string]$GitHubPassword, + [string]$GitHubUser, + [string]$GitHubEmail) +{ + $Env:GITHUB_TOKEN = $GitHubPassword + $Env:GITHUB_USER = $GitHubUser + $Env:GITHUB_EMAIL = $GitHubEmail +} +function CommitUpdatedVersions( + [hashtable]$updatedVars, + [xml]$dependencies, + [string]$depsPath) +{ + $count = $updatedVars.Count + if ($count -gt 0) { + & git add build\dependencies.props + + $subject = "Updating external dependencies" + + # Have to pipe null so that the output from this doesn't end up as part of the return value + $null = Invoke-Block { & git commit -m $subject } + + $body = "$subject`n`n" + + $body += "New versions:`n" + + foreach ($var in $updatedVars.GetEnumerator()) { + $body += " $($var.Name)`n" + } + + return $body + } +} + +function UpdateVersions([hashtable]$variables, [xml]$dependencies, [string]$depsPath) { + $updatedVars = @{} + + foreach ($varName in ($variables.Keys | sort)) { + $packageVersions = $variables[$varName] + if ($packageVersions.Length -gt 1) { + Write-Warning "Skipped $varName. Multiple version found. { $($packageVersions -join ', ') }." + continue + } + + $packageVersion = $packageVersions | Select-Object -First 1 + + $depVarNode = $dependencies.SelectSingleNode("//PropertyGroup[`@Label=`"Package Versions: Auto`"]/$varName") + if ($depVarNode -and $depVarNode.InnerText -ne $packageVersion) { + $depVarNode.InnerText = $packageVersion + Write-Host -f DarkGray " Updating $varName to $packageVersion" + $updatedVars[$varName] = $packageVersion + } + elseif ($depVarNode) { + Write-Host -f DarkBlue " Didn't update $varName to $packageVersion because it was $($depVarNode.InnerText)" + } + else { + # This isn't a dependency we use + } + } + + if ($updatedVars.Count -gt 0) { + Write-Host -f Cyan "Updating $count version variables in $depsPath" + SaveXml $dependencies $depsPath + } + else { + Write-Host -f Green "No changes found" + } + + return $updatedVars +} diff --git a/scripts/update-dependencies/Config.cs b/scripts/update-dependencies/Config.cs deleted file mode 100644 index 38a59ac062..0000000000 --- a/scripts/update-dependencies/Config.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace Microsoft.Dotnet.Scripts -{ - public class Config - { - public string UpdatedVersions { get; set; } - public string BuildXml { get; set; } - public string GithubUsername {get; set;} - public string GithubEmail {get; set;} - public string GithubToken {get; set;} - public string GithubUpstreamOwner {get; set;} = "aspnet"; - public string GithubProject {get; set;} = "Universe"; - public string GithubUpstreamBranch {get; set;} = "dev"; - public string[] GitHubPullRequestNotifications { get; set; } = new string[] { }; - - public string[] UpdatedVersionsList - { - get - { - return UpdatedVersions.Split('+',System.StringSplitOptions.RemoveEmptyEntries); - } - } - } -} diff --git a/scripts/update-dependencies/Program.cs b/scripts/update-dependencies/Program.cs deleted file mode 100644 index 06ec564689..0000000000 --- a/scripts/update-dependencies/Program.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Diagnostics; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; -using System.Xml; -using Microsoft.DotNet.VersionTools.Automation; -using Microsoft.Extensions.Configuration; - -namespace Microsoft.Dotnet.Scripts -{ - public static class Program - { - private static readonly Config _config = new Config(); - - public static async Task Main(string[] args) - { - Trace.Listeners.Add(new TextWriterTraceListener(Console.Out)); - - ParseArgs(args); - - await CreatePullRequest(); - } - - private static void ParseArgs(string[] args) - { - var builder = new ConfigurationBuilder().AddCommandLine(args); - var configRoot = builder.Build(); - configRoot.Bind(_config); - } - - private static async Task CreatePullRequest() - { - var gitHubAuth = new GitHubAuth(_config.GithubToken, _config.GithubUsername, _config.GithubEmail); - var origin = new GitHubProject(_config.GithubProject, _config.GithubUsername); - var upstreamBranch = new GitHubBranch(_config.GithubUpstreamBranch, new GitHubProject(_config.GithubProject, _config.GithubUpstreamOwner)); - - var commitMessage = $"Updating external dependencies to '{ await GetOrchestratedBuildId() }'"; - var body = string.Empty; - if (_config.GitHubPullRequestNotifications.Any()) - { - body += PullRequestCreator.NotificationString(_config.GitHubPullRequestNotifications); - } - - body += $"New versions:{Environment.NewLine}"; - - foreach (var updatedVersion in _config.UpdatedVersionsList) - { - body += $" {updatedVersion}{Environment.NewLine}"; - } - - await new PullRequestCreator(gitHubAuth, origin, upstreamBranch) - .CreateOrUpdateAsync(commitMessage, commitMessage + $" ({upstreamBranch.Name})", body); - } - - private static async Task GetOrchestratedBuildId() - { - var xmlUrl = _config.BuildXml; - - using (var client = new HttpClient()) - { - var response = await client.GetAsync(xmlUrl); - using (var bodyStream = await response.Content.ReadAsStreamAsync()) - { - var xmlDoc = new XmlDocument(); - xmlDoc.Load(bodyStream); - var orcBuilds = xmlDoc.GetElementsByTagName("OrchestratedBuild"); - - if (orcBuilds.Count < 1) - { - throw new ArgumentException($"{xmlUrl} didn't have an 'OrchestratedBuild' element."); - } - - var orcBuild = orcBuilds[0]; - - return orcBuild.Attributes["BuildId"].Value; - } - } - } - } -} diff --git a/scripts/update-dependencies/update-dependencies.csproj b/scripts/update-dependencies/update-dependencies.csproj deleted file mode 100644 index 2887812fbb..0000000000 --- a/scripts/update-dependencies/update-dependencies.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - Exe - netcoreapp2.0 - 7.1 - - https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json; - https://api.nuget.org/v3/index.json; - - - - - - - - - - - - diff --git a/version.props b/version.props index 17d1dca473..7c8a108995 100644 --- a/version.props +++ b/version.props @@ -1,7 +1,7 @@ 2.1.0 - preview2 + rc1 0.1.0 alpha2