From 177fa716342b19a3117d2c9be0cf4820390bee87 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Mon, 18 Sep 2017 16:44:12 -0700 Subject: [PATCH] Copy packages to ship/noship and verify coherent versions (#560) --- .editorconfig | 29 +++++ build/artifacts.props | 66 +++++++++++ build/repo.props | 2 + build/repo.targets | 24 +++- build/tasks/CopyPackagesToSplitFolders.cs | 100 +++++++++++++++++ build/tasks/GenerateLineup.cs | 2 +- build/tasks/ProjectModel/PackageInfo.cs | 16 +-- build/tasks/RepoTasks.tasks | 2 + build/tasks/Utilities/ArtifactInfo.cs | 9 +- build/tasks/Utilities/PackageCategory.cs | 14 +++ build/tasks/Utilities/PackageCollection.cs | 66 +++++++++++ build/tasks/VerifyCoherentVersions.cs | 121 +++++++++++++++++++++ 12 files changed, 436 insertions(+), 15 deletions(-) create mode 100644 .editorconfig create mode 100644 build/artifacts.props create mode 100644 build/tasks/CopyPackagesToSplitFolders.cs create mode 100644 build/tasks/Utilities/PackageCategory.cs create mode 100644 build/tasks/Utilities/PackageCollection.cs create mode 100644 build/tasks/VerifyCoherentVersions.cs diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4eb7559fce --- /dev/null +++ b/.editorconfig @@ -0,0 +1,29 @@ +; EditorConfig to support per-solution formatting. +; Use the EditorConfig VS add-in to make this work. +; http://editorconfig.org/ + +; This is the default for the codeline. +root = true + +[*] +indent_style = space +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{cs}] +indent_size = 4 +dotnet_sort_system_directives_first = true:warning + +[*.{xml,config,*proj,nuspec,props,resx,targets,yml,tasks}] +indent_size = 2 + +[*.json] +indent_size = 2 + +[*.{ps1,psm1}] +indent_size = 4 + +[*.sh] +indent_size = 4 +end_of_line = lf diff --git a/build/artifacts.props b/build/artifacts.props new file mode 100644 index 0000000000..d5f8ba5fe2 --- /dev/null +++ b/build/artifacts.props @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build/repo.props b/build/repo.props index 844540c47a..a98645c000 100644 --- a/build/repo.props +++ b/build/repo.props @@ -3,4 +3,6 @@ true + + diff --git a/build/repo.targets b/build/repo.targets index 3845106a4f..72eaac18f2 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -19,7 +19,9 @@ $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts - $(BuildDependsOn);CloneRepositories;BuildRepositories + $(CompileDependsOn);CloneRepositories;BuildRepositories + $(PackageDependsOn);SplitPackages + $(VerifyDependsOn);VerifyCoherentVersions @@ -181,6 +183,26 @@ BuildNumber="$(BuildNumber)" /> + + + + + + + + + + + + + + + + $(BuildDir)$(_RepositoryListFileName) diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs new file mode 100644 index 0000000000..8f780e270f --- /dev/null +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -0,0 +1,100 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class CopyPackagesToSplitFolders : Microsoft.Build.Utilities.Task + { + /// + /// The item group containing the nuget packages to split in different folders. + /// + [Required] + public ITaskItem[] Packages { get; set; } + + [Required] + public ITaskItem[] Files { get; set; } + + /// + /// The folder where packages should be copied. Subfolders will be created based on package category. + /// + [Required] + public string DestinationFolder { get; set; } + + public override bool Execute() + { + if (Files?.Length == 0) + { + Log.LogError("No packages were found."); + return false; + } + + var expectedPackages = PackageCollection.FromItemGroup(Packages); + + Directory.CreateDirectory(DestinationFolder); + + foreach (var file in Files) + { + PackageIdentity identity; + using (var reader = new PackageArchiveReader(file.ItemSpec)) + { + identity = reader.GetIdentity(); + } + + if (!expectedPackages.TryGetCategory(identity.Id, out var category)) + { + Log.LogError($"Unexpected package artifact with id: {identity.Id}"); + return false; + } + + string destDir; + switch (category) + { + case PackageCategory.Unknown: + throw new InvalidOperationException($"Package {identity} does not have a recognized package category."); + case PackageCategory.Shipping: + destDir = Path.Combine(DestinationFolder, "ship"); + break; + case PackageCategory.NoShip: + destDir = Path.Combine(DestinationFolder, "noship"); + break; + case PackageCategory.ShipOob: + destDir = Path.Combine(DestinationFolder, "shipoob"); + break; + default: + throw new NotImplementedException(); + } + + Directory.CreateDirectory(destDir); + + var destFile = Path.Combine(destDir, Path.GetFileName(file.ItemSpec)); + + Log.LogMessage($"Copying {file.ItemSpec} to {destFile}"); + + File.Copy(file.ItemSpec, destFile); + expectedPackages.Remove(identity.Id); + } + + if (expectedPackages.Count != 0) + { + var error = new StringBuilder(); + foreach (var key in expectedPackages.Keys) + { + error.Append(" - ").AppendLine(key); + } + + Log.LogError($"Expected the following packages, but they were not found:" + error.ToString()); + return false; + } + + return true; + } + } +} diff --git a/build/tasks/GenerateLineup.cs b/build/tasks/GenerateLineup.cs index 1acbd7c577..d1114814b5 100644 --- a/build/tasks/GenerateLineup.cs +++ b/build/tasks/GenerateLineup.cs @@ -47,7 +47,7 @@ namespace RepoTasks var root = new XElement("Project", props, items); var doc = new XDocument(root); - if (RestoreAdditionalSources.Length > 0) + if (RestoreAdditionalSources?.Length > 0) { var sources = RestoreAdditionalSources.Aggregate("$(RestoreAdditionalProjectSources)", (sum, piece) => sum + ";" + piece.ItemSpec); props.Add(new XElement("RestoreAdditionalProjectSources", sources)); diff --git a/build/tasks/ProjectModel/PackageInfo.cs b/build/tasks/ProjectModel/PackageInfo.cs index af1976686e..463fab690a 100644 --- a/build/tasks/ProjectModel/PackageInfo.cs +++ b/build/tasks/ProjectModel/PackageInfo.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; using NuGet.Frameworks; +using NuGet.Packaging; using NuGet.Versioning; namespace RepoTasks.ProjectModel @@ -12,8 +13,8 @@ namespace RepoTasks.ProjectModel internal class PackageInfo { public PackageInfo(string id, - string version, - IReadOnlyList frameworks, + NuGetVersion version, + IReadOnlyList dependencyGroups, string source, string packageType = "Dependency") { @@ -22,16 +23,11 @@ namespace RepoTasks.ProjectModel throw new ArgumentException(nameof(id)); } - if (string.IsNullOrEmpty(version)) - { - throw new ArgumentException(nameof(version)); - } - Id = id; - Version = NuGetVersion.Parse(version); - Frameworks = frameworks; + Version = version ?? throw new ArgumentNullException(nameof(version)); PackageType = packageType; Source = source; + DependencyGroups = dependencyGroups ?? Array.Empty(); } public string Id { get; } @@ -41,6 +37,6 @@ namespace RepoTasks.ProjectModel /// Can be a https feed or a file path. May be null. /// public string Source { get; } - public IReadOnlyList Frameworks { get; } + public IReadOnlyList DependencyGroups { get; } } } diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 505742d1e9..12ec53fe04 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -4,5 +4,7 @@ + + diff --git a/build/tasks/Utilities/ArtifactInfo.cs b/build/tasks/Utilities/ArtifactInfo.cs index f3bfa91ced..6e35289a3b 100644 --- a/build/tasks/Utilities/ArtifactInfo.cs +++ b/build/tasks/Utilities/ArtifactInfo.cs @@ -5,6 +5,9 @@ using System; using System.IO; using System.Linq; using NuGet.Frameworks; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; using Microsoft.Build.Framework; using RepoTasks.ProjectModel; @@ -50,10 +53,10 @@ namespace RepoTasks.Utilities { return new PackageInfo( item.GetMetadata("PackageId"), - item.GetMetadata("Version"), + NuGetVersion.Parse(item.GetMetadata("Version")), string.IsNullOrEmpty(item.GetMetadata("TargetFramework")) - ? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => NuGetFramework.Parse(s)).ToArray() - : new [] { NuGetFramework.Parse(item.GetMetadata("TargetFramework")) }, + ? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => new PackageDependencyGroup(NuGetFramework.Parse(s), Array.Empty())).ToArray() + : new [] { new PackageDependencyGroup(NuGetFramework.Parse(item.GetMetadata("TargetFramework")), Array.Empty()) }, Path.GetDirectoryName(item.ItemSpec), item.GetMetadata("PackageType")); } diff --git a/build/tasks/Utilities/PackageCategory.cs b/build/tasks/Utilities/PackageCategory.cs new file mode 100644 index 0000000000..80f2990244 --- /dev/null +++ b/build/tasks/Utilities/PackageCategory.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + + +namespace RepoTasks.Utilities +{ + public enum PackageCategory + { + Unknown = 0, + Shipping, + NoShip, + ShipOob + } +} diff --git a/build/tasks/Utilities/PackageCollection.cs b/build/tasks/Utilities/PackageCollection.cs new file mode 100644 index 0000000000..4aa778aa2a --- /dev/null +++ b/build/tasks/Utilities/PackageCollection.cs @@ -0,0 +1,66 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Build.Framework; + +namespace RepoTasks.Utilities +{ + public class PackageCollection + { + private readonly IDictionary _packages = new Dictionary(StringComparer.OrdinalIgnoreCase); + + private PackageCollection() + { + } + + public bool TryGetCategory(string packageId, out PackageCategory category) => _packages.TryGetValue(packageId, out category); + + public void Remove(string packageId) => _packages.Remove(packageId); + + public int Count => _packages.Count; + + public IEnumerable Keys => _packages.Keys; + + public static PackageCollection FromItemGroup(ITaskItem[] items) + { + var list = new PackageCollection(); + if (items == null) + { + return list; + } + + foreach (var item in items) + { + PackageCategory category; + switch (item.GetMetadata("Category")?.ToLowerInvariant()) + { + case "ship": + category = PackageCategory.Shipping; + break; + case "noship": + category = PackageCategory.NoShip; + break; + case "shipoob": + category = PackageCategory.ShipOob; + break; + default: + category = PackageCategory.Unknown; + break; + } + + if (list._packages.ContainsKey(item.ItemSpec)) + { + throw new InvalidDataException($"Duplicate package id detected: {item.ItemSpec}"); + } + + list._packages.Add(item.ItemSpec, category); + } + + return list; + } + } +} diff --git a/build/tasks/VerifyCoherentVersions.cs b/build/tasks/VerifyCoherentVersions.cs new file mode 100644 index 0000000000..6001041d0e --- /dev/null +++ b/build/tasks/VerifyCoherentVersions.cs @@ -0,0 +1,121 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using NuGet.Frameworks; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using RepoTasks.ProjectModel; +using RepoTasks.Utilities; + +namespace RepoTasks +{ + public class VerifyCoherentVersions : Microsoft.Build.Utilities.Task + { + public ITaskItem[] PackageFiles { get; set; } + + public override bool Execute() + { + var packageLookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var file in PackageFiles) + { + PackageInfo package; + using (var reader = new PackageArchiveReader(file.ItemSpec)) + { + var identity = reader.GetIdentity(); + var metadata = new PackageBuilder(reader.GetNuspec(), basePath: null); + package = new PackageInfo(identity.Id, identity.Version, + source: Path.GetDirectoryName(file.ItemSpec), + dependencyGroups: metadata.DependencyGroups.ToArray()); + } + + if (packageLookup.TryGetValue(package.Id, out var existingPackage)) + { + throw new Exception("Multiple copies of the following package were found: " + + Environment.NewLine + + existingPackage + + Environment.NewLine + + package); + } + + packageLookup[package.Id] = package; + } + + var dependencyIssues = new List(); + foreach (var packageInfo in packageLookup.Values) + { + dependencyIssues.AddRange(Visit(packageLookup, packageInfo)); + } + + var success = true; + foreach (var mismatch in dependencyIssues) + { + var message = $"{mismatch.Info.Id} depends on {mismatch.Dependency.Id} " + + $"v{mismatch.Dependency.VersionRange} ({mismatch.TargetFramework}) when the latest build is v{mismatch.Info.Version}."; + Log.LogError(message); + success = false; + } + + Log.LogMessage(MessageImportance.High, $"Verified {PackageFiles.Length} package(s) have coherent versions"); + return success; + } + + private class DependencyWithIssue + { + public PackageDependency Dependency { get; set; } + public PackageInfo Info { get; set; } + public NuGetFramework TargetFramework { get; set; } + } + + private IEnumerable Visit(IDictionary packageLookup, PackageInfo packageInfo) + { + Log.LogMessage(MessageImportance.Low, $"Processing package {packageInfo.Id}"); + try + { + var issues = new List(); + foreach (var dependencySet in packageInfo.DependencyGroups) + { + // If the package doens't target any frameworks, just accept it + if (dependencySet.TargetFramework == null) + { + continue; + } + + foreach (var dependency in dependencySet.Packages) + { + if (!packageLookup.TryGetValue(dependency.Id, out var dependencyPackageInfo)) + { + // External dependency + continue; + } + + if (dependencyPackageInfo.Version != dependency.VersionRange.MinVersion) + { + // For any dependency in the universe + // Add a mismatch if the min version doesn't work out + // (we only really care about >= minVersion) + issues.Add(new DependencyWithIssue + { + Dependency = dependency, + TargetFramework = dependencySet.TargetFramework, + Info = dependencyPackageInfo + }); + } + } + } + return issues; + } + catch + { + Log.LogError($"Unable to verify package {packageInfo.Id}"); + throw; + } + } + } +}