Copy packages to ship/noship and verify coherent versions (#560)

This commit is contained in:
Nate McMaster 2017-09-18 16:44:12 -07:00 committed by GitHub
parent 351156f455
commit 177fa71634
12 changed files with 436 additions and 15 deletions

29
.editorconfig Normal file
View File

@ -0,0 +1,29 @@
; EditorConfig to support per-solution formatting.
; Use the EditorConfig VS add-in to make this work.
; http://editorconfig.org/
; This is the default for the codeline.
root = true
[*]
indent_style = space
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.{cs}]
indent_size = 4
dotnet_sort_system_directives_first = true:warning
[*.{xml,config,*proj,nuspec,props,resx,targets,yml,tasks}]
indent_size = 2
[*.json]
indent_size = 2
[*.{ps1,psm1}]
indent_size = 4
[*.sh]
indent_size = 4
end_of_line = lf

66
build/artifacts.props Normal file
View File

@ -0,0 +1,66 @@
<Project>
<ItemGroup>
<PackageArtifact Include="Microsoft.AspNet.Identity.AspNetCoreCompat" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.All" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Diagnostics" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Diagnostics.Abstractions" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Diagnostics.Elm" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Diagnostics.Identity.Service" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Specification.Tests" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.Core" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.Abstractions" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.AzureKeyVault" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.EntityFrameworkCore" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.IntegratedWebClient" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.Mvc" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Identity.Service.Specification.Tests" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.MiddlewareAnalysis" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Abstractions" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.ApiExplorer" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Core" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Cors" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.DataAnnotations" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Formatters.Json" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Formatters.Xml" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Localization" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Razor" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.RazorPages" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Razor.ViewCompilation" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.TagHelpers" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.Testing" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.ViewFeatures" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.Mvc.WebApiCompatShim" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.NodeServices" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.AspNetCore.NodeServices.Sockets" Category="noship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.AspNetCore.SpaServices" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Design" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.InMemory" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Relational" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Relational.Design.Specification.Tests" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Relational.Specification.Tests" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Specification.Tests" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Sqlite" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Sqlite.Core" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.SqlServer" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Tools" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.EntityFrameworkCore.Tools.DotNet" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.Extensions.Identity.Core" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.Extensions.Identity.Stores" Category="ship" Metapackage="include" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.Contracts" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.Core" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.EntityFrameworkCore" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.Templating" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGeneration.Utils" Category="ship" Metapackage="exclude" />
<PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGenerators.Mvc" Category="ship" Metapackage="exclude" />
</ItemGroup>
</Project>

View File

@ -3,4 +3,6 @@
<!-- This repo does not have solutions to build -->
<DisableDefaultTargets>true</DisableDefaultTargets>
</PropertyGroup>
<Import Project="artifacts.props" />
</Project>

View File

@ -19,7 +19,9 @@
<PrepareDependsOn>$(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts</PrepareDependsOn>
<CleanDependsOn>$(CleanDependsOn);CleanUniverseArtifacts</CleanDependsOn>
<BuildDependsOn>$(BuildDependsOn);CloneRepositories;BuildRepositories</BuildDependsOn>
<CompileDependsOn>$(CompileDependsOn);CloneRepositories;BuildRepositories</CompileDependsOn>
<PackageDependsOn>$(PackageDependsOn);SplitPackages</PackageDependsOn>
<VerifyDependsOn>$(VerifyDependsOn);VerifyCoherentVersions</VerifyDependsOn>
</PropertyGroup>
<Import Project="$(_RepositoryListToImport)" />
@ -181,6 +183,26 @@
BuildNumber="$(BuildNumber)" />
</Target>
<Target Name="SplitPackages">
<ItemGroup>
<PackageArtifactFile Include="$(BuildDir)*.nupkg" Exclude="$(BuildDir)*.symbols.nupkg" />
</ItemGroup>
<RepoTasks.CopyPackagesToSplitFolders
Packages="@(PackageArtifact)"
Files="@(PackageArtifactFile)"
DestinationFolder="$(ArtifactsDir)" />
</Target>
<Target Name="VerifyCoherentVersions">
<ItemGroup>
<ShippingPackageFiles Include="$(ArtifactsDir)ship\*.nupkg" />
</ItemGroup>
<RepoTasks.VerifyCoherentVersions
PackageFiles="@(ShippingPackageFiles)" />
</Target>
<Target Name="_CreateRepositoriesListWithCommits" DependsOnTargets="_GetRepositoryCommits">
<PropertyGroup>
<RepositoryFileWithCommit>$(BuildDir)$(_RepositoryListFileName)</RepositoryFileWithCommit>

View File

@ -0,0 +1,100 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
using System.Text;
using Microsoft.Build.Framework;
using NuGet.Packaging;
using NuGet.Packaging.Core;
using RepoTasks.Utilities;
namespace RepoTasks
{
public class CopyPackagesToSplitFolders : Microsoft.Build.Utilities.Task
{
/// <summary>
/// The item group containing the nuget packages to split in different folders.
/// </summary>
[Required]
public ITaskItem[] Packages { get; set; }
[Required]
public ITaskItem[] Files { get; set; }
/// <summary>
/// The folder where packages should be copied. Subfolders will be created based on package category.
/// </summary>
[Required]
public string DestinationFolder { get; set; }
public override bool Execute()
{
if (Files?.Length == 0)
{
Log.LogError("No packages were found.");
return false;
}
var expectedPackages = PackageCollection.FromItemGroup(Packages);
Directory.CreateDirectory(DestinationFolder);
foreach (var file in Files)
{
PackageIdentity identity;
using (var reader = new PackageArchiveReader(file.ItemSpec))
{
identity = reader.GetIdentity();
}
if (!expectedPackages.TryGetCategory(identity.Id, out var category))
{
Log.LogError($"Unexpected package artifact with id: {identity.Id}");
return false;
}
string destDir;
switch (category)
{
case PackageCategory.Unknown:
throw new InvalidOperationException($"Package {identity} does not have a recognized package category.");
case PackageCategory.Shipping:
destDir = Path.Combine(DestinationFolder, "ship");
break;
case PackageCategory.NoShip:
destDir = Path.Combine(DestinationFolder, "noship");
break;
case PackageCategory.ShipOob:
destDir = Path.Combine(DestinationFolder, "shipoob");
break;
default:
throw new NotImplementedException();
}
Directory.CreateDirectory(destDir);
var destFile = Path.Combine(destDir, Path.GetFileName(file.ItemSpec));
Log.LogMessage($"Copying {file.ItemSpec} to {destFile}");
File.Copy(file.ItemSpec, destFile);
expectedPackages.Remove(identity.Id);
}
if (expectedPackages.Count != 0)
{
var error = new StringBuilder();
foreach (var key in expectedPackages.Keys)
{
error.Append(" - ").AppendLine(key);
}
Log.LogError($"Expected the following packages, but they were not found:" + error.ToString());
return false;
}
return true;
}
}
}

View File

@ -47,7 +47,7 @@ namespace RepoTasks
var root = new XElement("Project", props, items);
var doc = new XDocument(root);
if (RestoreAdditionalSources.Length > 0)
if (RestoreAdditionalSources?.Length > 0)
{
var sources = RestoreAdditionalSources.Aggregate("$(RestoreAdditionalProjectSources)", (sum, piece) => sum + ";" + piece.ItemSpec);
props.Add(new XElement("RestoreAdditionalProjectSources", sources));

View File

@ -5,6 +5,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using NuGet.Frameworks;
using NuGet.Packaging;
using NuGet.Versioning;
namespace RepoTasks.ProjectModel
@ -12,8 +13,8 @@ namespace RepoTasks.ProjectModel
internal class PackageInfo
{
public PackageInfo(string id,
string version,
IReadOnlyList<NuGetFramework> frameworks,
NuGetVersion version,
IReadOnlyList<PackageDependencyGroup> dependencyGroups,
string source,
string packageType = "Dependency")
{
@ -22,16 +23,11 @@ namespace RepoTasks.ProjectModel
throw new ArgumentException(nameof(id));
}
if (string.IsNullOrEmpty(version))
{
throw new ArgumentException(nameof(version));
}
Id = id;
Version = NuGetVersion.Parse(version);
Frameworks = frameworks;
Version = version ?? throw new ArgumentNullException(nameof(version));
PackageType = packageType;
Source = source;
DependencyGroups = dependencyGroups ?? Array.Empty<PackageDependencyGroup>();
}
public string Id { get; }
@ -41,6 +37,6 @@ namespace RepoTasks.ProjectModel
/// Can be a https feed or a file path. May be null.
/// </summary>
public string Source { get; }
public IReadOnlyList<NuGetFramework> Frameworks { get; }
public IReadOnlyList<PackageDependencyGroup> DependencyGroups { get; }
}
}

View File

@ -4,5 +4,7 @@
</PropertyGroup>
<UsingTask TaskName="RepoTasks.AnalyzeBuildGraph" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.CopyPackagesToSplitFolders" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.GenerateLineup" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.VerifyCoherentVersions" AssemblyFile="$(_RepoTaskAssembly)" />
</Project>

View File

@ -5,6 +5,9 @@ using System;
using System.IO;
using System.Linq;
using NuGet.Frameworks;
using NuGet.Packaging;
using NuGet.Packaging.Core;
using NuGet.Versioning;
using Microsoft.Build.Framework;
using RepoTasks.ProjectModel;
@ -50,10 +53,10 @@ namespace RepoTasks.Utilities
{
return new PackageInfo(
item.GetMetadata("PackageId"),
item.GetMetadata("Version"),
NuGetVersion.Parse(item.GetMetadata("Version")),
string.IsNullOrEmpty(item.GetMetadata("TargetFramework"))
? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => NuGetFramework.Parse(s)).ToArray()
: new [] { NuGetFramework.Parse(item.GetMetadata("TargetFramework")) },
? MSBuildListSplitter.SplitItemList(item.GetMetadata("TargetFramework")).Select(s => new PackageDependencyGroup(NuGetFramework.Parse(s), Array.Empty<PackageDependency>())).ToArray()
: new [] { new PackageDependencyGroup(NuGetFramework.Parse(item.GetMetadata("TargetFramework")), Array.Empty<PackageDependency>()) },
Path.GetDirectoryName(item.ItemSpec),
item.GetMetadata("PackageType"));
}

View File

@ -0,0 +1,14 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace RepoTasks.Utilities
{
public enum PackageCategory
{
Unknown = 0,
Shipping,
NoShip,
ShipOob
}
}

View File

@ -0,0 +1,66 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Build.Framework;
namespace RepoTasks.Utilities
{
public class PackageCollection
{
private readonly IDictionary<string, PackageCategory> _packages = new Dictionary<string, PackageCategory>(StringComparer.OrdinalIgnoreCase);
private PackageCollection()
{
}
public bool TryGetCategory(string packageId, out PackageCategory category) => _packages.TryGetValue(packageId, out category);
public void Remove(string packageId) => _packages.Remove(packageId);
public int Count => _packages.Count;
public IEnumerable<string> Keys => _packages.Keys;
public static PackageCollection FromItemGroup(ITaskItem[] items)
{
var list = new PackageCollection();
if (items == null)
{
return list;
}
foreach (var item in items)
{
PackageCategory category;
switch (item.GetMetadata("Category")?.ToLowerInvariant())
{
case "ship":
category = PackageCategory.Shipping;
break;
case "noship":
category = PackageCategory.NoShip;
break;
case "shipoob":
category = PackageCategory.ShipOob;
break;
default:
category = PackageCategory.Unknown;
break;
}
if (list._packages.ContainsKey(item.ItemSpec))
{
throw new InvalidDataException($"Duplicate package id detected: {item.ItemSpec}");
}
list._packages.Add(item.ItemSpec, category);
}
return list;
}
}
}

View File

@ -0,0 +1,121 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.Build.Framework;
using NuGet.Frameworks;
using NuGet.Packaging;
using NuGet.Packaging.Core;
using NuGet.Versioning;
using RepoTasks.ProjectModel;
using RepoTasks.Utilities;
namespace RepoTasks
{
public class VerifyCoherentVersions : Microsoft.Build.Utilities.Task
{
public ITaskItem[] PackageFiles { get; set; }
public override bool Execute()
{
var packageLookup = new Dictionary<string, PackageInfo>(StringComparer.OrdinalIgnoreCase);
foreach (var file in PackageFiles)
{
PackageInfo package;
using (var reader = new PackageArchiveReader(file.ItemSpec))
{
var identity = reader.GetIdentity();
var metadata = new PackageBuilder(reader.GetNuspec(), basePath: null);
package = new PackageInfo(identity.Id, identity.Version,
source: Path.GetDirectoryName(file.ItemSpec),
dependencyGroups: metadata.DependencyGroups.ToArray());
}
if (packageLookup.TryGetValue(package.Id, out var existingPackage))
{
throw new Exception("Multiple copies of the following package were found: " +
Environment.NewLine +
existingPackage +
Environment.NewLine +
package);
}
packageLookup[package.Id] = package;
}
var dependencyIssues = new List<DependencyWithIssue>();
foreach (var packageInfo in packageLookup.Values)
{
dependencyIssues.AddRange(Visit(packageLookup, packageInfo));
}
var success = true;
foreach (var mismatch in dependencyIssues)
{
var message = $"{mismatch.Info.Id} depends on {mismatch.Dependency.Id} " +
$"v{mismatch.Dependency.VersionRange} ({mismatch.TargetFramework}) when the latest build is v{mismatch.Info.Version}.";
Log.LogError(message);
success = false;
}
Log.LogMessage(MessageImportance.High, $"Verified {PackageFiles.Length} package(s) have coherent versions");
return success;
}
private class DependencyWithIssue
{
public PackageDependency Dependency { get; set; }
public PackageInfo Info { get; set; }
public NuGetFramework TargetFramework { get; set; }
}
private IEnumerable<DependencyWithIssue> Visit(IDictionary<string, PackageInfo> packageLookup, PackageInfo packageInfo)
{
Log.LogMessage(MessageImportance.Low, $"Processing package {packageInfo.Id}");
try
{
var issues = new List<DependencyWithIssue>();
foreach (var dependencySet in packageInfo.DependencyGroups)
{
// If the package doens't target any frameworks, just accept it
if (dependencySet.TargetFramework == null)
{
continue;
}
foreach (var dependency in dependencySet.Packages)
{
if (!packageLookup.TryGetValue(dependency.Id, out var dependencyPackageInfo))
{
// External dependency
continue;
}
if (dependencyPackageInfo.Version != dependency.VersionRange.MinVersion)
{
// For any dependency in the universe
// Add a mismatch if the min version doesn't work out
// (we only really care about >= minVersion)
issues.Add(new DependencyWithIssue
{
Dependency = dependency,
TargetFramework = dependencySet.TargetFramework,
Info = dependencyPackageInfo
});
}
}
}
return issues;
}
catch
{
Log.LogError($"Unable to verify package {packageInfo.Id}");
throw;
}
}
}
}