huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update PackageReferences to non-floating versions when building multiple repos

This commit is contained in:
Nate McMaster 2017-09-18 12:05:12 -07:00
parent 51dbdbeac2
commit 4924f79efc
8 changed files with 38 additions and 431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -8,6 +8,8 @@ _ReSharper.*
*.pidb *.pidb
*.vspx *.vspx
*.psess *.psess
*.binlog
*.log
packages packages
target target
artifacts artifacts

30
build/RepositoryBuild.targets
View File

@ -17,15 +17,29 @@
<BatchBuilds Condition="'$(BatchBuilds)'==''">false</BatchBuilds> <BatchBuilds Condition="'$(BatchBuilds)'==''">false</BatchBuilds>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<PinPackageSources Include="$(BuildDir)" />
<PinPackageSources Include="$(_DependencyPackagesDirectory)" Condition="Exists('$(_DependencyPackagesDirectory)')" />
</ItemGroup>
<RepoTasks.GenerateLineup
Artifacts="@(ArtifactInfo)"
Repository="%(RepositoryBuildOrder.Identity)"
OutputPath="%(RepositoryBuildOrder.RepositoryPath)\build\dependencies.g.targets"
RestoreAdditionalSources="@(PinPackageSources)"
UseFloatingVersions="false"
Condition="'%(RepositoryBuildOrder.RepositoryPath)' != ''"
BuildNumber="$(BuildNumber)" />
<MSBuild <MSBuild
Projects="@(BatchedRepository)" Projects="@(BatchedRepository)"
BuildInParallel="$(BatchBuilds)" BuildInParallel="$(BatchBuilds)"
StopOnFirstFailure="true" StopOnFirstFailure="true"
Targets="_BuildRepository" Targets="_BuildRepository"
Properties="BuildGroup=%(BatchedRepository.BuildGroup)" /> Properties="BuildGroup=%(BatchedRepository.BuildGroup);BuildNumber=$(BuildNumber);IsFinalBuild=$(IsFinalBuild);Configuration=$(Configuration)" />
</Target> </Target>
<Target Name="_BuildRepository" DependsOnTargets="_PinVersions"> <Target Name="_BuildRepository">
<PropertyGroup> <PropertyGroup>
<!-- If there are duplicate properties, the properties which are defined later in the order would override the earlier ones --> <!-- If there are duplicate properties, the properties which are defined later in the order would override the earlier ones -->
<RepositoryBuildArguments>$(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash)</RepositoryBuildArguments> <RepositoryBuildArguments>$(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash)</RepositoryBuildArguments>
@ -68,16 +82,4 @@
<Message Text="============ Done building $(RepositoryToBuild) ============" Importance="High" /> <Message Text="============ Done building $(RepositoryToBuild) ============" Importance="High" />
</Target> </Target>
<Target Name="_PinVersions">
<ItemGroup>
<PinPackageSources Include="$(BuildDir)" />
<PinPackageSources Include="$(_DependencyPackagesDirectory)" Condition="Exists('$(_DependencyPackagesDirectory)')" />
</ItemGroup>
<RepoTasks.PinVersions
GraphSpecsRoot="$(_RestoreGraphSpecsDirectory)"
BuildRepositoryRoot="$(BuildRepositoryRoot)"
PackageSources="@(PinPackageSources)" />
</Target>
</Project> </Project>

17
build/repo.targets
View File

@ -106,7 +106,7 @@
</Target> </Target>
<Target Name="BuildRepositories" <Target Name="BuildRepositories"
DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;ComputeGraph;_BuildRepositories" /> DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;ComputeGraph;_BuildRepositories" />
<Target Name="ResolveRepoInfo" DependsOnTargets="_PrepareRepositories"> <Target Name="ResolveRepoInfo" DependsOnTargets="_PrepareRepositories">
<MSBuild Projects="$(MSBuildProjectFullPath)" <MSBuild Projects="$(MSBuildProjectFullPath)"
@ -180,21 +180,6 @@
BuildNumber="$(BuildNumber)" /> BuildNumber="$(BuildNumber)" />
</Target> </Target>
<Target Name="_UpdateNuGetConfig">
<UpdatePackageSource
NuGetConfigPath="$(_CloneRepositoryRoot)%(Repository.Identity)\NuGet.config"
SourceName="Dependencies"
SourceUri="$(_DependencyPackagesDirectory)"
Condition="Exists('$(_DependencyPackagesDirectory)')" />
<MakeDir Directories="$(BuildDir)" Condition="!Exists('$(BuildDir)')" />
<UpdatePackageSource
NuGetConfigPath="$(_CloneRepositoryRoot)%(Repository.Identity)\NuGet.config"
SourceName="Artifacts"
SourceUri="$(BuildDir)" />
</Target>
<Target Name="_CreateRepositoriesListWithCommits" DependsOnTargets="_GetRepositoryCommits"> <Target Name="_CreateRepositoriesListWithCommits" DependsOnTargets="_GetRepositoryCommits">
<PropertyGroup> <PropertyGroup>
<RepositoryFileWithCommit>$(BuildDir)$(_RepositoryListFileName)</RepositoryFileWithCommit> <RepositoryFileWithCommit>$(BuildDir)$(_RepositoryListFileName)</RepositoryFileWithCommit>

4
build/tasks/AnalyzeBuildGraph.cs
View File

@ -136,8 +136,8 @@ namespace RepoTasks
inconsistentVersions.Add(new VersionMismatch inconsistentVersions.Add(new VersionMismatch
{ {
Solution = sln, Solution = solution,
Project = proj, Project = project,
PackageId = dependency.Key, PackageId = dependency.Key,
ActualVersion = dependency.Value.Version, ActualVersion = dependency.Value.Version,
ExpectedVersion = package.PackageInfo.Version, ExpectedVersion = package.PackageInfo.Version,

31
build/tasks/GenerateLineup.cs
View File

@ -25,6 +25,9 @@ namespace RepoTasks
// Can be set to filter the lists of packages when produce a list for a specific repository // Can be set to filter the lists of packages when produce a list for a specific repository
public string Repository { get; set; } public string Repository { get; set; }
// Items to add to the RestoreAdditionalProjectSources list in project
public ITaskItem[] RestoreAdditionalSources { get; set; }
public bool UseFloatingVersions { get; set; } public bool UseFloatingVersions { get; set; }
public string BuildNumber { get; set; } public string BuildNumber { get; set; }
@ -40,25 +43,25 @@ namespace RepoTasks
} }
var items = new XElement("ItemGroup"); var items = new XElement("ItemGroup");
var root = new XElement("Project", items); var props = new XElement("PropertyGroup");
var root = new XElement("Project", props, items);
var doc = new XDocument(root); var doc = new XDocument(root);
if (RestoreAdditionalSources.Length > 0)
{
var sources = RestoreAdditionalSources.Aggregate("$(RestoreAdditionalProjectSources)", (sum, piece) => sum + ";" + piece.ItemSpec);
props.Add(new XElement("RestoreAdditionalProjectSources", sources));
}
var packages = new List<PackageInfo>(); var packages = new List<PackageInfo>();
foreach (var item in Artifacts) foreach (var pkg in Artifacts.Select(ArtifactInfo.Parse)
.OfType<ArtifactInfo.Package>()
.Where(p => !p.IsSymbolsArtifact
&& (string.IsNullOrEmpty(Repository)
|| !Repository.Equals(p.RepoName, StringComparison.OrdinalIgnoreCase))))
{ {
var info = ArtifactInfo.Parse(item); packages.Add(pkg.PackageInfo);
switch (info)
{
case ArtifactInfo.Package pkg when (!pkg.IsSymbolsArtifact):
// TODO filter this list based on topological sort info
if (string.IsNullOrEmpty(Repository)
|| !Repository.Equals(pkg.RepoName, StringComparison.OrdinalIgnoreCase))
{
packages.Add(pkg.PackageInfo);
}
break;
}
} }
foreach (var pkg in packages.OrderBy(i => i.Id)) foreach (var pkg in packages.OrderBy(i => i.Id))

46
build/tasks/PinVersions.cs
View File

@ -1,46 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Linq;
using Microsoft.Build.Framework;
using RepoTasks.VersionPinning;
namespace RepoTasks
{
public class PinVersions : Microsoft.Build.Utilities.Task
{
[Required]
public string BuildRepositoryRoot { get; set; }
[Required]
public ITaskItem[] PackageSources { get; set; }
public string GraphSpecsRoot { get; set; }
public override bool Execute()
{
if (PackageSources?.Length == 0)
{
Log.LogError($"Missing PackageSources. At least one item source must be specified.");
return false;
}
var graphSpecProvider = !string.IsNullOrEmpty(GraphSpecsRoot)
? new DependencyGraphSpecProvider(GraphSpecsRoot)
: DependencyGraphSpecProvider.Default;
using (graphSpecProvider)
{
var pinVersionUtility = new PinVersionUtility(
BuildRepositoryRoot,
PackageSources.Select(i => i.ItemSpec).ToList(),
graphSpecProvider,
Log);
pinVersionUtility.Execute();
}
return true;
}
}
}

98
build/tasks/VersionPinning/DependencyGraphSpecProvider.cs
View File

@ -1,98 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using NuGet.ProjectModel;
namespace RepoTasks.VersionPinning
{
public class DependencyGraphSpecProvider : IDisposable
{
private readonly string _packageSpecDirectory;
private readonly bool _deleteSpecDirectoryOnDispose;
private readonly string _dotnetPath;
public DependencyGraphSpecProvider(string packageSpecDirectory)
: this(packageSpecDirectory, deleteSpecDirectoryOnDispose: false)
{
}
private DependencyGraphSpecProvider(string packageSpecDirectory, bool deleteSpecDirectoryOnDispose)
{
_packageSpecDirectory = packageSpecDirectory;
_deleteSpecDirectoryOnDispose = deleteSpecDirectoryOnDispose;
_dotnetPath = Process.GetCurrentProcess().MainModule.FileName;
}
public static DependencyGraphSpecProvider Default { get; } =
new DependencyGraphSpecProvider(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), deleteSpecDirectoryOnDispose: true);
public DependencyGraphSpec GetDependencyGraphSpec(string repositoryName, string solutionPath)
{
var outputFile = Path.Combine(_packageSpecDirectory, repositoryName, Path.GetFileName(solutionPath) + ".json");
if (!File.Exists(outputFile))
{
RunMSBuild(solutionPath, outputFile);
}
return DependencyGraphSpec.Load(outputFile);
}
private void RunMSBuild(string solutionPath, string outputFile)
{
var psi = new ProcessStartInfo(_dotnetPath);
var arguments = new List<string>
{
"msbuild",
$"\"{solutionPath}\"",
"/t:GenerateRestoreGraphFile",
"/nologo",
"/v:q",
"/p:BuildProjectReferences=false",
$"/p:RestoreGraphOutputPath=\"{outputFile}\"",
"/p:KoreBuildRestoreTargetsImported=true",
};
psi.Arguments = string.Join(" ", arguments);
psi.RedirectStandardOutput = true;
var process = new Process
{
StartInfo = psi,
EnableRaisingEvents = true,
};
process.OutputDataReceived += (sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Console.WriteLine(args.Data);
}
};
using (process)
{
process.Start();
process.BeginOutputReadLine();
process.WaitForExit(60 * 5000);
if (process.ExitCode != 0)
{
throw new Exception($"{psi.FileName} {psi.Arguments} failed. Exit code {process.ExitCode}.");
}
}
}
public void Dispose()
{
if (_deleteSpecDirectoryOnDispose)
{
Directory.Delete(_packageSpecDirectory, recursive: true);
}
}
}
}

241
build/tasks/VersionPinning/PinVersionUtility.cs
View File

@ -1,241 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using NuGet.Common;
using NuGet.Frameworks;
using NuGet.LibraryModel;
using NuGet.ProjectModel;
using NuGet.Protocol;
using NuGet.Protocol.Core.Types;
using NuGet.Versioning;
namespace RepoTasks.VersionPinning
{
internal class PinVersionUtility
{
private readonly string _repositoryRoot;
private readonly FindPackageByIdResource[] _findPackageResources;
private readonly ConcurrentDictionary<string, Task<NuGetVersion>> _exactMatches = new ConcurrentDictionary<string, Task<NuGetVersion>>(StringComparer.OrdinalIgnoreCase);
private readonly DependencyGraphSpecProvider _provider;
private readonly SourceCacheContext _sourceCacheContext;
private readonly TaskLoggingHelper _logger;
public PinVersionUtility(
string repositoryRoot,
List<string> pinSources,
DependencyGraphSpecProvider provider,
TaskLoggingHelper logger)
{
_repositoryRoot = repositoryRoot;
_findPackageResources = new FindPackageByIdResource[pinSources.Count];
for (var i = 0; i < pinSources.Count; i++)
{
var repository = FactoryExtensionsV3.GetCoreV3(Repository.Factory, pinSources[i].Trim());
_findPackageResources[i] = repository.GetResource<FindPackageByIdResource>();
}
_provider = provider;
_sourceCacheContext = new SourceCacheContext();
_logger = logger;
}
public void Execute()
{
_logger.LogMessage(MessageImportance.High, $"Pinning package references for projects in {_repositoryRoot}");
var solutionPinMetadata = GetProjectPinVersionMetadata();
foreach (var cliToolReference in solutionPinMetadata.CLIToolReferences)
{
_logger.LogMessage(MessageImportance.Normal, $"Pinning CLI Tool {cliToolReference.Item1.Name}({cliToolReference.Item1.VersionRange} to {cliToolReference.Item2} for all projects in {_repositoryRoot}.");
}
foreach (var item in solutionPinMetadata.PinVersionLookup)
{
var projectPinMetadata = item.Value;
var specProject = projectPinMetadata.PackageSpec;
if (!(projectPinMetadata.Packages.Any() || solutionPinMetadata.CLIToolReferences.Any()))
{
_logger.LogMessage(MessageImportance.Normal, $"No package or tool references to pin for {specProject.FilePath}.");
continue;
}
var projectFileInfo = new FileInfo(specProject.FilePath);
var pinnedReferencesFile = Path.Combine(
specProject.RestoreMetadata.OutputPath,
projectFileInfo.Name + ".pinnedversions.targets");
Directory.CreateDirectory(Path.GetDirectoryName(pinnedReferencesFile));
if (projectPinMetadata.Packages.Any())
{
_logger.LogMessage(MessageImportance.Normal, $"Pinning package versions for {specProject.FilePath}.");
}
var pinnedReferences = new XElement("ItemGroup", new XAttribute("Condition", "'$(PolicyDesignTimeBuild)' != 'true' AND !Exists('$(MSBuildThisFileDirectory)$(MSBuildProjectFile).nugetpolicy.g.targets')"));
foreach (var packageReference in projectPinMetadata.Packages)
{
(var tfm, var libraryRange, var exactVersion) = packageReference;
_logger.LogMessage(MessageImportance.Normal, $"Pinning reference {libraryRange.Name}({libraryRange.VersionRange} to {exactVersion}.");
var metadata = new List<XAttribute>
{
new XAttribute("Update", libraryRange.Name),
new XAttribute("Version", exactVersion.ToNormalizedString()),
};
if (tfm != NuGetFramework.AnyFramework)
{
metadata.Add(new XAttribute("Condition", $"'$(TargetFramework)'=='{tfm.GetShortFolderName()}'"));
}
pinnedReferences.Add(new XElement("PackageReference", metadata));
}
// CLI Tool references are specified at solution level.
foreach (var toolReference in solutionPinMetadata.CLIToolReferences)
{
(var libraryRange, var exactVersion) = toolReference;
var metadata = new List<XAttribute>
{
new XAttribute("Update", libraryRange.Name),
new XAttribute("Version", exactVersion.ToNormalizedString()),
};
pinnedReferences.Add(new XElement("DotNetCliToolReference", metadata));
}
var pinnedVersionRoot = new XElement("Project", pinnedReferences);
File.WriteAllText(pinnedReferencesFile, pinnedVersionRoot.ToString());
}
}
private SolutionPinVersionMetadata GetProjectPinVersionMetadata()
{
var repositoryDirectoryInfo = new DirectoryInfo(_repositoryRoot);
var projects = new Dictionary<string, ProjectPinVersionMetadata>(StringComparer.OrdinalIgnoreCase);
var cliToolReferences = new List<(LibraryRange, NuGetVersion)>();
foreach (var slnFile in repositoryDirectoryInfo.EnumerateFiles("*.sln"))
{
var graphSpec = _provider.GetDependencyGraphSpec(repositoryDirectoryInfo.Name, slnFile.FullName);
foreach (var specProject in graphSpec.Projects)
{
if (!projects.TryGetValue(specProject.FilePath, out var pinMetadata))
{
pinMetadata = new ProjectPinVersionMetadata(specProject);
projects[specProject.FilePath] = pinMetadata;
}
var allDependencies = specProject.Dependencies.Select(dependency => new { Dependency = dependency, FrameworkName = NuGetFramework.AnyFramework })
.Concat(specProject.TargetFrameworks.SelectMany(tfm => tfm.Dependencies.Select(dependency => new { Dependency = dependency, tfm.FrameworkName })))
.Where(d => d.Dependency.LibraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package));
foreach (var dependency in allDependencies)
{
var reference = dependency.Dependency;
var versionRange = reference.LibraryRange.VersionRange;
if (!versionRange.IsFloating)
{
continue;
}
var exactVersion = GetExactVersion(reference.Name, versionRange);
if (exactVersion == null)
{
continue;
}
var projectStyle = specProject.RestoreMetadata.ProjectStyle;
if (projectStyle == ProjectStyle.PackageReference)
{
pinMetadata.Packages.Add((dependency.FrameworkName, reference.LibraryRange, exactVersion));
}
else if (projectStyle == ProjectStyle.DotnetCliTool)
{
cliToolReferences.Add((reference.LibraryRange, exactVersion));
}
else
{
throw new NotSupportedException($"Unknown project style '{projectStyle}'.");
}
}
}
}
return new SolutionPinVersionMetadata(projects, cliToolReferences);
}
private NuGetVersion GetExactVersion(string name, VersionRange range)
{
if (range.MinVersion == null)
{
throw new Exception($"Unsupported version range {range}.");
}
if (!_exactMatches.TryGetValue(name, out var versionTask))
{
versionTask = _exactMatches.GetOrAdd(name, GetExactVersionAsync(name, range.MinVersion));
}
return versionTask.Result;
}
private async Task<NuGetVersion> GetExactVersionAsync(string name, NuGetVersion floatingVersion)
{
foreach (var findPackageResource in _findPackageResources)
{
var packageVersions = await findPackageResource.GetAllVersionsAsync(name, _sourceCacheContext, NullLogger.Instance, default(CancellationToken));
var matchingVersions = packageVersions.Where(v => v.Version == floatingVersion.Version).ToList();
switch (matchingVersions.Count)
{
case 0:
continue;
case 1:
return matchingVersions[0];
default:
throw new Exception($"More than one version for {name} found that matches the specified version constraint: {string.Join(" ", matchingVersions)}.");
}
}
return null;
}
private struct SolutionPinVersionMetadata
{
public SolutionPinVersionMetadata(
IDictionary<string, ProjectPinVersionMetadata> pinVersionLookup,
List<(LibraryRange, NuGetVersion)> cliToolReferences)
{
PinVersionLookup = pinVersionLookup;
CLIToolReferences = cliToolReferences;
}
public IDictionary<string, ProjectPinVersionMetadata> PinVersionLookup { get; }
public List<(LibraryRange, NuGetVersion)> CLIToolReferences { get; }
}
private struct ProjectPinVersionMetadata
{
public ProjectPinVersionMetadata(PackageSpec packageSpec)
{
PackageSpec = packageSpec;
Packages = new List<(NuGetFramework, LibraryRange, NuGetVersion)>();
}
public PackageSpec PackageSpec { get; }
public List<(NuGetFramework, LibraryRange, NuGetVersion)> Packages { get; }
}
}
}