diff --git a/eng/tools/BaselineGenerator/BaselineGenerator.csproj b/eng/tools/BaselineGenerator/BaselineGenerator.csproj index 8e7d5913a6..53f9d15055 100644 --- a/eng/tools/BaselineGenerator/BaselineGenerator.csproj +++ b/eng/tools/BaselineGenerator/BaselineGenerator.csproj @@ -3,12 +3,11 @@ Exe netcoreapp2.1 - -o "$(MSBuildThisFileDirectory)../../Baseline.Designer.props" $(MSBuildThisFileDirectory)../../ - + diff --git a/eng/tools/BaselineGenerator/Program.cs b/eng/tools/BaselineGenerator/Program.cs index 7b28b83ae0..b8089832f0 100644 --- a/eng/tools/BaselineGenerator/Program.cs +++ b/eng/tools/BaselineGenerator/Program.cs @@ -3,14 +3,20 @@ using System; using System.IO; +using System.Linq; using System.Net.Http; using System.Text; +using System.Threading; using System.Threading.Tasks; using System.Xml; using System.Xml.Linq; using Microsoft.Extensions.CommandLineUtils; +using NuGet.Common; +using NuGet.Configuration; using NuGet.Packaging; -using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; namespace PackageBaselineGenerator { @@ -26,11 +32,13 @@ namespace PackageBaselineGenerator private readonly CommandOption _source; private readonly CommandOption _output; + private readonly CommandOption _update; public Program() { - _source = Option("-s|--source ", "The NuGet v2 source of the package to fetch", CommandOptionType.SingleValue); + _source = Option("-s|--package-source ", "The NuGet source of packages to fetch", CommandOptionType.SingleValue); _output = Option("-o|--output ", "The generated file output path", CommandOptionType.SingleValue); + _update = Option("-u|--update", "Regenerate the input (Baseline.xml) file.", CommandOptionType.NoValue); Invoke = () => Run().GetAwaiter().GetResult(); } @@ -38,22 +46,43 @@ namespace PackageBaselineGenerator private async Task Run() { var source = _source.HasValue() - ? _source.Value() - : "https://www.nuget.org/api/v2/package"; + ? _source.Value().TrimEnd('/') + : "https://api.nuget.org/v3/index.json"; + if (_output.HasValue() && _update.HasValue()) + { + await Error.WriteLineAsync("'--output' and '--update' options must not be used together."); + return 1; + } - var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") != null - ? Environment.GetEnvironmentVariable("NUGET_PACKAGES") - : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); + var inputPath = Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml"); + var input = XDocument.Load(inputPath); + var packageSource = new PackageSource(source); + var providers = Repository.Provider.GetCoreV3(); // Get v2 and v3 API support + var sourceRepository = new SourceRepository(packageSource, providers); + if (_update.HasValue()) + { + return await RunUpdateAsync(inputPath, input, sourceRepository); + } - var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp"); - Directory.CreateDirectory(tempDir); - - var input = XDocument.Load(Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml")); + var feedType = await sourceRepository.GetFeedType(CancellationToken.None); + var feedV3 = feedType == FeedType.HttpV3; + var packageBase = source + "/package"; + if (feedV3) + { + var resources = await sourceRepository.GetResourceAsync(); + packageBase = resources.GetServiceEntryUri(ServiceTypes.PackageBaseAddress).ToString().TrimEnd('/'); + } var output = _output.HasValue() ? _output.Value() : Path.Combine(Directory.GetCurrentDirectory(), "Baseline.Designer.props"); + var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") ?? + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); + + var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp"); + Directory.CreateDirectory(tempDir); + var baselineVersion = input.Root.Attribute("Version").Value; var doc = new XDocument( @@ -64,7 +93,6 @@ namespace PackageBaselineGenerator new XElement("AspNetCoreBaselineVersion", baselineVersion)))); var client = new HttpClient(); - foreach (var pkg in input.Root.Descendants("Package")) { var id = pkg.Attribute("Id").Value; @@ -78,32 +106,32 @@ namespace PackageBaselineGenerator if (!File.Exists(nupkgPath)) { - var url = $"{source}/{id}/{version}"; - using (var file = File.Create(nupkgPath)) + var url = feedV3 ? + $"{packageBase}/{id.ToLowerInvariant()}/{version}/{id.ToLowerInvariant()}.{version}.nupkg" : + $"{packageBase}/{id}/{version}"; + + Console.WriteLine($"Downloading {url}"); + using (var response = await client.GetStreamAsync(url)) { - Console.WriteLine($"Downloading {url}"); - var response = await client.GetStreamAsync(url); - await response.CopyToAsync(file); + using (var file = File.Create(nupkgPath)) + { + await response.CopyToAsync(file); + } } } - using (var reader = new PackageArchiveReader(nupkgPath)) { - var first = true; + doc.Root.Add(new XComment($" Package: {id}")); + + var propertyGroup = new XElement( + "PropertyGroup", + new XAttribute("Condition", $" '$(PackageId)' == '{id}' "), + new XElement("BaselinePackageVersion", version)); + doc.Root.Add(propertyGroup); + foreach (var group in reader.NuspecReader.GetDependencyGroups()) { - if (first) - { - first = false; - doc.Root.Add(new XComment($" Package: {id}")); - - var propertyGroup = new XElement("PropertyGroup", - new XAttribute("Condition", $" '$(PackageId)' == '{id}' "), - new XElement("BaselinePackageVersion", version)); - doc.Root.Add(propertyGroup); - } - var itemGroup = new XElement("ItemGroup", new XAttribute("Condition", $" '$(PackageId)' == '{id}' AND '$(TargetFramework)' == '{group.TargetFramework.GetShortFolderName()}' ")); doc.Root.Add(itemGroup); @@ -121,12 +149,181 @@ namespace PackageBaselineGenerator Encoding = Encoding.UTF8, Indent = true, }; + using (var writer = XmlWriter.Create(output, settings)) { doc.Save(writer); } + Console.WriteLine($"Generated file in {output}"); + return 0; } + + private async Task RunUpdateAsync( + string documentPath, + XDocument document, + SourceRepository sourceRepository) + { + var packageMetadataResource = await sourceRepository.GetResourceAsync(); + var logger = new Logger(Error, Out); + var hasChanged = false; + using (var cacheContext = new SourceCacheContext { NoCache = true }) + { + var versionAttribute = document.Root.Attribute("Version"); + hasChanged = await TryUpdateVersionAsync( + versionAttribute, + "Microsoft.AspNetCore.App", + packageMetadataResource, + logger, + cacheContext); + + foreach (var package in document.Root.Descendants("Package")) + { + var id = package.Attribute("Id").Value; + versionAttribute = package.Attribute("Version"); + var attributeChanged = await TryUpdateVersionAsync( + versionAttribute, + id, + packageMetadataResource, + logger, + cacheContext); + + hasChanged |= attributeChanged; + } + } + + if (hasChanged) + { + await Out.WriteLineAsync($"Updating {documentPath}."); + + var settings = new XmlWriterSettings + { + Async = true, + CheckCharacters = true, + CloseOutput = false, + Encoding = Encoding.UTF8, + Indent = true, + IndentChars = " ", + NewLineOnAttributes = false, + OmitXmlDeclaration = true, + WriteEndDocumentOnClose = true, + }; + + using (var stream = File.OpenWrite(documentPath)) + { + using (var writer = XmlWriter.Create(stream, settings)) + { + await document.SaveAsync(writer, CancellationToken.None); + } + } + } + else + { + await Out.WriteLineAsync("No new versions found"); + } + + return 0; + } + + private static async Task TryUpdateVersionAsync( + XAttribute versionAttribute, + string packageId, + PackageMetadataResource packageMetadataResource, + ILogger logger, + SourceCacheContext cacheContext) + { + var searchMetadata = await packageMetadataResource.GetMetadataAsync( + packageId, + includePrerelease: false, + includeUnlisted: true, // Microsoft.AspNetCore.DataOrotection.Redis package is not listed. + sourceCacheContext: cacheContext, + log: logger, + token: CancellationToken.None); + + var currentVersion = NuGetVersion.Parse(versionAttribute.Value); + var versionRange = new VersionRange( + currentVersion, + new FloatRange(NuGetVersionFloatBehavior.Patch, currentVersion)); + + var latestVersion = versionRange.FindBestMatch( + searchMetadata.Select(metadata => metadata.Identity.Version)); + + if (latestVersion == null) + { + logger.LogWarning($"Unable to find latest version of '{packageId}'."); + return false; + } + + var hasChanged = false; + if (latestVersion != currentVersion) + { + hasChanged = true; + versionAttribute.Value = latestVersion.ToNormalizedString(); + } + + return hasChanged; + } + + private class Logger : ILogger + { + private readonly TextWriter _error; + private readonly TextWriter _out; + + public Logger(TextWriter error, TextWriter @out) + { + _error = error; + _out = @out; + } + + public void Log(LogLevel level, string data) + { + switch (level) + { + case LogLevel.Debug: + LogDebug(data); + break; + case LogLevel.Error: + LogError(data); + break; + case LogLevel.Information: + LogInformation(data); + break; + case LogLevel.Minimal: + LogMinimal(data); + break; + case LogLevel.Verbose: + LogVerbose(data); + break; + case LogLevel.Warning: + LogWarning(data); + break; + } + } + + public void Log(ILogMessage message) => Log(message.Level, message.Message); + + public Task LogAsync(LogLevel level, string data) + { + Log(level, data); + return Task.CompletedTask; + } + + public Task LogAsync(ILogMessage message) => LogAsync(message.Level, message.Message); + + public void LogDebug(string data) => _out.WriteLine($"Debug: {data}"); + + public void LogError(string data) => _error.WriteLine($"Error: {data}"); + + public void LogInformation(string data) => _out.WriteLine($"Information: {data}"); + + public void LogInformationSummary(string data) => _out.WriteLine($"Summary: {data}"); + + public void LogMinimal(string data) => _out.WriteLine($"Minimal: {data}"); + + public void LogVerbose(string data) => _out.WriteLine($"Verbose: {data}"); + + public void LogWarning(string data) => _out.WriteLine($"Warning: {data}"); + } } } diff --git a/eng/tools/BaselineGenerator/README.md b/eng/tools/BaselineGenerator/README.md index 1fd73b17f7..fa3e4e0bda 100644 --- a/eng/tools/BaselineGenerator/README.md +++ b/eng/tools/BaselineGenerator/README.md @@ -1,10 +1,20 @@ -BaselineGenerator -================= +# BaselineGenerator This tool is used to generate an MSBuild file which sets the "baseline" against which servicing updates are built. ## Usage -1. Add to the [Baseline.xml](/eng/Baseline.xml) a list of package ID's and their latest released versions. The source of this information can typically - be found in the build.xml file generated during ProdCon builds. See https://github.com/dotnet/versions/blob/master/build-info/dotnet/product/cli/release/2.1.6/build.xml for example. +Add `--package-source {source}` to the commands below if the packages of interest are not all hosted on NuGet.org. + +### Auto-update + +1. Run `dotnet run --update` in this project folder. +2. Run `dotnet run` in this project. + +### Manual update + +1. Add to the [Baseline.xml](/eng/Baseline.xml) a list of package ID's and their latest released versions. The source of +this information can typically be found in the build.xml file generated during ProdCon builds. See + for example. +Update the version at the top of baseline.xml to match prior release (even if no packages changed in the prior release). 2. Run `dotnet run` on this project.