Merge in 'release/2.1' changes

eng/tools/BaselineGenerator/BaselineGenerator.csproj

@@ -3,12 +3,11 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp2.1</TargetFramework>
-    <StartArguments>-o "$(MSBuildThisFileDirectory)../../Baseline.Designer.props"</StartArguments>
     <StartWorkingDirectory>$(MSBuildThisFileDirectory)../../</StartWorkingDirectory>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NuGet.Packaging" Version="4.8.0" />
+    <PackageReference Include="NuGet.Protocol" Version="4.8.2" />
     <PackageReference Include="Microsoft.Extensions.CommandLineUtils" Version="1.1.0" />
   </ItemGroup>
 

eng/tools/BaselineGenerator/Program.cs

@@ -3,14 +3,20 @@
 
 using System;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using System.Xml;
 using System.Xml.Linq;
 using Microsoft.Extensions.CommandLineUtils;
+using NuGet.Common;
+using NuGet.Configuration;
 using NuGet.Packaging;
-using NuGet.Packaging.Core;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
 
 namespace PackageBaselineGenerator
 {
@@ -26,11 +32,13 @@ namespace PackageBaselineGenerator
 
         private readonly CommandOption _source;
         private readonly CommandOption _output;
+        private readonly CommandOption _update;
 
         public Program()
         {
-            _source = Option("-s|--source <SOURCE>", "The NuGet v2 source of the package to fetch", CommandOptionType.SingleValue);
+            _source = Option("-s|--package-source <SOURCE>", "The NuGet source of packages to fetch", CommandOptionType.SingleValue);
             _output = Option("-o|--output <OUT>", "The generated file output path", CommandOptionType.SingleValue);
+            _update = Option("-u|--update", "Regenerate the input (Baseline.xml) file.", CommandOptionType.NoValue);
 
             Invoke = () => Run().GetAwaiter().GetResult();
         }
@@ -38,22 +46,43 @@ namespace PackageBaselineGenerator
         private async Task<int> Run()
         {
             var source = _source.HasValue()
-                ? _source.Value()
+                ? _source.Value().TrimEnd('/')
-                : "https://www.nuget.org/api/v2/package";
+                : "https://api.nuget.org/v3/index.json";
+            if (_output.HasValue() && _update.HasValue())
+            {
+                await Error.WriteLineAsync("'--output' and '--update' options must not be used together.");
+                return 1;
+            }
 
-            var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") != null
+            var inputPath = Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml");
-                ? Environment.GetEnvironmentVariable("NUGET_PACKAGES")
+            var input = XDocument.Load(inputPath);
-                : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages");
+            var packageSource = new PackageSource(source);
+            var providers = Repository.Provider.GetCoreV3(); // Get v2 and v3 API support
+            var sourceRepository = new SourceRepository(packageSource, providers);
+            if (_update.HasValue())
+            {
+                return await RunUpdateAsync(inputPath, input, sourceRepository);
+            }
 
-            var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp");
+            var feedType = await sourceRepository.GetFeedType(CancellationToken.None);
-            Directory.CreateDirectory(tempDir);
+            var feedV3 = feedType == FeedType.HttpV3;
-
+            var packageBase = source + "/package";
-            var input = XDocument.Load(Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml"));
+            if (feedV3)
+            {
+                var resources = await sourceRepository.GetResourceAsync<ServiceIndexResourceV3>();
+                packageBase = resources.GetServiceEntryUri(ServiceTypes.PackageBaseAddress).ToString().TrimEnd('/');
+            }
 
             var output = _output.HasValue()
                 ? _output.Value()
                 : Path.Combine(Directory.GetCurrentDirectory(), "Baseline.Designer.props");
 
+            var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") ??
+                Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages");
+
+            var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp");
+            Directory.CreateDirectory(tempDir);
+
             var baselineVersion = input.Root.Attribute("Version").Value;
 
             var doc = new XDocument(
@@ -64,7 +93,6 @@ namespace PackageBaselineGenerator
                         new XElement("AspNetCoreBaselineVersion", baselineVersion))));
 
             var client = new HttpClient();
-
             foreach (var pkg in input.Root.Descendants("Package"))
             {
                 var id = pkg.Attribute("Id").Value;
@@ -78,32 +106,32 @@ namespace PackageBaselineGenerator
 
                 if (!File.Exists(nupkgPath))
                 {
-                    var url = $"{source}/{id}/{version}";
+                    var url = feedV3 ?
-                    using (var file = File.Create(nupkgPath))
+                        $"{packageBase}/{id.ToLowerInvariant()}/{version}/{id.ToLowerInvariant()}.{version}.nupkg" :
+                        $"{packageBase}/{id}/{version}";
+
+                    Console.WriteLine($"Downloading {url}");
+                    using (var response = await client.GetStreamAsync(url))
                     {
-                        Console.WriteLine($"Downloading {url}");
+                        using (var file = File.Create(nupkgPath))
-                        var response = await client.GetStreamAsync(url);
+                        {
-                        await response.CopyToAsync(file);
+                            await response.CopyToAsync(file);
+                        }
                     }
                 }
 
-
                 using (var reader = new PackageArchiveReader(nupkgPath))
                 {
-                    var first = true;
+                    doc.Root.Add(new XComment($" Package: {id}"));
+
+                    var propertyGroup = new XElement(
+                        "PropertyGroup",
+                        new XAttribute("Condition", $" '$(PackageId)' == '{id}' "),
+                        new XElement("BaselinePackageVersion", version));
+                    doc.Root.Add(propertyGroup);
+
                     foreach (var group in reader.NuspecReader.GetDependencyGroups())
                     {
-                        if (first)
-                        {
-                            first = false;
-                            doc.Root.Add(new XComment($" Package: {id}"));
-
-                            var propertyGroup = new XElement("PropertyGroup",
-                                new XAttribute("Condition", $" '$(PackageId)' == '{id}' "),
-                                new XElement("BaselinePackageVersion", version));
-                            doc.Root.Add(propertyGroup);
-                        }
-
                         var itemGroup = new XElement("ItemGroup", new XAttribute("Condition", $" '$(PackageId)' == '{id}' AND '$(TargetFramework)' == '{group.TargetFramework.GetShortFolderName()}' "));
                         doc.Root.Add(itemGroup);
 
@@ -121,12 +149,181 @@ namespace PackageBaselineGenerator
                 Encoding = Encoding.UTF8,
                 Indent = true,
             };
+
             using (var writer = XmlWriter.Create(output, settings))
             {
                 doc.Save(writer);
             }
 
+            Console.WriteLine($"Generated file in {output}");
+
             return 0;
         }
+
+        private async Task<int> RunUpdateAsync(
+            string documentPath,
+            XDocument document,
+            SourceRepository sourceRepository)
+        {
+            var packageMetadataResource = await sourceRepository.GetResourceAsync<PackageMetadataResource>();
+            var logger = new Logger(Error, Out);
+            var hasChanged = false;
+            using (var cacheContext = new SourceCacheContext { NoCache = true })
+            {
+                var versionAttribute = document.Root.Attribute("Version");
+                hasChanged = await TryUpdateVersionAsync(
+                    versionAttribute,
+                    "Microsoft.AspNetCore.App",
+                    packageMetadataResource,
+                    logger,
+                    cacheContext);
+
+                foreach (var package in document.Root.Descendants("Package"))
+                {
+                    var id = package.Attribute("Id").Value;
+                    versionAttribute = package.Attribute("Version");
+                    var attributeChanged = await TryUpdateVersionAsync(
+                        versionAttribute,
+                        id,
+                        packageMetadataResource,
+                        logger,
+                        cacheContext);
+
+                    hasChanged |= attributeChanged;
+                }
+            }
+
+            if (hasChanged)
+            {
+                await Out.WriteLineAsync($"Updating {documentPath}.");
+
+                var settings = new XmlWriterSettings
+                {
+                    Async = true,
+                    CheckCharacters = true,
+                    CloseOutput = false,
+                    Encoding = Encoding.UTF8,
+                    Indent = true,
+                    IndentChars = "  ",
+                    NewLineOnAttributes = false,
+                    OmitXmlDeclaration = true,
+                    WriteEndDocumentOnClose = true,
+                };
+
+                using (var stream = File.OpenWrite(documentPath))
+                {
+                    using (var writer = XmlWriter.Create(stream, settings))
+                    {
+                        await document.SaveAsync(writer, CancellationToken.None);
+                    }
+                }
+            }
+            else
+            {
+                await Out.WriteLineAsync("No new versions found");
+            }
+
+            return 0;
+        }
+
+        private static async Task<bool> TryUpdateVersionAsync(
+            XAttribute versionAttribute,
+            string packageId,
+            PackageMetadataResource packageMetadataResource,
+            ILogger logger,
+            SourceCacheContext cacheContext)
+        {
+            var searchMetadata = await packageMetadataResource.GetMetadataAsync(
+                packageId,
+                includePrerelease: false,
+                includeUnlisted: true, // Microsoft.AspNetCore.DataOrotection.Redis package is not listed.
+                sourceCacheContext: cacheContext,
+                log: logger,
+                token: CancellationToken.None);
+
+            var currentVersion = NuGetVersion.Parse(versionAttribute.Value);
+            var versionRange = new VersionRange(
+                currentVersion,
+                new FloatRange(NuGetVersionFloatBehavior.Patch, currentVersion));
+
+            var latestVersion = versionRange.FindBestMatch(
+                searchMetadata.Select(metadata => metadata.Identity.Version));
+
+            if (latestVersion == null)
+            {
+                logger.LogWarning($"Unable to find latest version of '{packageId}'.");
+                return false;
+            }
+
+            var hasChanged = false;
+            if (latestVersion != currentVersion)
+            {
+                hasChanged = true;
+                versionAttribute.Value = latestVersion.ToNormalizedString();
+            }
+
+            return hasChanged;
+        }
+
+        private class Logger : ILogger
+        {
+            private readonly TextWriter _error;
+            private readonly TextWriter _out;
+
+            public Logger(TextWriter error, TextWriter @out)
+            {
+                _error = error;
+                _out = @out;
+            }
+
+            public void Log(LogLevel level, string data)
+            {
+                switch (level)
+                {
+                    case LogLevel.Debug:
+                        LogDebug(data);
+                        break;
+                    case LogLevel.Error:
+                        LogError(data);
+                        break;
+                    case LogLevel.Information:
+                        LogInformation(data);
+                        break;
+                    case LogLevel.Minimal:
+                        LogMinimal(data);
+                        break;
+                    case LogLevel.Verbose:
+                        LogVerbose(data);
+                        break;
+                    case LogLevel.Warning:
+                        LogWarning(data);
+                        break;
+                }
+            }
+
+            public void Log(ILogMessage message) => Log(message.Level, message.Message);
+
+            public Task LogAsync(LogLevel level, string data)
+            {
+                Log(level, data);
+                return Task.CompletedTask;
+            }
+
+            public Task LogAsync(ILogMessage message) => LogAsync(message.Level, message.Message);
+
+            public void LogDebug(string data) => _out.WriteLine($"Debug: {data}");
+
+            public void LogError(string data) => _error.WriteLine($"Error: {data}");
+
+            public void LogInformation(string data) => _out.WriteLine($"Information: {data}");
+
+            public void LogInformationSummary(string data) => _out.WriteLine($"Summary: {data}");
+
+            public void LogMinimal(string data) => _out.WriteLine($"Minimal: {data}");
+
+            public void LogVerbose(string data) => _out.WriteLine($"Verbose: {data}");
+
+            public void LogWarning(string data) => _out.WriteLine($"Warning: {data}");
+        }
     }
 }

eng/tools/BaselineGenerator/README.md

@@ -1,10 +1,20 @@
-BaselineGenerator
+# BaselineGenerator
-=================
 
 This tool is used to generate an MSBuild file which sets the "baseline" against which servicing updates are built.
 
 ## Usage
 
-1. Add to the [Baseline.xml](/eng/Baseline.xml) a list of package ID's and their latest released versions. The source of this information can typically
+Add `--package-source {source}` to the commands below if the packages of interest are not all hosted on NuGet.org.
-  be found in the build.xml file generated during ProdCon builds. See https://github.com/dotnet/versions/blob/master/build-info/dotnet/product/cli/release/2.1.6/build.xml for example.
+
+### Auto-update
+
+1. Run `dotnet run --update` in this project folder.
+2. Run `dotnet run` in this project.
+
+### Manual update
+
+1. Add to the [Baseline.xml](/eng/Baseline.xml) a list of package ID's and their latest released versions. The source of
+this information can typically be found in the build.xml file generated during ProdCon builds. See
+<https://github.com/dotnet/versions/blob/master/build-info/dotnet/product/cli/release/2.1.6/build.xml> for example.
+Update the version at the top of baseline.xml to match prior release (even if no packages changed in the prior release).
 2. Run `dotnet run` on this project.