diff --git a/build/dependencies.props b/build/dependencies.props index 7094a9050b..4d8c14566c 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -10,6 +10,8 @@ false + + false @@ -52,17 +54,16 @@ - + - https://dotnet.myget.org/F/aspnetcore-ci-dev/api/v3/index.json + https://dotnet.myget.org/F/aspnetcore-release/api/v3/index.json - - + @@ -71,6 +72,7 @@ + diff --git a/build/repo.targets b/build/repo.targets index 2ddd3f45e9..73a7bb8306 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -16,10 +16,15 @@ <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify + + $(IntermediateDir)mirror\ + + $(IntermediateDir)ext\ Patch20_ $(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts $(CleanDependsOn);CleanUniverseArtifacts + $(RestoreDependsOn);RestoreExternalDependencies $(CompileDependsOn);CloneRepositories;BuildRepositories $(PackageDependsOn);SplitPackages $(VerifyDependsOn);VerifyCoherentVersions @@ -27,6 +32,19 @@ + + + + + + + + + @@ -192,11 +210,12 @@ + diff --git a/build/tasks/CopyPackagesToSplitFolders.cs b/build/tasks/CopyPackagesToSplitFolders.cs index 8f780e270f..d9e43808ba 100644 --- a/build/tasks/CopyPackagesToSplitFolders.cs +++ b/build/tasks/CopyPackagesToSplitFolders.cs @@ -68,6 +68,9 @@ namespace RepoTasks case PackageCategory.ShipOob: destDir = Path.Combine(DestinationFolder, "shipoob"); break; + case PackageCategory.Mirror: + destDir = Path.Combine(DestinationFolder, "mirror"); + break; default: throw new NotImplementedException(); } diff --git a/build/tasks/DownloadNuGetPackages.cs b/build/tasks/DownloadNuGetPackages.cs new file mode 100644 index 0000000000..8412cc5fbd --- /dev/null +++ b/build/tasks/DownloadNuGetPackages.cs @@ -0,0 +1,196 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Build; +using NuGet.Commands; +using NuGet.Configuration; +using NuGet.DependencyResolver; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; +using Task = System.Threading.Tasks.Task; + +namespace RepoTasks +{ + public class DownloadNuGetPackages : Microsoft.Build.Utilities.Task, ICancelableTask + { + private static readonly Task FalseTask = Task.FromResult(false); + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + + [Required] + public ITaskItem[] Packages { get; set; } + + [Required] + public string DestinationFolder { get; set; } + + [Output] + public ITaskItem[] Files { get; set; } + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + public async Task ExecuteAsync() + { + DestinationFolder = DestinationFolder.Replace('\\', '/'); + + var requests = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var files = new List(); + var downloadCount = 0; + foreach (var item in Packages) + { + var id = item.ItemSpec; + var rawVersion = item.GetMetadata("Version"); + if (!NuGetVersion.TryParse(rawVersion, out var version)) + { + Log.LogError($"Package '{id}' has an invalid 'Version' metadata value: '{rawVersion}'."); + return false; + } + + var source = item.GetMetadata("Source"); + if (string.IsNullOrEmpty(source)) + { + Log.LogError($"Package '{id}' is missing the 'Source' metadata value."); + return false; + } + + if (!requests.TryGetValue(source, out var packages)) + { + packages = requests[source] = new List(); + } + + var request = new PackageIdentity(id, version); + var dest = GetExpectedOutputPath(request); + files.Add(new TaskItem(dest)); + if (File.Exists(dest)) + { + Log.LogMessage($"Skipping {request.Id} {request.Version}. Already exists in '{dest}'"); + continue; + } + else + { + downloadCount++; + packages.Add(request); + } + } + + Files = files.ToArray(); + + if (downloadCount == 0) + { + Log.LogMessage("All packages are downloaded."); + return true; + } + + Directory.CreateDirectory(DestinationFolder); + var logger = new MSBuildLogger(Log); + var timer = Stopwatch.StartNew(); + + logger.LogMinimal($"Downloading {downloadCount} package(s)"); + + using (var cacheContext = new SourceCacheContext()) + { + var defaultSettings = Settings.LoadDefaultSettings(root: null, configFileName: null, machineWideSettings: null); + var sourceProvider = new CachingSourceProvider(new PackageSourceProvider(defaultSettings)); + var tasks = new List>(); + + foreach (var feed in requests) + { + var repo = sourceProvider.CreateRepository(new PackageSource(feed.Key)); + tasks.Add(DownloadPackagesAsync(repo, feed.Value, cacheContext, logger, _cts.Token)); + } + + var all = Task.WhenAll(tasks); + var wait = TimeSpan.FromSeconds(Math.Max(downloadCount * 5, 30)); + var delay = Task.Delay(wait); + + var finished = await Task.WhenAny(all, delay); + if (ReferenceEquals(delay, finished)) + { + Log.LogError($"Timed out after {wait.TotalSeconds}s"); + Cancel(); + return false; + } + + if (!tasks.All(a => a.Result)) + { + Log.LogError("Failed to download all packages"); + return false; + } + + timer.Stop(); + logger.LogMinimal($"Finished downloading {downloadCount} package(s) in {timer.ElapsedMilliseconds}ms"); + return true; + } + } + + private async Task DownloadPackagesAsync( + SourceRepository repo, + IEnumerable requests, + SourceCacheContext cacheContext, + NuGet.Common.ILogger logger, + CancellationToken cancellationToken) + { + var remoteLibraryProvider = new SourceRepositoryDependencyProvider(repo, logger, cacheContext, ignoreFailedSources: false, ignoreWarning: false); + var downloads = new List>(); + var metadataResource = await repo.GetResourceAsync(); + + foreach (var request in requests) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (metadataResource != null && !await metadataResource.Exists(request, logger, cancellationToken)) + { + logger.LogError($"Package {request.Id} {request.Version} is not available on '{repo}'"); + downloads.Add(FalseTask); + continue; + } + + var download = DownloadPackageAsync(cacheContext, logger, remoteLibraryProvider, request, cancellationToken); + downloads.Add(download); + } + + await Task.WhenAll(downloads); + return downloads.All(d => d.Result); + } + + private async Task DownloadPackageAsync(SourceCacheContext cacheContext, + NuGet.Common.ILogger logger, + SourceRepositoryDependencyProvider remoteLibraryProvider, + PackageIdentity request, + CancellationToken cancellationToken) + { + var dest = GetExpectedOutputPath(request); + logger.LogInformation($"Downloading {request.Id} {request.Version} to '{dest}'"); + + using (var packageDependency = await remoteLibraryProvider.GetPackageDownloaderAsync(request, cacheContext, logger, cancellationToken)) + { + if (!await packageDependency.CopyNupkgFileToAsync(dest, cancellationToken)) + { + logger.LogError($"Could not download {request.Id} {request.Version} from {remoteLibraryProvider.Source}"); + return false; + } + } + + return true; + } + + private string GetExpectedOutputPath(PackageIdentity request) + { + return Path.Combine(DestinationFolder, $"{request.Id.ToLowerInvariant()}.{request.Version.ToNormalizedString()}.nupkg"); + } + } +} diff --git a/build/tasks/Logger/MSBuildLogger.cs b/build/tasks/Logger/MSBuildLogger.cs new file mode 100644 index 0000000000..458bd698d9 --- /dev/null +++ b/build/tasks/Logger/MSBuildLogger.cs @@ -0,0 +1,200 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Common; + +namespace NuGet.Build +{ + /// + /// TaskLoggingHelper -> ILogger + /// + internal class MSBuildLogger : LoggerBase, Common.ILogger + { + private readonly TaskLoggingHelper _taskLogging; + + private delegate void LogMessageWithDetails(string subcategory, + string code, + string helpKeyword, + string file, + int lineNumber, + int columnNumber, + int endLineNumber, + int endColumnNumber, + MessageImportance importance, + string message, + params object[] messageArgs); + + private delegate void LogErrorWithDetails(string subcategory, + string code, + string helpKeyword, + string file, + int lineNumber, + int columnNumber, + int endLineNumber, + int endColumnNumber, + string message, + params object[] messageArgs); + + private delegate void LogMessageAsString(MessageImportance importance, + string message, + params object[] messageArgs); + + private delegate void LogErrorAsString(string message, + params object[] messageArgs); + + public MSBuildLogger(TaskLoggingHelper taskLogging) + { + _taskLogging = taskLogging ?? throw new ArgumentNullException(nameof(taskLogging)); + } + + public override void Log(ILogMessage message) + { + if (DisplayMessage(message.Level)) + { + if (RuntimeEnvironmentHelper.IsMono) + { + LogForMono(message); + return; + } + else + { + var logMessage = message as IRestoreLogMessage; + + if (logMessage == null) + { + logMessage = new RestoreLogMessage(message.Level, message.Message) + { + Code = message.Code, + FilePath = message.ProjectPath + }; + } + LogForNonMono(logMessage); + } + } + } + + /// + /// Log using with metadata for non mono platforms. + /// + private void LogForNonMono(IRestoreLogMessage message) + { + switch (message.Level) + { + case LogLevel.Error: + LogError(message, _taskLogging.LogError, _taskLogging.LogError); + break; + + case LogLevel.Warning: + LogError(message, _taskLogging.LogWarning, _taskLogging.LogWarning); + break; + + case LogLevel.Minimal: + LogMessage(message, MessageImportance.High, _taskLogging.LogMessage, _taskLogging.LogMessage); + break; + + case LogLevel.Information: + LogMessage(message, MessageImportance.Normal, _taskLogging.LogMessage, _taskLogging.LogMessage); + break; + + case LogLevel.Debug: + case LogLevel.Verbose: + default: + // Default to LogLevel.Debug and low importance + LogMessage(message, MessageImportance.Low, _taskLogging.LogMessage, _taskLogging.LogMessage); + break; + } + } + + /// + /// Log using basic methods to avoid missing methods on mono. + /// + private void LogForMono(ILogMessage message) + { + switch (message.Level) + { + case LogLevel.Error: + _taskLogging.LogError(message.Message); + break; + + case LogLevel.Warning: + _taskLogging.LogWarning(message.Message); + break; + + case LogLevel.Minimal: + _taskLogging.LogMessage(MessageImportance.High, message.Message); + break; + + case LogLevel.Information: + _taskLogging.LogMessage(MessageImportance.Normal, message.Message); + break; + + case LogLevel.Debug: + case LogLevel.Verbose: + default: + // Default to LogLevel.Debug and low importance + _taskLogging.LogMessage(MessageImportance.Low, message.Message); + break; + } + + return; + } + + private void LogMessage(IRestoreLogMessage logMessage, + MessageImportance importance, + LogMessageWithDetails logWithDetails, + LogMessageAsString logAsString) + { + if (logMessage.Code > NuGetLogCode.Undefined) + { + // NuGet does not currently have a subcategory while throwing logs, hence string.Empty + logWithDetails(string.Empty, + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + logMessage.FilePath, + logMessage.StartLineNumber, + logMessage.StartColumnNumber, + logMessage.EndLineNumber, + logMessage.EndColumnNumber, + importance, + logMessage.Message); + } + else + { + logAsString(importance, logMessage.Message); + } + } + + private void LogError(IRestoreLogMessage logMessage, + LogErrorWithDetails logWithDetails, + LogErrorAsString logAsString) + { + if (logMessage.Code > NuGetLogCode.Undefined) + { + // NuGet does not currently have a subcategory while throwing logs, hence string.Empty + logWithDetails(string.Empty, + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + Enum.GetName(typeof(NuGetLogCode), logMessage.Code), + logMessage.FilePath, + logMessage.StartLineNumber, + logMessage.StartColumnNumber, + logMessage.EndLineNumber, + logMessage.EndColumnNumber, + logMessage.Message); + } + else + { + logAsString(logMessage.Message); + } + } + + public override System.Threading.Tasks.Task LogAsync(ILogMessage message) + { + Log(message); + + return System.Threading.Tasks.Task.FromResult(0); + } + } +} diff --git a/build/tasks/RepoTasks.csproj b/build/tasks/RepoTasks.csproj index 084b189145..39574ccdb1 100644 --- a/build/tasks/RepoTasks.csproj +++ b/build/tasks/RepoTasks.csproj @@ -6,7 +6,7 @@ - + diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks index 12ec53fe04..d2df6c836a 100644 --- a/build/tasks/RepoTasks.tasks +++ b/build/tasks/RepoTasks.tasks @@ -5,6 +5,7 @@ + diff --git a/build/tasks/Utilities/PackageCategory.cs b/build/tasks/Utilities/PackageCategory.cs index 80f2990244..f51142bdb7 100644 --- a/build/tasks/Utilities/PackageCategory.cs +++ b/build/tasks/Utilities/PackageCategory.cs @@ -9,6 +9,7 @@ namespace RepoTasks.Utilities Unknown = 0, Shipping, NoShip, - ShipOob + ShipOob, + Mirror, } } diff --git a/build/tasks/Utilities/PackageCollection.cs b/build/tasks/Utilities/PackageCollection.cs index 4aa778aa2a..d066360f53 100644 --- a/build/tasks/Utilities/PackageCollection.cs +++ b/build/tasks/Utilities/PackageCollection.cs @@ -47,6 +47,9 @@ namespace RepoTasks.Utilities case "shipoob": category = PackageCategory.ShipOob; break; + case "mirror": + category = PackageCategory.Mirror; + break; default: category = PackageCategory.Unknown; break;