Mirror external dependencies

build/dependencies.props

@@ -10,6 +10,8 @@
       <NoWarn></NoWarn>
       <!-- This dependency is 'Private', aka. it should not end up as a public-facing external dependency. This is validated by checking the nuspec on generated packages. -->
       <Private>false</Private>
+      <!-- When true, this dependency should be mirrored to aspnetcore's nightly build feeds. -->
+      <Mirror>false</Mirror>
     </ExternalDependency>
   </ItemDefinitionGroup>
 
@@ -52,17 +54,16 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ExternalDependency Include="Microsoft.AspNetCore.AspNetCoreModule" Version="1.0.0-pre-10057" Source="$(AspNetCoreModuleFeed)" Private="true" />
+    <ExternalDependency Include="Microsoft.AspNetCore.AspNetCoreModule" Version="1.0.0-pre-10057" Source="$(AspNetCoreModuleFeed)" Private="true" Mirror="true" />
   </ItemGroup>
 
   <!-- ASP.NET Core (non-Universe builds) -->
   <PropertyGroup>
-    <AspNetCoreFeed>https://dotnet.myget.org/F/aspnetcore-ci-dev/api/v3/index.json</AspNetCoreFeed>
+    <AspNetCoreFeed>https://dotnet.myget.org/F/aspnetcore-release/api/v3/index.json</AspNetCoreFeed>
   </PropertyGroup>
 
   <ItemGroup>
-    <ExternalDependency Include="Libuv" Version="1.10.0" Source="$(AspNetCoreFeed)"/>
+    <ExternalDependency Include="Microsoft.AspNetCore.Internal.CoreFxLab.Sources" Version="2.0.0-rtm-21470" Source="$(AspNetCoreFeed)" Private="true" />
-    <ExternalDependency Include="Microsoft.AspNetCore.Internal.CoreFxLab.Sources" Version="2.0.0-rtm-21470" Source="$(DefaultNuGetFeed)" Private="true" />
   </ItemGroup>
 
   <!-- nuget.org -->
@@ -71,6 +72,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ExternalDependency Include="Libuv" Version="1.10.0" Source="$(DefaultNuGetFeed)" />
     <ExternalDependency Include="BenchmarkDotNet" Version="0.10.3" Source="$(DefaultNuGetFeed)" Private="true"/>
     <ExternalDependency Include="EntityFramework" Version="6.1.3" Source="$(DefaultNuGetFeed)" Private="true" />
     <ExternalDependency Include="FSharp.Core" Version="4.2.1" Source="$(DefaultNuGetFeed)" Private="true" />

build/repo.targets

@@ -16,10 +16,15 @@
 
     <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages</_RepositoryBuildTargets>
     <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify</_RepositoryBuildTargets>
+    <!-- For external packages that come from feeds will mirrored to aspnetcore feeds. -->
+    <IntermediateMirrorPackageDir>$(IntermediateDir)mirror\</IntermediateMirrorPackageDir>
+    <!-- For external packages that come from feeds we don't mirror. -->
+    <IntermediateExternalPackageDir>$(IntermediateDir)ext\</IntermediateExternalPackageDir>
     <TcVcsIdPrefix>Patch20_</TcVcsIdPrefix>
 
     <PrepareDependsOn>$(PrepareDependsOn);CleanArtifacts;CleanUniverseArtifacts</PrepareDependsOn>
     <CleanDependsOn>$(CleanDependsOn);CleanUniverseArtifacts</CleanDependsOn>
+    <RestoreDependsOn>$(RestoreDependsOn);RestoreExternalDependencies</RestoreDependsOn>
     <CompileDependsOn>$(CompileDependsOn);CloneRepositories;BuildRepositories</CompileDependsOn>
     <PackageDependsOn>$(PackageDependsOn);SplitPackages</PackageDependsOn>
     <VerifyDependsOn>$(VerifyDependsOn);VerifyCoherentVersions</VerifyDependsOn>
@@ -27,6 +32,19 @@
 
   <Import Project="$(_RepositoryListToImport)" />
 
+  <Target Name="RestoreExternalDependencies">
+    <RepoTasks.DownloadNuGetPackages
+      Packages="@(ExternalDependency->WithMetadataValue('Mirror', 'false'))"
+      DestinationFolder="$(IntermediateExternalPackageDir)" />
+
+    <RepoTasks.DownloadNuGetPackages
+      Packages="@(ExternalDependency->WithMetadataValue('Mirror', 'true'))"
+      DestinationFolder="$(IntermediateMirrorPackageDir)">
+
+      <Output TaskParameter="Files" ItemName="MirroredPackageFile" />
+    </RepoTasks.DownloadNuGetPackages>
+  </Target>
+
   <Target Name="CleanUniverseArtifacts">
     <RemoveDir Directories="$(_CloneRepositoryRoot)" Condition="Exists('$(_CloneRepositoryRoot)') AND '$(ClearRepositoryCloneRoot)' != 'false'" />
     <RemoveDir Directories="$(RepositoryRoot)obj" Condition="Exists('$(RepositoryRoot)obj')" />
@@ -192,11 +210,12 @@
   <Target Name="SplitPackages">
     <ItemGroup>
       <PackageArtifactFile Include="$(BuildDir)*.nupkg" Exclude="$(BuildDir)*.symbols.nupkg" />
+      <ExternalDependencyPackage Include="@(ExternalDependency->WithMetadataValue('Mirror', 'true'))" Category="mirror" />
     </ItemGroup>
 
     <RepoTasks.CopyPackagesToSplitFolders
-      Packages="@(PackageArtifact)"
+      Packages="@(PackageArtifact);@(ExternalDependencyPackage)"
-      Files="@(PackageArtifactFile)"
+      Files="@(PackageArtifactFile);@(MirroredPackageFile)"
       DestinationFolder="$(ArtifactsDir)" />
   </Target>
 

build/tasks/CopyPackagesToSplitFolders.cs

@@ -68,6 +68,9 @@ namespace RepoTasks
                     case PackageCategory.ShipOob:
                         destDir = Path.Combine(DestinationFolder, "shipoob");
                         break;
+                    case PackageCategory.Mirror:
+                        destDir = Path.Combine(DestinationFolder, "mirror");
+                        break;
                     default:
                         throw new NotImplementedException();
                 }

build/tasks/DownloadNuGetPackages.cs

@@ -0,0 +1,196 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using NuGet.Build;
+using NuGet.Commands;
+using NuGet.Configuration;
+using NuGet.DependencyResolver;
+using NuGet.Packaging.Core;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
+using Task = System.Threading.Tasks.Task;
+
+namespace RepoTasks
+{
+    public class DownloadNuGetPackages : Microsoft.Build.Utilities.Task, ICancelableTask
+    {
+        private static readonly Task<bool> FalseTask = Task.FromResult(false);
+        private readonly CancellationTokenSource _cts = new CancellationTokenSource();
+
+        [Required]
+        public ITaskItem[] Packages { get; set; }
+
+        [Required]
+        public string DestinationFolder { get; set; }
+
+        [Output]
+        public ITaskItem[] Files { get; set; }
+
+        public void Cancel() => _cts.Cancel();
+
+        public override bool Execute()
+        {
+            return ExecuteAsync().GetAwaiter().GetResult();
+        }
+
+        public async Task<bool> ExecuteAsync()
+        {
+            DestinationFolder = DestinationFolder.Replace('\\', '/');
+
+            var requests = new Dictionary<string, List<PackageIdentity>>(StringComparer.OrdinalIgnoreCase);
+            var files = new List<ITaskItem>();
+            var downloadCount = 0;
+            foreach (var item in Packages)
+            {
+                var id = item.ItemSpec;
+                var rawVersion = item.GetMetadata("Version");
+                if (!NuGetVersion.TryParse(rawVersion, out var version))
+                {
+                    Log.LogError($"Package '{id}' has an invalid 'Version' metadata value: '{rawVersion}'.");
+                    return false;
+                }
+
+                var source = item.GetMetadata("Source");
+                if (string.IsNullOrEmpty(source))
+                {
+                    Log.LogError($"Package '{id}' is missing the 'Source' metadata value.");
+                    return false;
+                }
+
+                if (!requests.TryGetValue(source, out var packages))
+                {
+                    packages = requests[source] = new List<PackageIdentity>();
+                }
+
+                var request = new PackageIdentity(id, version);
+                var dest = GetExpectedOutputPath(request);
+                files.Add(new TaskItem(dest));
+                if (File.Exists(dest))
+                {
+                    Log.LogMessage($"Skipping {request.Id} {request.Version}. Already exists in '{dest}'");
+                    continue;
+                }
+                else
+                {
+                    downloadCount++;
+                    packages.Add(request);
+                }
+            }
+
+            Files = files.ToArray();
+
+            if (downloadCount == 0)
+            {
+                Log.LogMessage("All packages are downloaded.");
+                return true;
+            }
+
+            Directory.CreateDirectory(DestinationFolder);
+            var logger = new MSBuildLogger(Log);
+            var timer = Stopwatch.StartNew();
+
+            logger.LogMinimal($"Downloading {downloadCount} package(s)");
+
+            using (var cacheContext = new SourceCacheContext())
+            {
+                var defaultSettings = Settings.LoadDefaultSettings(root: null, configFileName: null, machineWideSettings: null);
+                var sourceProvider = new CachingSourceProvider(new PackageSourceProvider(defaultSettings));
+                var tasks = new List<Task<bool>>();
+
+                foreach (var feed in requests)
+                {
+                    var repo = sourceProvider.CreateRepository(new PackageSource(feed.Key));
+                    tasks.Add(DownloadPackagesAsync(repo, feed.Value, cacheContext, logger, _cts.Token));
+                }
+
+                var all = Task.WhenAll(tasks);
+                var wait = TimeSpan.FromSeconds(Math.Max(downloadCount * 5, 30));
+                var delay = Task.Delay(wait);
+
+                var finished = await Task.WhenAny(all, delay);
+                if (ReferenceEquals(delay, finished))
+                {
+                    Log.LogError($"Timed out after {wait.TotalSeconds}s");
+                    Cancel();
+                    return false;
+                }
+
+                if (!tasks.All(a => a.Result))
+                {
+                    Log.LogError("Failed to download all packages");
+                    return false;
+                }
+
+                timer.Stop();
+                logger.LogMinimal($"Finished downloading {downloadCount} package(s) in {timer.ElapsedMilliseconds}ms");
+                return true;
+            }
+        }
+
+        private async Task<bool> DownloadPackagesAsync(
+            SourceRepository repo,
+            IEnumerable<PackageIdentity> requests,
+            SourceCacheContext cacheContext,
+            NuGet.Common.ILogger logger,
+            CancellationToken cancellationToken)
+        {
+            var remoteLibraryProvider = new SourceRepositoryDependencyProvider(repo, logger, cacheContext, ignoreFailedSources: false, ignoreWarning: false);
+            var downloads = new List<Task<bool>>();
+            var metadataResource = await repo.GetResourceAsync<MetadataResource>();
+
+            foreach (var request in requests)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                if (metadataResource != null && !await metadataResource.Exists(request, logger, cancellationToken))
+                {
+                    logger.LogError($"Package {request.Id} {request.Version} is not available on '{repo}'");
+                    downloads.Add(FalseTask);
+                    continue;
+                }
+
+                var download = DownloadPackageAsync(cacheContext, logger, remoteLibraryProvider, request, cancellationToken);
+                downloads.Add(download);
+            }
+
+            await Task.WhenAll(downloads);
+            return downloads.All(d => d.Result);
+        }
+
+        private async Task<bool> DownloadPackageAsync(SourceCacheContext cacheContext,
+            NuGet.Common.ILogger logger,
+            SourceRepositoryDependencyProvider remoteLibraryProvider,
+            PackageIdentity request,
+            CancellationToken cancellationToken)
+        {
+            var dest = GetExpectedOutputPath(request);
+            logger.LogInformation($"Downloading {request.Id} {request.Version} to '{dest}'");
+
+            using (var packageDependency = await remoteLibraryProvider.GetPackageDownloaderAsync(request, cacheContext, logger, cancellationToken))
+            {
+                if (!await packageDependency.CopyNupkgFileToAsync(dest, cancellationToken))
+                {
+                    logger.LogError($"Could not download {request.Id} {request.Version} from {remoteLibraryProvider.Source}");
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private string GetExpectedOutputPath(PackageIdentity request)
+        {
+            return Path.Combine(DestinationFolder, $"{request.Id.ToLowerInvariant()}.{request.Version.ToNormalizedString()}.nupkg");
+        }
+    }
+}

build/tasks/Logger/MSBuildLogger.cs

@@ -0,0 +1,200 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using NuGet.Common;
+
+namespace NuGet.Build
+{
+    /// <summary>
+    /// TaskLoggingHelper -> ILogger
+    /// </summary>
+    internal class MSBuildLogger : LoggerBase, Common.ILogger
+    {
+        private readonly TaskLoggingHelper _taskLogging;
+
+        private delegate void LogMessageWithDetails(string subcategory,
+            string code,
+            string helpKeyword,
+            string file,
+            int lineNumber,
+            int columnNumber,
+            int endLineNumber,
+            int endColumnNumber,
+            MessageImportance importance,
+            string message,
+            params object[] messageArgs);
+
+        private delegate void LogErrorWithDetails(string subcategory,
+            string code,
+            string helpKeyword,
+            string file,
+            int lineNumber,
+            int columnNumber,
+            int endLineNumber,
+            int endColumnNumber,
+            string message,
+            params object[] messageArgs);
+
+        private delegate void LogMessageAsString(MessageImportance importance,
+            string message,
+            params object[] messageArgs);
+
+        private delegate void LogErrorAsString(string message,
+            params object[] messageArgs);
+
+        public MSBuildLogger(TaskLoggingHelper taskLogging)
+        {
+            _taskLogging = taskLogging ?? throw new ArgumentNullException(nameof(taskLogging));
+        }
+
+        public override void Log(ILogMessage message)
+        {
+            if (DisplayMessage(message.Level))
+            {
+                if (RuntimeEnvironmentHelper.IsMono)
+                {
+                    LogForMono(message);
+                    return;
+                }
+                else
+                {
+                    var logMessage = message as IRestoreLogMessage;
+
+                    if (logMessage == null)
+                    {
+                        logMessage = new RestoreLogMessage(message.Level, message.Message)
+                        {
+                            Code = message.Code,
+                            FilePath = message.ProjectPath
+                        };
+                    }
+                    LogForNonMono(logMessage);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Log using with metadata for non mono platforms.
+        /// </summary>
+        private void LogForNonMono(IRestoreLogMessage message)
+        {
+            switch (message.Level)
+            {
+                case LogLevel.Error:
+                    LogError(message, _taskLogging.LogError, _taskLogging.LogError);
+                    break;
+
+                case LogLevel.Warning:
+                    LogError(message, _taskLogging.LogWarning, _taskLogging.LogWarning);
+                    break;
+
+                case LogLevel.Minimal:
+                    LogMessage(message, MessageImportance.High, _taskLogging.LogMessage, _taskLogging.LogMessage);
+                    break;
+
+                case LogLevel.Information:
+                    LogMessage(message, MessageImportance.Normal, _taskLogging.LogMessage, _taskLogging.LogMessage);
+                    break;
+
+                case LogLevel.Debug:
+                case LogLevel.Verbose:
+                default:
+                    // Default to LogLevel.Debug and low importance
+                    LogMessage(message, MessageImportance.Low, _taskLogging.LogMessage, _taskLogging.LogMessage);
+                    break;
+            }
+        }
+
+        /// <summary>
+        /// Log using basic methods to avoid missing methods on mono.
+        /// </summary>
+        private void LogForMono(ILogMessage message)
+        {
+            switch (message.Level)
+            {
+                case LogLevel.Error:
+                    _taskLogging.LogError(message.Message);
+                    break;
+
+                case LogLevel.Warning:
+                    _taskLogging.LogWarning(message.Message);
+                    break;
+
+                case LogLevel.Minimal:
+                    _taskLogging.LogMessage(MessageImportance.High, message.Message);
+                    break;
+
+                case LogLevel.Information:
+                    _taskLogging.LogMessage(MessageImportance.Normal, message.Message);
+                    break;
+
+                case LogLevel.Debug:
+                case LogLevel.Verbose:
+                default:
+                    // Default to LogLevel.Debug and low importance
+                    _taskLogging.LogMessage(MessageImportance.Low, message.Message);
+                    break;
+            }
+
+            return;
+        }
+
+        private void LogMessage(IRestoreLogMessage logMessage,
+            MessageImportance importance,
+            LogMessageWithDetails logWithDetails,
+            LogMessageAsString logAsString)
+        {
+            if (logMessage.Code > NuGetLogCode.Undefined)
+            {
+                // NuGet does not currently have a subcategory while throwing logs, hence string.Empty
+                logWithDetails(string.Empty,
+                    Enum.GetName(typeof(NuGetLogCode), logMessage.Code),
+                    Enum.GetName(typeof(NuGetLogCode), logMessage.Code),
+                    logMessage.FilePath,
+                    logMessage.StartLineNumber,
+                    logMessage.StartColumnNumber,
+                    logMessage.EndLineNumber,
+                    logMessage.EndColumnNumber,
+                    importance,
+                    logMessage.Message);
+            }
+            else
+            {
+                logAsString(importance, logMessage.Message);
+            }
+        }
+
+        private void LogError(IRestoreLogMessage logMessage,
+            LogErrorWithDetails logWithDetails,
+            LogErrorAsString logAsString)
+        {
+            if (logMessage.Code > NuGetLogCode.Undefined)
+            {
+                // NuGet does not currently have a subcategory while throwing logs, hence string.Empty
+                logWithDetails(string.Empty,
+                    Enum.GetName(typeof(NuGetLogCode), logMessage.Code),
+                    Enum.GetName(typeof(NuGetLogCode), logMessage.Code),
+                    logMessage.FilePath,
+                    logMessage.StartLineNumber,
+                    logMessage.StartColumnNumber,
+                    logMessage.EndLineNumber,
+                    logMessage.EndColumnNumber,
+                    logMessage.Message);
+            }
+            else
+            {
+                logAsString(logMessage.Message);
+            }
+        }
+
+        public override System.Threading.Tasks.Task LogAsync(ILogMessage message)
+        {
+            Log(message);
+
+            return System.Threading.Tasks.Task.FromResult(0);
+        }
+    }
+}

build/tasks/RepoTasks.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NuGet.ProjectModel" Version="4.3.0" PrivateAssets="All" />
+    <PackageReference Include="NuGet.Build.Tasks" Version="$(NuGetInMSBuildVersion)" PrivateAssets="All" />
   </ItemGroup>
 
   <Import Project="$(RepoTasksSdkPath)\Sdk.targets" Condition="'$(RepoTasksSdkPath)' != '' "/>

build/tasks/RepoTasks.tasks

@@ -5,6 +5,7 @@
 
   <UsingTask TaskName="RepoTasks.AnalyzeBuildGraph" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.CopyPackagesToSplitFolders" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.DownloadNuGetPackages" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.GenerateLineup" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.VerifyCoherentVersions" AssemblyFile="$(_RepoTaskAssembly)" />
 </Project>

build/tasks/Utilities/PackageCategory.cs

@@ -9,6 +9,7 @@ namespace RepoTasks.Utilities
         Unknown = 0,
         Shipping,
         NoShip,
-        ShipOob
+        ShipOob,
+        Mirror,
     }
 }

build/tasks/Utilities/PackageCollection.cs

@@ -47,6 +47,9 @@ namespace RepoTasks.Utilities
                     case "shipoob":
                         category = PackageCategory.ShipOob;
                         break;
+                    case "mirror":
+                        category = PackageCategory.Mirror;
+                        break;
                     default:
                         category = PackageCategory.Unknown;
                         break;