Support partial-graph builds

This merges pre-existing packages from previous builds of Universe with the list of packages to be produced in the current run.

Pre-existing packages are expected to be in .deps/build/ and are expected to have only one version of a package id in that folder.

This also requires any additional lineups to be in .deps/lineups/ so they can be filtered from the Internal.AspNetCore.Universe.Lineup file

build/repo.targets

@@ -7,6 +7,7 @@
     <LineupBuildDir>$(ArtifactsDir)lineups\</LineupBuildDir>
     <_CloneRepositoryRoot>$(RepositoryRoot).r\</_CloneRepositoryRoot>
     <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\</_DependencyBuildDirectory>
+    <_DependencyLineupDir>$(RepositoryRoot).deps\lineups\</_DependencyLineupDir>
     <_DependencyPackagesDirectory>$(_DependencyBuildDirectory)</_DependencyPackagesDirectory>
     <_RestoreGraphSpecsDirectory>$(IntermediateDir)package-specs\</_RestoreGraphSpecsDirectory>
 
@@ -32,11 +33,40 @@
     <MakeDir Directories="$(BuildDir);$(LineupBuildDir)" />
   </Target>
 
-  <Target Name="GenerateLineup" DependsOnTargets="_GenerateBuildGraph">
+  <Target Name="_ReadArtifactDependencyPackages">
+    <ItemGroup>
+      <_DependencyPackageFiles Include="$(_DependencyPackagesDirectory)*.nupkg"
+                               Exclude="$(_DependencyPackagesDirectory)*.symbols.nupkg"
+                               Condition=" '$(_DependencyPackagesDirectory)' != '' " />
+      <_DependencyLineupFiles Include="$(_DependencyLineupDir)*.nupkg"
+                              Exclude="$(_DependencyLineupDir)Internal.AspNetCore.Universe.Lineup.*.nupkg"
+                              Condition=" '$(_DependencyLineupDir)' != '' " />
+    </ItemGroup>
+
+    <RepoTasks.ReadPackageIdentity PackageFiles="@(_DependencyPackageFiles)">
+      <Output TaskParameter="PackageDefinitions" ItemName="ArtifactDependency" />
+    </RepoTasks.ReadPackageIdentity>
+
+    <RepoTasks.ReadPackageDependencies PackageFiles="@(_DependencyLineupFiles)">
+      <Output TaskParameter="PackageDefinitions" ItemName="DependencyLineupDependency" />
+    </RepoTasks.ReadPackageDependencies>
+  </Target>
+
+  <Target Name="GenerateLineup" DependsOnTargets="_GenerateBuildGraph;_ReadArtifactDependencyPackages">
+    <ItemGroup>
+      <_Dependency Remove="@(_Dependency)" />
+
+      <!-- order matters. It defines the precendence of what ends up in the lineup. -->
+      <_Dependency Include="@(PackagesProduced)" />
+      <_Dependency Include="@(ExternalDependency)" Exclude="@(_Dependency)"  />
+      <!-- avoid duplicating dependency lineups into the universe lineup -->
+      <_Dependency Include="@(ArtifactDependency)" Exclude="@(_Dependency);@(DependencyLineupDependency)" />
+    </ItemGroup>
+
     <PackNuSpec NuSpecPath="$(MSBuildThisFileDirectory)..\lineups\Internal.AspNetCore.Universe.Lineup.nuspec"
                 DestinationFolder="$(LineupBuildDir)"
                 Properties="version=$(Version)"
-                Dependencies="@(PackagesProduced);@(ExternalDependency)">
+                Dependencies="@(_Dependency)">
       <Output TaskParameter="Packages" ItemName="LineupPackage" />
     </PackNuSpec>
   </Target>

build/tasks/ReadPackageDependencies.cs

@@ -0,0 +1,44 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections;
+using System.Linq;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using NuGet.Packaging;
+
+namespace RepoTasks
+{
+    public class ReadPackageDependencies : Task
+    {
+        [Required]
+        public ITaskItem[] PackageFiles { get; set; }
+
+        [Output]
+        public ITaskItem[] PackageDefinitions { get; set; }
+
+        public override bool Execute()
+        {
+            PackageDefinitions = PackageFiles.SelectMany(item =>
+            {
+                using (var package = new PackageArchiveReader(item.ItemSpec))
+                {
+                    var identity = package.GetIdentity();
+                    var metadata = new NuspecReader(package.GetNuspec());
+                    var groups = metadata.GetDependencyGroups()?.ToList();
+                    if (groups == null)
+                    {
+                        return Enumerable.Empty<ITaskItem>();
+                    }
+
+                    return groups.SelectMany(g =>
+                        g.Packages.Select(p => new TaskItem(p.Id, new Hashtable { ["Version"] = p.VersionRange.MinVersion.ToString() })));
+                }
+            })
+            .ToArray();
+
+            return true;
+        }
+    }
+}

build/tasks/ReadPackageIdentity.cs

@@ -0,0 +1,37 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using NuGet.Packaging;
+
+namespace RepoTasks
+{
+    public class ReadPackageIdentity : Task
+    {
+        [Required]
+        public ITaskItem[] PackageFiles { get; set; }
+
+        [Output]
+        public ITaskItem[] PackageDefinitions { get; set; }
+
+        public override bool Execute()
+        {
+            PackageDefinitions = PackageFiles.Select(item =>
+            {
+                using (var package = new PackageArchiveReader(item.ItemSpec))
+                {
+                    var identity = package.GetIdentity();
+                    var packageItem = new TaskItem(identity.Id);
+                    packageItem.SetMetadata("Version", identity.Version.ToString());
+                    return packageItem;
+                }
+            })
+            .ToArray();
+
+            return true;
+        }
+    }
+}

build/tasks/RepoTasks.tasks

@@ -5,4 +5,6 @@
 
   <UsingTask TaskName="RepoTasks.CalculateBuildGraph" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.PinVersions" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ReadPackageIdentity" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ReadPackageDependencies" AssemblyFile="$(_RepoTaskAssembly)" />
 </Project>