Produce a lineup package containing all packages produced

build/RepositoryBuild.targets

@@ -1,5 +1,5 @@
 <Project>
-  <Target Name="_BuildRepositories">
+  <Target Name="_BuildRepositories" DependsOnTargets="GenerateLineup">
     <ItemGroup>
       <BatchedRepository Include="$(MSBuildProjectFullPath)">
         <BuildGroup>%(RepositoryToBuildInOrder.Order)</BuildGroup>
@@ -28,6 +28,11 @@
   <Target Name="_BuildRepository" DependsOnTargets="_PinVersions">
     <PropertyGroup>
       <!-- If there are duplicate properties, the properties which are defined later in the order would override the earlier ones -->
+      <RepositoryBuildArguments>$(RepositoryBuildArguments) '/p:Universe_Version=$(Version)'</RepositoryBuildArguments>
+      <RepositoryBuildArguments>$(RepositoryBuildArguments) '/p:Universe_LineupBuildDir=$(LineupBuildDir)'</RepositoryBuildArguments>
+      <RepositoryBuildArguments>$(RepositoryBuildArguments) '/p:Universe_IntermediateDir=$(IntermediateDir)'</RepositoryBuildArguments>
+      <RepositoryBuildArguments>$(RepositoryBuildArguments) '/p:CustomBeforeKoreBuildProps=$(MSBuildThisFileDirectory)repobuild\BeforeKoreBuild.props'</RepositoryBuildArguments>
+      <RepositoryBuildArguments>$(RepositoryBuildArguments) '/p:CustomAfterKoreBuildTargets=$(MSBuildThisFileDirectory)repobuild\AfterKoreBuild.targets'</RepositoryBuildArguments>
       <RepositoryBuildArguments>$(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash)</RepositoryBuildArguments>
       <RepositoryBuildArguments>$(RepositoryBuildArguments) /noconsolelogger '/l:RepoTasks.FlowLogger,$(MSBuildThisFileDirectory)tasks\bin\publish\RepoTasks.dll;Summary;FlowId=$(RepositoryToBuild)'</RepositoryBuildArguments>
 

build/dependencies.props

@@ -0,0 +1,26 @@
+<Project>
+
+  <!-- ASP.NET Core (non-Universe builds) -->
+  <PropertyGroup>
+    <AspNetCoreFeed>https://dotnet.myget.org/F/aspnetcore-ci-dev/api/v3/index.json</AspNetCoreFeed>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ExternalDependency Include="Libuv" Version="1.10.0" Source="$(AspNetCoreFeed)" />
+  </ItemGroup>
+
+  <!-- nuget.org -->
+  <PropertyGroup>
+    <DefaultNuGetFeed>https://api.nuget.org/v3/index.json</DefaultNuGetFeed>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ExternalDependency Include="BenchmarkDotNet" Version="0.10.3" Source="$(DefaultNuGetFeed)"/>
+    <ExternalDependency Include="Microsoft.NET.Test.Sdk" Version="15.3.0" Source="$(DefaultNuGetFeed)"/>
+    <ExternalDependency Include="Moq" Version="4.7.49" Source="$(DefaultNuGetFeed)"/>
+    <ExternalDependency Include="Newtonsoft.Json" Version="10.0.1" Source="$(DefaultNuGetFeed)"/>
+    <ExternalDependency Include="xunit.runner.visualstudio" Version="2.3.0-beta2-build1317" Source="$(DefaultNuGetFeed)"/>
+    <ExternalDependency Include="xunit" Version="2.3.0-beta2-build3683" Source="$(DefaultNuGetFeed)"/>
+  </ItemGroup>
+
+</Project>

build/repo.props

@@ -1,6 +1,13 @@
 <Project>
+  <Import Project="..\version.xml" />
+  <Import Project="dependencies.props" />
+
   <PropertyGroup>
     <!-- This repo does not have solutions to build -->
     <DisableDefaultTargets>true</DisableDefaultTargets>
+    <Version>$(VersionPrefix)</Version>
+    <Version Condition="'$(VersionSuffix)' != ''">$(Version)-$(VersionSuffix)</Version>
+    <Version Condition="'$(BuildNumber)' != ''">$(Version)-$(BuildNumber)</Version>
   </PropertyGroup>
+
 </Project>

build/repo.targets

@@ -4,10 +4,11 @@
   <PropertyGroup>
     <NuGetPublishVolatileFeed>https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package</NuGetPublishVolatileFeed>
 
+    <LineupBuildDir>$(ArtifactsDir)lineups\</LineupBuildDir>
     <_CloneRepositoryRoot>$(RepositoryRoot).r\</_CloneRepositoryRoot>
     <_DependencyBuildDirectory>$(RepositoryRoot).deps\build\</_DependencyBuildDirectory>
     <_DependencyPackagesDirectory>$(_DependencyBuildDirectory)</_DependencyPackagesDirectory>
-    <_RestoreGraphSpecsDirectory>$(RepositoryRoot)obj\package-specs\</_RestoreGraphSpecsDirectory>
+    <_RestoreGraphSpecsDirectory>$(IntermediateDir)package-specs\</_RestoreGraphSpecsDirectory>
 
     <_RepositoryListFileName>Repositories.props</_RepositoryListFileName>
     <_DefaultRepositoryList>$(MSBuildThisFileDirectory)$(_RepositoryListFileName)</_DefaultRepositoryList>
@@ -26,8 +27,25 @@
   <Import Project="$(_RepositoryListToImport)" />
 
   <Target Name="CleanUniverseArtifacts">
-    <RemoveDir Directories="$(RepositoryRoot)obj" Condition="Exists('$(RepositoryRoot)obj')" />
     <RemoveDir Directories="$(_CloneRepositoryRoot)" Condition="Exists('$(_CloneRepositoryRoot)') AND '$(ClearRepositoryCloneRoot)' != 'false'" />
+    <RemoveDir Directories="$(BuildDir);$(LineupBuildDir);$(IntermediateDir)" />
+    <MakeDir Directories="$(BuildDir);$(LineupBuildDir)" />
+  </Target>
+
+  <Target Name="GenerateLineup" DependsOnTargets="_GenerateBuildGraph">
+    <PackNuSpec NuSpecPath="$(MSBuildThisFileDirectory)..\lineups\Internal.AspNetCore.Universe.Lineup.nuspec"
+                DestinationFolder="$(LineupBuildDir)"
+                Properties="version=$(Version)"
+                Dependencies="@(PackagesProduced);@(ExternalDependency)">
+      <Output TaskParameter="Packages" ItemName="LineupPackage" />
+    </PackNuSpec>
+  </Target>
+
+  <Target Name="PublishLineupPackage" Condition="'$(PublishPackages)'=='true' AND @(LineupPackage->Count()) != 0" >
+    <PushNuGetPackages
+      Packages="@(LineupPackage)"
+      Feed="$(NuGetPublishVolatileFeed)"
+      ApiKey="$(APIKey)" />
   </Target>
 
   <Target Name="_PrepareRepositories">
@@ -107,7 +125,7 @@
   </Target>
 
   <Target Name="BuildRepositories"
-     DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;_GenerateBuildGraph;_BuildRepositories" />
+     DependsOnTargets="_PrepareRepositories;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;_GenerateBuildGraph;_BuildRepositories;PublishLineupPackage" />
 
   <Target Name="_PrepareRestoreGraphSpecs" DependsOnTargets="_PrepareRepositories">
 
@@ -137,12 +155,19 @@
     <MSBuild
       Projects="@(Solution)"
       Targets="GenerateRestoreGraphFile"
-      BuildInParallel="$(BuildInParallel)" />
+      Properties="BuildNumber=$(BuildNumber);KoreBuildRestoreTargetsImported=true"
+      BuildInParallel="$(BuildInParallel)"
+      ContinueOnError="true" />
   </Target>
 
   <Target Name="_GenerateBuildGraph" DependsOnTargets="_GenerateRestoreGraphSpecs">
-    <RepoTasks.CalculateBuildGraph Repositories="@(Repository)" StartGraphAt="$(BuildGraphOf)" PackageSpecsDirectory="$(_RestoreGraphSpecsDirectory)">
+    <RepoTasks.CalculateBuildGraph
+      Repositories="@(Repository)"
+      StartGraphAt="$(BuildGraphOf)"
+      DefaultPackageVersion="$(Version)"
+      PackageSpecsDirectory="$(_RestoreGraphSpecsDirectory)">
       <Output TaskParameter="RepositoriesToBuildInOrder" ItemName="RepositoryToBuildInOrder" />
+      <Output TaskParameter="PackagesProduced" ItemName="PackagesProduced" />
     </RepoTasks.CalculateBuildGraph>
   </Target>
 

build/repobuild/AfterKoreBuild.targets

@@ -0,0 +1,6 @@
+<!-- This file is imported in the context of an individual repo, not the universe build. -->
+<Project>
+  <ItemGroup>
+    <PackageLineup Update="Internal.AspNetCore.Universe.Lineup" Version="$(Universe_Version)" />
+  </ItemGroup>
+</Project>

build/repobuild/BeforeKoreBuild.props

@@ -0,0 +1,9 @@
+<!-- This file is imported in the context of an individual repo, not the universe build. -->
+<Project>
+  <PropertyGroup>
+    <!-- Restore locally-built lineups into a temp directory, not the global cache. -->
+    <PolicyRestorePackagesPath>$([MSBuild]::NormalizeDirectory($(Universe_IntermediateDir)))packages\</PolicyRestorePackagesPath>
+    <!-- Restore from the locally-built artifacts directory -->
+    <PolicyRestoreAdditionalSources>$(PolicyRestoreAdditionalSources);$([MSBuild]::NormalizeDirectory($(Universe_LineupBuildDir)))</PolicyRestoreAdditionalSources>
+  </PropertyGroup>
+</Project>

build/tasks/BuildGraph/DependencyGraphSpecProvider.cs

@@ -1,3 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
 using System.IO;
 using NuGet.ProjectModel;
 

build/tasks/BuildGraph/GraphBuilder.cs

@@ -1,12 +1,17 @@
-using System;
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using Microsoft.Build.Utilities;
 
 namespace RepoTools.BuildGraph
 {
     public static class GraphBuilder
     {
-        public static IList<GraphNode> Generate(IList<Repository> repositories, string root)
+        public static IList<GraphNode> Generate(IList<Repository> repositories, string root, TaskLoggingHelper log)
         {
             // Build global list of primary projects
             var primaryProjects = repositories.SelectMany(c => c.Projects)
@@ -31,7 +36,15 @@ namespace RepoTools.BuildGraph
                         var dependencyRepository = dependencyProject.Repository;
                         var dependencyNode = graphNodes[dependencyRepository];
 
-                        thisProjectRepositoryNode.Incoming.Add(dependencyNode);
+                        if (ReferenceEquals(thisProjectRepositoryNode, dependencyNode))
+                        {
+                            log.LogWarning("{0} has a package reference to a package produced in the same repo. {1} -> {2}", project.Repository.Name, Path.GetFileName(project.Path), packageDependency);
+                        }
+                        else
+                        {
+                            thisProjectRepositoryNode.Incoming.Add(dependencyNode);
+                        }
+
                         dependencyNode.Outgoing.Add(thisProjectRepositoryNode);
                     }
                 }

build/tasks/BuildGraph/GraphNode.cs

@@ -1,4 +1,7 @@
-using System.Collections.Generic;
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
 using System.Diagnostics;
 
 namespace RepoTools.BuildGraph

build/tasks/BuildGraph/Project.cs

@@ -1,4 +1,7 @@
-using System;
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 
@@ -14,6 +17,8 @@ namespace RepoTools.BuildGraph
 
         public string Name { get; }
 
+        public string Version { get; set; }
+
         public string Path { get; set; }
 
         public Repository Repository { get; set; }

build/tasks/BuildGraph/Repository.cs

@@ -1,4 +1,7 @@
-using System;
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -49,6 +52,7 @@ namespace RepoTools.BuildGraph
             var repository = new Repository(name);
 
             ReadSharedSourceProjects(Path.Combine(repositoryPath, "shared"), repository, repository.Projects);
+
             var srcDirectory = Path.GetFullPath(Path.Combine(repositoryPath, "src"))
                 .Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
 
@@ -78,6 +82,7 @@ namespace RepoTools.BuildGraph
                         {
                             Repository = repository,
                             Path = specProject.FilePath,
+                            Version = specProject.Version?.ToString(),
                         };
 
                         projectGroup.Add(project);

build/tasks/BuildGraph/TopologicalSort.cs

@@ -1,4 +1,7 @@
-using System;
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
 

build/tasks/CalculateBuildGraph.cs

@@ -15,19 +15,28 @@ namespace RepoTasks
         [Required]
         public ITaskItem[] Repositories { get; set; }
 
-        [Output]
+        /// <summary>
-        public ITaskItem[] RepositoriesToBuildInOrder { get; set; }
+        /// Directory that contains the package spec files.
+        /// </summary>
+        [Required]
+        public string PackageSpecsDirectory { get; set; }
+
+        /// <summary>
+        /// Default to use for packages that may be produced from nuspec, not csproj. (e.g. .Sources packages).
+        /// </summary>
+        [Required]
+        public string DefaultPackageVersion { get; set; }
 
         /// <summary>
         /// The repository at which to root the graph at
         /// </summary>
         public string StartGraphAt { get; set; }
 
-        /// <summary>
+        [Output]
-        /// Directory that contains the package spec files.
+        public ITaskItem[] RepositoriesToBuildInOrder { get; set; }
-        /// </summary>
+
-        [Required]
+        [Output]
-        public string PackageSpecsDirectory { get; set; }
+        public ITaskItem[] PackagesProduced { get; set; }
 
         public override bool Execute()
         {
@@ -36,7 +45,7 @@ namespace RepoTasks
             var repositoryPaths = Repositories.Select(r => r.GetMetadata("RepositoryPath")).ToList();
             var repositories = Repository.ReadAllRepositories(repositoryPaths, graphSpecProvider);
 
-            var graph = GraphBuilder.Generate(repositories, StartGraphAt);
+            var graph = GraphBuilder.Generate(repositories, StartGraphAt, Log);
             var repositoriesWithOrder = new List<(ITaskItem repository, int order)>();
             foreach (var repositoryTaskItem in Repositories)
             {
@@ -59,6 +68,17 @@ namespace RepoTasks
                 .Select(r => r.repository)
                 .ToArray();
 
+            var packages = new List<ITaskItem>();
+            foreach (var project in repositories.SelectMany(p => p.Projects))
+            {
+                var pkg = new TaskItem(project.Name);
+                var version = project.Version ?? DefaultPackageVersion;
+                pkg.SetMetadata("Version", version);
+                packages.Add(pkg);
+            }
+
+            PackagesProduced = packages.ToArray();
+
             return true;
         }
     }

build/tasks/RepoTasks.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NuGet.ProjectModel" Version="$(NuGetInMSBuildVersion)" />
+    <PackageReference Include="NuGet.ProjectModel" Version="$(NuGetInMSBuildVersion)" PrivateAssets="All" />
   </ItemGroup>
 
   <Import Project="$(RepoTasksSdkPath)\Sdk.targets" Condition="'$(RepoTasksSdkPath)' != '' "/>

build/tasks/VersionPinning/PinVersionUtility.cs

@@ -81,7 +81,7 @@ namespace RepoTasks.VersionPinning
                     _logger.LogMessage(MessageImportance.Normal, $"Pinning package versions for {specProject.FilePath}.");
                 }
 
-                var pinnedReferences = new XElement("ItemGroup");
+                var pinnedReferences = new XElement("ItemGroup", new XAttribute("Condition", "'$(PolicyDesignTimeBuild)' != 'true' AND !Exists('$(MSBuildThisFileDirectory)$(MSBuildProjectFile).nugetpolicy.g.targets')"));
                 foreach (var packageReference in projectPinMetadata.Packages)
                 {
                     (var tfm, var libraryRange, var exactVersion) = packageReference;

lineups/Internal.AspNetCore.Universe.Lineup.nuspec

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
+  <metadata>
+    <id>Internal.AspNetCore.Universe.Lineup</id>
+    <version>$version$</version>
+    <authors>Microsoft</authors>
+    <description>This package used to unify dependency versions across all Universe repos. Internal use only.</description>
+    <packageTypes>
+      <packageType name="lineup" />
+    </packageTypes>
+    <dependencies>
+      <!--
+        The list of dependencies will be augmented automatically by the build script.
+      -->
+    </dependencies>
+  </metadata>
+</package>