Fix #785 - publish nuget packages and the shared runtime at the same time

build/Publish.targets

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <DependencyAssetsDir>$(RepositoryRoot).deps\assets\</DependencyAssetsDir>
     <DependencyPackagesDir>$(RepositoryRoot).deps\packages\</DependencyPackagesDir>
-    <DependencySymbolsDir>$(RepositoryRoot).deps\symbols\</DependencySymbolsDir>
+    <DependencyMirrorDir>$(RepositoryRoot).deps\mirror\</DependencyMirrorDir>
     <!-- This file is used by the dotnet/cli to determine if our shared framework aligns with the version they pull. -->
     <BaseRuntimeVersionFileName>aspnetcore_base_runtime.version</BaseRuntimeVersionFileName>
     <BaseRuntimeVersionFile>$(IntermediateDir)$(BaseRuntimeVersionFileName)</BaseRuntimeVersionFile>
@@ -17,6 +17,7 @@
       CopyToPublishArtifacts;
       PublishToAzureFeed;
       PublishToTransportFeed;
+      PublishToMyGet;
     </PublishDependsOn>
 
     <!-- Settings for pushing to the transport feed -->
@@ -68,14 +69,12 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <!-- symbols -->
+      <!-- symbols archives -->
       <FilesToPublish Include="$(DependencyAssetsDir)$(SymbolsArchiveBaseFileName)-%(RuntimeSymbolsArchive.Identity)%(RuntimeSymbolsArchive.FileExt)" Condition=" '%(RuntimeSymbolsArchive.Identity)' != '' ">
         <RelativeBlobPath>$(BlobBasePath)$(SymbolsArchiveBaseFileName)-%(RuntimeSymbolsArchive.Identity)%(RuntimeSymbolsArchive.FileExt)</RelativeBlobPath>
         <ManifestArtifactData>ShipInstaller=dotnetcli</ManifestArtifactData>
       </FilesToPublish>
 
-      <PackagesToPublishToTransport Include="$(DependencySymbolsDir)\**\*.symbols.nupkg" />
-
       <!-- Intermediate files passed on to the dotnet-CLI. -->
       <FilesToPublish Include="$(DependencyAssetsDir)$(PackageArchiveFileName)" >
         <RelativeBlobPath>$(BlobBasePath)$(PackageArchiveFileName)</RelativeBlobPath>
@@ -124,33 +123,61 @@
         <Overwrite>true</Overwrite>
       </FilesToPublish>
 
-      <_Artifact
+      <!-- Packages -->
+      <PackageToMirror Include="$(DependencyMirrorDir)%(ExternalDependency.Identity).%(ExternalDependency.Version).nupkg"
+                        Condition=" '%(ExternalDependency.Mirror)' == 'true' " />
+
+      <_PackageArtifactInfo
         Include="@(ArtifactInfo)"
         ArtifactPath="$(DependencyPackagesDir)%(ArtifactInfo.PackageId).%(ArtifactInfo.Version).nupkg"
         Condition="'%(ArtifactInfo.ArtifactType)' == 'NuGetPackage'" />
+
+      <_SymbolsPackageArtifactInfo
+        Include="@(ArtifactInfo)"
+        ArtifactPath="$(DependencyPackagesDir)%(ArtifactInfo.PackageId).%(ArtifactInfo.Version).symbols.nupkg"
+        Condition="'%(ArtifactInfo.ArtifactType)' == 'NuGetSymbolsPackage'" />
+
+      <NpmPackageToPublish Include="$(DependencyAssetsDir)%(ArtifactInfo.FileName)%(ArtifactInfo.Extension)" Condition="'%(ArtifactInfo.ArtifactType)' == 'NpmPackage'" />
     </ItemGroup>
 
+    <!-- Join required because shipping category is stored in universe (PackageArtifact), but information about package ID and version comes from repos (ArtifactInfo). -->
     <RepoTasks.JoinItems
-      Left="@(_Artifact)" LeftKey="PackageId" LeftMetadata="*" LeftItemSpec="Identity"
+      Left="@(_PackageArtifactInfo)" LeftKey="PackageId" LeftMetadata="*" LeftItemSpec="Identity"
       Right="@(PackageArtifact)" RightMetadata="Category">
-      <Output TaskParameter="JoinResult" ItemName="PackagesToPublish" />
+      <Output TaskParameter="JoinResult" ItemName="_PackageArtifactInfoWithCategory" />
     </RepoTasks.JoinItems>
 
     <ItemGroup>
-      <_MissingPackages Include="@(PackageArtifact)" />
+      <_MissingPackageArtifact Include="@(PackageArtifact)" />
-      <_MissingPackages Remove="%(PackagesToPublish.PackageId)" />
+      <_MissingPackageArtifact Remove="%(_PackageArtifactInfoWithCategory.PackageId)" />
     </ItemGroup>
 
-    <Error Text="Missing expected packages from PackagesToPublish: %0A - @(_MissingPackages, '%0A - ')" Condition="@(_MissingPackages->Count()) != 0" />
+    <RepoTasks.JoinItems
+      Left="@(_SymbolsPackageArtifactInfo)" LeftKey="PackageId" LeftMetadata="*" LeftItemSpec="Identity"
+      Right="@(PackageArtifact)" RightMetadata="Category">
+      <Output TaskParameter="JoinResult" ItemName="_SymbolsArtifactInfoWithCategory" />
+    </RepoTasks.JoinItems>
+
+    <!--
+      Add symbols packages to PackageToPublish after validating PackageToPublish matches PackageArtifact.
+      We don't always produce a symbols package for each regular packages.
+    -->
+    <ItemGroup>
+      <PackageToPublish Include="%(_PackageArtifactInfoWithCategory.ArtifactPath)" Category="%(_PackageArtifactInfoWithCategory.Category)" />
+      <PackageToPublish Include="%(_SymbolsArtifactInfoWithCategory.ArtifactPath)" Category="%(_SymbolsArtifactInfoWithCategory.Category)" />
+    </ItemGroup>
   </Target>
 
   <Target Name="_CheckFilesToPublish">
     <ItemGroup>
-      <_MissingFiles Include="%(FilesToPublish.Identity)" Condition="!Exists(%(FilesToPublish.Identity))" />
+      <_MissingArtifactFile Include="@(FilesToPublish)" Condition="!Exists(%(FilesToPublish.Identity))" />
-      <_MissingFiles Include="%(PackagesToPublish.ArtifactPath)" Condition="!Exists(%(PackagesToPublish.ArtifactPath))" />
+      <_MissingArtifactFile Include="@(NpmPackageToPublish)" Condition="!Exists(%(NpmPackageToPublish.Identity))" />
+      <_MissingArtifactFile Include="@(PackageToMirror)" Condition="!Exists(%(PackageToMirror.Identity))" />
+      <_MissingArtifactFile Include="@(PackageToPublish)" Condition="!Exists(%(PackageToPublish.Identity))" />
     </ItemGroup>
 
-    <Error Text="Missing expected files:%0A - @(_MissingFiles, '%0A - ')" Condition="@(_MissingFiles->Count()) != 0" />
+    <Error Text="Missing expected files:%0A - @(_MissingArtifactFile, '%0A - ')" Condition="@(_MissingArtifactFile->Count()) != 0" />
+    <Error Text="Missing expected packages from PackageToPublish. These were defined as expected PackageArtifact's in artifacts.props: %0A - @(_MissingPackageArtifact, '%0A - ')" Condition="@(_MissingPackageArtifact->Count()) != 0" />
   </Target>
 
   <Target Name="CopyToPublishArtifacts" DependsOnTargets="_CheckFilesToPublish">
@@ -158,6 +185,48 @@
     <Copy SourceFiles="@(FilesToPublish)" DestinationFolder="$(ArtifactsDir)files\" />
   </Target>
 
+  <Target Name="PublishToMyGet"
+          DependsOnTargets="_CheckFilesToPublish;GetToolsets"
+          Condition="'$(PublishToMyget)' == 'true'">
+
+    <Error Text="Missing required property: PublishMyGetFeedUrl"  Condition=" '$(PublishMyGetFeedUrl)' == '' "/>
+    <Error Text="Missing required property: PublishMyGetNpmRegistryUrl"  Condition=" '$(PublishMyGetNpmRegistryUrl)' == '' "/>
+    <Error Text="Missing required property: PublishMyGetFeedKey"  Condition=" '$(PublishMyGetFeedKey)' == '' "/>
+
+    <!-- Contains the super set of packages that we'd like to publish to MyGet and Azure blob -->
+    <ItemGroup>
+      <PackageToPublishToMyGet Include="@(PackageToPublish)" />
+      <PackageToPublishToMyGet Include="@(PackageToMirror)" />
+    </ItemGroup>
+
+    <Error Message="No packages found to publish" Condition="@(PackageToPublishToMyGet->Count()) == 0" />
+
+    <PushNuGetPackages
+      Packages="@(PackageToPublishToMyGet)"
+      Feed="$(PublishMyGetFeedUrl)"
+      ApiKey="$(PublishMyGetFeedKey)" />
+
+    <PropertyGroup>
+      <AuthTokenSetting>$(PublishMyGetNpmRegistryUrl.Replace("https:", "")):_authToken</AuthTokenSetting>
+    </PropertyGroup>
+
+    <Exec Command="npm config set &quot;$(AuthTokenSetting)&quot; $(PublishMyGetFeedKey)"
+          StandardOutputImportance="Normal" />
+
+    <Run FileName="npm"
+         Arguments="publish;--registry;$(PublishMyGetNpmRegistryUrl);%(NpmPackageToPublish.Identity)"
+         MaxRetries="5"
+         UseShellExecute="true"
+         ContinueOnError="true">
+      <Output TaskParameter="ExitCode" ItemName="_NpmExitCodes" />
+    </Run>
+
+    <Exec Command="npm config delete $(AuthTokenSetting)"
+          StandardOutputImportance="Normal" />
+
+    <Error Text="Publishing npm modules failed" Condition="%(_NpmExitCodes.Identity) != 0" />
+  </Target>
+
   <Target Name="PublishToAzureFeed"
     DependsOnTargets="_CheckFilesToPublish"
     Condition="'$(PublishToAzureFeed)' == 'true'">
@@ -186,12 +255,12 @@
     Condition="'$(PublishToTransportFeed)' == 'true'">
 
     <ItemGroup>
-      <PackagesToPublishToTransport
+      <PackageToPublishToTransport
-        Include="%(PackagesToPublish.ArtifactPath)"
+        Include="@(PackageToPublish)"
         Condition="'%(PackageToPublish.Category)' == 'ship'" />
 
-      <PackagesToPublishToTransport
+      <PackageToPublishToTransport
-        Include="%(PackagesToPublish.ArtifactPath)"
+        Include="@(PackageToPublish)"
         ManifestArtifactData="NonShipping=true"
         Condition="'%(PackageToPublish.Category)' != 'ship'" />
 
@@ -202,19 +271,19 @@
         Condition=" '%(FilesToPublish.IsDuplicateUpload)' != 'true' " />
     </ItemGroup>
 
-    <Message Text="Publishing %(PackagesToPublishToTransport.FullPath)" Importance="high" />
+    <Message Text="Publishing %(PackageToPublishToTransport.FullPath)" Importance="high" />
     <Message Text="@(FilesToPublishToTransport -> '%(FullPath) -> %(RelativeBlobPath)','%0A')" Importance="high" />
 
     <PushToBlobFeed ExpectedFeedUrl="$(PublishBlobFeedUrl)"
                     AccountKey="$(PublishBlobFeedKey)"
-                    ItemsToPush="@(PackagesToPublishToTransport)"
+                    ItemsToPush="@(PackageToPublishToTransport)"
                     Overwrite="$(PushToBlobFeed_Overwrite)"
                     ManifestBuildId="$(Version)"
                     ManifestBuildData="ProductVersion=$(PackageVersion);UniverseCommitHash=$(CommitHash)"
                     ManifestCommit="$(CommitHash)"
                     ManifestName="aspnet"
                     MaxClients="$(PushToBlobFeed_MaxClients)"
-                    Condition="@(PackagesToPublish->Count()) != 0" />
+                    Condition="@(PackageToPublish->Count()) != 0" />
 
     <PushToBlobFeed ExpectedFeedUrl="$(PublishBlobFeedUrl)"
                     AccountKey="$(PublishBlobFeedKey)"

build/artifacts.props

@@ -300,5 +300,7 @@
     <PackageArtifact Include="Microsoft.VisualStudio.Web.CodeGenerators.Mvc" Category="ship" LZMA="true" />
     <PackageArtifact Include="Microsoft.Web.Xdt.Extensions" Category="shipoob" />
     <PackageArtifact Include="RazorPageGenerator" Category="noship" />
+
+    <PackageArtifact Include="Internal.AspNetCore.Universe.Lineup" Category="noship" PackageType="Lineup" />
   </ItemGroup>
 </Project>

build/repo.props

@@ -12,7 +12,6 @@
     <IsFinalBuild Condition="'$(IsFinalBuild)' == ''">false</IsFinalBuild>
 
     <SubmoduleRoot>$(RepositoryRoot)modules\</SubmoduleRoot>
-    <LineupBuildDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir)))lineups\</LineupBuildDir>
 
     <DependencyPackageDir>$(RepositoryRoot).deps\build\</DependencyPackageDir>
     <DependencyMirrorPackageDir>$(RepositoryRoot).deps\mirror\</DependencyMirrorPackageDir>
@@ -49,9 +48,13 @@
       FeedCredential="$(DotNetAssetRootAccessTokenSuffix)" />
   </ItemGroup>
 
+  <!-- Properties for publishing -->
   <PropertyGroup>
+    <!-- myget = non-orchestrated builds -->
+    <PublishToMyGet Condition=" $(PublishType.Contains('myget')) ">true</PublishToMyGet>
     <!-- azure = non-orchestrated builds -->
     <PublishToAzureFeed Condition="$(PublishType.Contains('azure'))">true</PublishToAzureFeed>
+
     <!-- blob = orchestrated builds -->
     <PublishToTransportFeed Condition="$(PublishType.Contains('blob'))">true</PublishToTransportFeed>
   </PropertyGroup>

build/repo.targets

@@ -22,11 +22,11 @@
     <PackageDependsOn Condition="'$(TestOnly)' != 'true'">$(PackageDependsOn);BuildMetapackages;BuildTemplates;SplitPackages</PackageDependsOn>
     <TestDependsOn>$(TestDependsOn);_TestRepositories</TestDependsOn>
     <VerifyDependsOn Condition="'$(TestOnly)' != 'true'">$(VerifyDependsOn);VerifyCoherentVersions</VerifyDependsOn>
-    <GetArtifactInfoDependsOn>$(GetArtifactInfoDependsOn);ResolveRepoInfo</GetArtifactInfoDependsOn>
+    <GetArtifactInfoDependsOn>$(GetArtifactInfoDependsOn);ResolveRepoInfo;GetLineupPackageInfo</GetArtifactInfoDependsOn>
   </PropertyGroup>
 
   <Target Name="PrepareOutputPaths">
-    <MakeDir Directories="$(ArtifactsDir);$(BuildDir);$(LineupBuildDir)" />
+    <MakeDir Directories="$(ArtifactsDir);$(BuildDir)" />
   </Target>
 
   <Target Name="RestoreExternalDependencies">
@@ -88,12 +88,22 @@
     <Error Text="No solutions were found in '$(SubmoduleRoot)'" Condition="@(Solution->Count()) == 0" />
   </Target>
 
-  <Target Name="GeneratePropsFiles" DependsOnTargets="ResolveRepoInfo">
+  <Target Name="GetLineupPackageInfo">
+    <ItemGroup>
+      <ArtifactInfo Include="$(BuildDir)Internal.AspNetCore.Universe.Lineup.$(Version).nupkg">
+        <ArtifactType>NuGetPackage</ArtifactType>
+        <PackageId>Internal.AspNetCore.Universe.Lineup</PackageId>
+        <Version>$(Version)</Version>
+        <Category>noship</Category>
+        <IsLineup>true</IsLineup>
+      </ArtifactInfo>
+    </ItemGroup>
+  </Target>
+
+  <Target Name="GeneratePropsFiles" DependsOnTargets="GetArtifactInfo">
     <ItemGroup>
       <_LineupPackages Include="@(ExternalDependency)" />
       <_LineupPackages Include="%(ArtifactInfo.PackageId)" Version="%(ArtifactInfo.Version)" Condition=" '%(ArtifactInfo.ArtifactType)' == 'NuGetPackage' " />
-      <_LineupPackages Include="Microsoft.AspNetCore.All" Version="$(PackageVersion)" />
-      <_LineupPackages Include="Microsoft.AspNetCore.App" Version="$(PackageVersion)" />
 
       <_RestoreSources Include="$(DotNetAdditionalRestoreSources)" Condition="'$(DotNetAdditionalRestoreSources)' != ''" />
       <_RestoreSources Include="$(DependencyPackageDir)" Condition="'$(DependencyPackageDir)' != '' AND Exists('$(DependencyPackageDir)')" />
@@ -113,7 +123,7 @@
       OutputPath="$(GeneratedRestoreSourcesPropsPath)" />
 
     <PackNuSpec NuSpecPath="$(MSBuildThisFileDirectory)lineups\Internal.AspNetCore.Universe.Lineup.nuspec"
-                DestinationFolder="$(LineupBuildDir)"
+                DestinationFolder="$(BuildDir)"
                 Overwrite="true"
                 Properties="version=$(Version);dependenciesPropsFile=$(GeneratedPackageVersionPropsPath)">
       <Output TaskParameter="Packages" ItemName="LineupPackage" />

build/tasks/AnalyzeBuildGraph.cs

@@ -196,7 +196,7 @@ namespace RepoTasks
                     };
 
                     var packages = artifacts
-                        .Where(a => a.RepoName.Equals(repoName, StringComparison.OrdinalIgnoreCase))
+                        .Where(a => string.Equals(a.RepoName, repoName, StringComparison.OrdinalIgnoreCase))
                         .ToDictionary(p => p.PackageInfo.Id, p => p, StringComparer.OrdinalIgnoreCase);
 
                     foreach (var proj in s.Projects)

korebuild.json

@@ -1,4 +1,10 @@
 {
   "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/dev/tools/korebuild.schema.json",
-  "channel": "dev"
+  "channel": "dev",
+  "toolsets": {
+    "nodejs": {
+      "minVersion": "8.0",
+      "required": true
+    }
+  }
 }