Add targets to deploy all known installers to an azure blob feed

- Add PublishToAzureBlob task - Move badge/version txt file generation to publish step
2018-01-05 22:33:38 -08:00 · 2018-01-05 22:33:38 -08:00 · a54852fe6e
parent 058aeba5eb
commit a54852fe6e
8 changed files with 288 additions and 23 deletions
--- a/build/Publish.targets
+++ b/build/Publish.targets
@ -0,0 +1,135 @@
+<Project>
+
+  <PropertyGroup>
+    <DependencyAssetsDir>$(RepositoryRoot).deps\assets\</DependencyAssetsDir>
+    <!-- This file is used by the dotnet/cli to determine if our shared framework aligns with the version they pull. -->
+    <BaseRuntimeVersionFileName>aspnetcore_base_runtime.version</BaseRuntimeVersionFileName>
+    <BaseRuntimeVersionFile>$(ArtifactsDir)$(BaseRuntimeVersionFileName)</BaseRuntimeVersionFile>
+    <LatestRuntimeVersionFileName>latest.aspnetcore.version</LatestRuntimeVersionFileName>
+    <LatestRuntimeVersionFile>$(ArtifactsDir)$(LatestRuntimeVersionFileName)</LatestRuntimeVersionFile>
+  </PropertyGroup>
+
+  <Target Name="Publish"
+    DependsOnTargets="GetFilesToPublish;PublishToAzureFeed" />
+
+  <Target Name="GeneratePublishFiles" DependsOnTargets="ResolveCommitHash">
+    <MakeDir Directories="$(ArtifactsDir)" />
+
+    <!--
+      Used by the dotnet/cli build to determine which version of Microsoft.NETCore.App is used.
+    -->
+    <WriteLinesToFile File="$(BaseRuntimeVersionFile)" Lines="$(MicrosoftNETCoreApp21PackageVersion)" Overwrite="true" />
+
+    <!--
+      Used by the downloader scripts when pulling from a 'channel' instead of a specific version.
+      The second line must be the package version.
+      See dotnet-install.ps1/sh.
+    -->
+    <WriteLinesToFile
+      File="$(LatestRuntimeVersionFile)"
+      Lines="$(CommitHash);$(PackageVersion)"
+      Overwrite="true" />
+
+    <ItemGroup>
+      <SharedFxVersionBadge Include="$(ArtifactsDir)$(SharedFxInstallerName)-%(AllSharedFxRIDs.Identity)-version-badge.svg" />
+    </ItemGroup>
+
+    <GenerateSvgBadge
+      OutputPath="%(SharedFxVersionBadge.Identity)"
+      Label="version"
+      Value="$(PackageVersion)" />
+  </Target>
+
+  <Target Name="GetFilesToPublish" DependsOnTargets="GeneratePublishFiles">
+    <PropertyGroup>
+      <BlobBasePath>Runtime/$(PackageVersion)/</BlobBasePath>
+      <AliasBlobBasePath>Runtime/$(SharedFxCliBlobChannel)/</AliasBlobBasePath>
+      <PackageArchiveFileName>nuGetPackagesArchive-$(PackageVersion).lzma</PackageArchiveFileName>
+      <InstallerBaseFileName>aspnetcore-runtime-$(PackageVersion)</InstallerBaseFileName>
+      <InstallerAliasBaseFileName>aspnetcore-runtime-latest</InstallerAliasBaseFileName>
+      <SymbolsArchiveBaseFileName>aspnetcore-runtime-symbols-$(PackageVersion)</SymbolsArchiveBaseFileName>
+      <IntermediateInstallerBaseFileName>aspnetcore-runtime-internal-$(PackageVersion)</IntermediateInstallerBaseFileName>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <!-- symbols -->
+      <FilesToPublish Include="$(DependencyAssetsDir)-$(SymbolsArchiveBaseFileName)-%(RuntimeSymbolsArchive.Identity)%(RuntimeSymbolsArchive.FileExt)" Condition=" '%(RuntimeSymbolsArchive.Identity)' != '' ">
+        <RelativeBlobPath>$(BlobBasePath)$(SymbolsArchiveBaseFileName)-%(RuntimeSymbolsArchive.Identity)%(RuntimeSymbolsArchive.FileExt)</RelativeBlobPath>
+      </FilesToPublish>
+
+      <!-- Intermediate files passed on to the dotnet-CLI. -->
+      <FilesToPublish Include="$(DependencyAssetsDir)$(PackageArchiveFileName)" >
+        <RelativeBlobPath>$(BlobBasePath)$(PackageArchiveFileName)</RelativeBlobPath>
+      </FilesToPublish>
+
+      <FilesToPublish Include="$(DependencyAssetsDir)$(IntermediateInstallerBaseFileName)-%(IntermediateInstaller.Identity)%(IntermediateInstaller.FileExt)" Condition=" '%(IntermediateInstaller.Identity)' != '' ">
+        <RelativeBlobPath>$(BlobBasePath)$(IntermediateInstallerBaseFileName)-%(IntermediateInstaller.Identity)%(IntermediateInstaller.FileExt)</RelativeBlobPath>
+      </FilesToPublish>
+
+      <FilesToPublish Include="$(BaseRuntimeVersionFile)">
+        <RelativeBlobPath>$(BlobBasePath)$(BaseRuntimeVersionFileName)</RelativeBlobPath>
+        <ContentType>text/plain</ContentType>
+      </FilesToPublish>
+
+      <!-- Archive installers -->
+      <FilesToPublish Include="$(DependencyAssetsDir)$(InstallerBaseFileName)-%(NativeInstaller.Identity)%(NativeInstaller.FileExt)" Condition=" '%(NativeInstaller.FileExt)' != '' ">
+        <RelativeBlobPath>$(BlobBasePath)$(InstallerBaseFileName)-%(NativeInstaller.Identity)%(NativeInstaller.FileExt)</RelativeBlobPath>
+      </FilesToPublish>
+
+      <FilesToPublish Include="$(DependencyAssetsDir)$(InstallerBaseFileName)-%(NativeInstaller.Identity)%(NativeInstaller.FileExt)" Condition=" '%(NativeInstaller.FileExt)' != '' ">
+        <RelativeBlobPath>$(AliasBlobBasePath)$(InstallerAliasBaseFileName)-%(NativeInstaller.Identity)%(NativeInstaller.FileExt)</RelativeBlobPath>
+        <Overwrite>true</Overwrite>
+      </FilesToPublish>
+
+      <!-- Support for README badges and dotnet-install.ps1/sh -->
+      <FilesToPublish Include="@(SharedFxVersionBadge)">
+        <RelativeBlobPath>$(AliasBlobBasePath)%(SharedFxVersionBadge.FileName)%(SharedFxVersionBadge.Extension)</RelativeBlobPath>
+        <CacheControl>no-cache, no-store, must-revalidate</CacheControl>
+        <ContentType>image/svg+xml</ContentType>
+        <Overwrite>true</Overwrite>
+      </FilesToPublish>
+
+      <FilesToPublish Include="$(LatestRuntimeVersionFile)">
+        <RelativeBlobPath>$(AliasBlobBasePath)$(LatestRuntimeVersionFileName)</RelativeBlobPath>
+        <CacheControl>no-cache, no-store, must-revalidate</CacheControl>
+        <ContentType>text/plain</ContentType>
+        <Overwrite>true</Overwrite>
+      </FilesToPublish>
+    </ItemGroup>
+
+    <Message Text="Publish @(FilesToPublish -> Count()) file(s)" Importance="High" />
+    <Message Text="@(FilesToPublish -> '%(FullPath) -> %(RelativeBlobPath)','%0A')" Importance="High" />
+  </Target>
+
+  <Target Name="_CheckFilesToPublish">
+    <ItemGroup>
+      <_MissingFiles Include="%(FilesToPublish.Identity)" Condition=" ! Exists(%(FilesToPublish.Identity))" />
+    </ItemGroup>
+
+    <Error Text="Missing expected files:%0A - @(_MissingFiles, '%0A - ')" Condition="@(_MissingFiles->Count()) != 0" />
+  </Target>
+
+  <Target Name="PublishToAzureFeed"
+    DependsOnTargets="_CheckFilesToPublish"
+    Condition="'$(PublishToAzureFeed)' == 'true'">
+
+    <PropertyGroup>
+      <!--
+        Allow setting AzureBlobRelativePathBase to control the base path of all uploaded blobs.
+        AzureBlobRelativePathBase should end in a slash.
+      -->
+      <AzureBlobRelativePathBase Condition="'$(AzureBlobRelativePathBase)' != '' AND !HasTrailingSlash('$(AzureBlobRelativePathBase)')">$(AzureBlobRelativePathBase)/</AzureBlobRelativePathBase>
+    </PropertyGroup>
+
+    <ItemGroup Condition=" '$(AzureBlobRelativePathBase)' != '' ">
+      <FilesToPublish Update="@(FilesToPublish)" RelativeBlobPath="$(AzureBlobRelativePathBase)%(FilesToPublish.RelativeBlobPath)" />
+    </ItemGroup>
+
+    <RepoTasks.PublishToAzureBlob
+        AccountName="$(AzureAccountName)"
+        SharedAccessToken="$(AzureSharedAccessToken)"
+        ContainerName="$(AzureContainerName)"
+        Files="@(FilesToPublish)" />
+  </Target>
+
+</Project>
--- a/build/SharedFx.targets
+++ b/build/SharedFx.targets
@ -41,7 +41,7 @@
      <RIDIsAcceptable Condition="'%(AllSharedFxRIDs.Identity)' == '$(SharedFxRID)'">true</RIDIsAcceptable>
    </PropertyGroup>

-    <Error Text="&quot;$(SharedFxRID)&quot; not acceptable as a SharedFxRID, please specify an acceptable value: {@(_AcceptableSharedFxRIDs)}." Condition="'$(RIDIsAcceptable)' != 'true'"/>
+    <Error Text="&quot;$(SharedFxRID)&quot; not acceptable as a SharedFxRID, please specify an acceptable value: {@(AllSharedFxRIDs)}." Condition="'$(RIDIsAcceptable)' != 'true'"/>

    <!-- Clear working directory -->
    <RemoveDir Directories="$(_WorkRoot)" />
@ -134,22 +134,6 @@
      Overwrite="true" />

    <MakeDir Directories="$(SharedFxOutputPath)" />
-
-    <!--
-      Used by the downloader scripts when pulling from a 'channel' instead of a specific version.
-      The second line must be the package version.
-      See dotnet-install.ps1/sh.
-    -->
-
-    <WriteLinesToFile
-      File="$(SharedFxOutputPath)latest.aspnetcore.version"
-      Lines="@(VersionLines)"
-      Overwrite="true" />
-
-    <GenerateSvgBadge
-      OutputPath="$(SharedFxOutputPath)$(SharedFxInstallerName)-$(SharedFxRID)-version-badge.svg"
-      Label="version"
-      Value="$(PackageVersion)" />
  </Target>

  <Target Name="PrepareForCrossGen" DependsOnTargets="PrepareForSharedFx;ResolveSharedFxFiles">
--- a/build/repo.props
+++ b/build/repo.props
@ -18,6 +18,37 @@
    <DependencyMirrorPackageDir>$(RepositoryRoot).deps\mirror\</DependencyMirrorPackageDir>
  </PropertyGroup>

+  <ItemGroup>
+    <IntermediateInstaller Include="win-x86" FileExt=".zip" />
+    <IntermediateInstaller Include="win-x86" FileExt=".msi" />
+    <IntermediateInstaller Include="win-x86" FileExt=".wixlib" />
+    <IntermediateInstaller Include="win-x64" FileExt=".msi" />
+    <IntermediateInstaller Include="win-x64" FileExt=".zip" />
+    <IntermediateInstaller Include="win-x64" FileExt=".wixlib" />
+    <IntermediateInstaller Include="osx-x64" FileExt=".tar.gz" />
+    <IntermediateInstaller Include="linux-x64" FileExt=".tar.gz" />
+
+    <RuntimeSymbolsArchive Include="win-x86" FileExt=".zip" />
+    <RuntimeSymbolsArchive Include="win-x64" FileExt=".zip" />
+    <RuntimeSymbolsArchive Include="linux-x64" FileExt=".tar.gz" />
+
+    <NativeInstaller Include="win-x86" FileExt=".exe" />
+    <NativeInstaller Include="win-x86" FileExt=".zip" />
+    <NativeInstaller Include="win-x64" FileExt=".exe" />
+    <NativeInstaller Include="win-x64" FileExt=".zip" />
+    <NativeInstaller Include="osx-x64" FileExt=".tar.gz" />
+    <NativeInstaller Include="linux-x64" FileExt=".tar.gz" />
+
+    <NativeInstaller Include="debian.8" FileExt=".deb" />
+    <NativeInstaller Include="debian.9" FileExt=".deb" />
+    <NativeInstaller Include="ubuntu.14.04" FileExt=".deb" />
+    <NativeInstaller Include="ubuntu.16.04" FileExt=".deb" />
+    <NativeInstaller Include="ubuntu.17.04" FileExt=".deb" />
+
+    <NativeInstaller Include="rhel.7" FileExt=".rpm" />
+    <NativeInstaller Include="rh.rhel.7" FileExt=".rpm" />
+  </ItemGroup>
+
  <ItemGroup>
    <!-- Explicitly require the 2.0.x and 2.1.0-* version of shared runtime used by universe -->
    <DotNetCoreRuntime Include="$(MicrosoftNETCoreApp20PackageVersion)" />
@ -26,6 +57,10 @@
      FeedCredential="$(DotNetAssetRootAccessTokenSuffix)" />
  </ItemGroup>

+  <PropertyGroup>
+    <PublishToAzureFeed Condition="$(PublishType.Contains('azure'))">true</PublishToAzureFeed>
+  </PropertyGroup>
+
  <Import Project="external-dependencies.props" />
  <Import Project="artifacts.props" />
  <Import Project="submodules.props" />
--- a/build/repo.targets
+++ b/build/repo.targets
@ -5,6 +5,7 @@
  <Import Project="Templating.targets" />
  <Import Project="SharedFx.targets" />
  <Import Project="SharedFxInstaller.targets" />
+  <Import Project="Publish.targets" />

  <PropertyGroup>
    <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(SkipTests)'=='true'">/t:Package /t:VerifyPackages</_RepositoryBuildTargets>
--- a/build/tasks/PublishToAzureBlob.cs
+++ b/build/tasks/PublishToAzureBlob.cs
@ -0,0 +1,100 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace RepoTasks
+{
+    /// <summary>
+    /// Publish files to an Azure storage blob
+    /// </summary>
+    public class PublishToAzureBlob : Microsoft.Build.Utilities.Task, ICancelableTask
+    {
+        private CancellationTokenSource _cts = new CancellationTokenSource();
+
+        /// <summary>
+        /// The files to publish.
+        /// </summary>
+        [Required]
+        public ITaskItem[] Files { get; set; }
+
+        /// <summary>
+        /// The Azure blob storage account name.
+        /// </summary>
+        [Required]
+        public string AccountName { get; set; }
+
+        /// <summary>
+        /// The SAS token used to write to Azure.
+        /// </summary>
+        [Required]
+        public string SharedAccessToken { get; set; }
+
+        /// <summary>
+        /// The Azure blob storage container name
+        /// </summary>
+        [Required]
+        public string ContainerName { get; set; }
+
+        public void Cancel() => _cts.Cancel();
+
+        public override bool Execute()
+            => ExecuteAsync().Result;
+
+        private async Task<bool> ExecuteAsync()
+        {
+            var connectionString = $"BlobEndpoint=https://{AccountName}.blob.core.windows.net;SharedAccessSignature={SharedAccessToken}";
+
+            var account = CloudStorageAccount.Parse(connectionString);
+            var client = account.CreateCloudBlobClient();
+            var container = client.GetContainerReference(ContainerName);
+
+            var ctx = new OperationContext();
+
+            foreach (var item in Files)
+            {
+                // normalize slashes
+                var dest = item.GetMetadata("RelativeBlobPath")
+                    .Replace('\\', '/')
+                    .Replace("//", "/");
+                var contentType = item.GetMetadata("ContentType");
+                var cacheControl = item.GetMetadata("CacheControl");
+
+                if (string.IsNullOrEmpty(dest))
+                {
+                    Log.LogError($"Item {item.ItemSpec} is missing required metadata 'RelativeBlobPath'");
+                    return false;
+                }
+
+                var blob = container.GetBlockBlobReference(dest);
+
+                if (!string.IsNullOrEmpty(cacheControl))
+                {
+                    blob.Properties.CacheControl = cacheControl;
+                }
+
+                if (!string.IsNullOrEmpty(contentType))
+                {
+                    blob.Properties.ContentType = contentType;
+                }
+
+                Log.LogMessage(MessageImportance.High, $"Publishing {item.ItemSpec} to https://{AccountName}.blob.core.windows.net/{ContainerName}/{dest}");
+
+                var accessCondition = bool.TryParse(item.GetMetadata("Overwrite"), out var overwrite) && overwrite
+                    ? AccessCondition.GenerateEmptyCondition()
+                    : AccessCondition.GenerateIfNotExistsCondition();
+
+                await blob.UploadFromFileAsync(item.ItemSpec, accessCondition, new BlobRequestOptions(), ctx, _cts.Token);
+            }
+
+            return true;
+        }
+    }
+}
--- a/build/tasks/RepoTasks.csproj
+++ b/build/tasks/RepoTasks.csproj
@ -8,6 +8,7 @@
  <ItemGroup>
    <PackageReference Include="NuGet.Build.Tasks" Version="$(NuGetInMSBuildVersion)" PrivateAssets="All" />
    <PackageReference Include="Microsoft.Extensions.DependencyModel" Version="2.0.0" PrivateAssets="All" />
+    <PackageReference Include="WindowsAzure.Storage" Version="8.7.0" />
  </ItemGroup>

  <Import Project="$(RepoTasksSdkPath)\Sdk.targets" Condition="'$(RepoTasksSdkPath)' != '' "/>
--- a/build/tasks/RepoTasks.tasks
+++ b/build/tasks/RepoTasks.tasks
@ -3,12 +3,19 @@
    <_RepoTaskAssembly>$(MSBuildThisFileDirectory)bin\publish\RepoTasks.dll</_RepoTaskAssembly>
  </PropertyGroup>

-  <UsingTask TaskName="RepoTasks.AnalyzeBuildGraph" AssemblyFile="$(_RepoTaskAssembly)" />
-  <UsingTask TaskName="RepoTasks.CopyPackagesToSplitFolders" AssemblyFile="$(_RepoTaskAssembly)" />
-  <UsingTask TaskName="RepoTasks.GenerateRestoreSourcesPropsFile" AssemblyFile="$(_RepoTaskAssembly)" />
-  <UsingTask TaskName="RepoTasks.VerifyCoherentVersions" AssemblyFile="$(_RepoTaskAssembly)" />
-  <UsingTask TaskName="RepoTasks.AddMetapackageReferences" AssemblyFile="$(_RepoTaskAssembly)" />
  <UsingTask TaskName="RepoTasks.AddArchiveReferences" AssemblyFile="$(_RepoTaskAssembly)" />
-  <UsingTask TaskName="RepoTasks.TrimDeps" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.AddMetapackageReferences" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.AddRSReferences" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.AnalyzeBuildGraph" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ComposeNewStore" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ConsolidateManifests" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.CopyPackagesToSplitFolders" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.CreateCommonManifest" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.GenerateRestoreSourcesPropsFile" AssemblyFile="$(_RepoTaskAssembly)" />
  <UsingTask TaskName="RepoTasks.ProcessSharedFrameworkDeps" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.PublishToAzureBlob" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ReplaceInFile" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ResolveHostingStartupPackages" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.TrimDeps" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.VerifyCoherentVersions" AssemblyFile="$(_RepoTaskAssembly)" />
 </Project>
--- a/version.props
+++ b/version.props
@ -5,5 +5,7 @@
    <PackageVersion Condition="'$(IsFinalBuild)' == 'true' AND '$(VersionSuffix)' == 'rtm' ">$(VersionPrefix)</PackageVersion>
    <PackageVersion Condition="'$(IsFinalBuild)' == 'true' AND '$(VersionSuffix)' != 'rtm' ">$(VersionPrefix)-$(VersionSuffix)-final</PackageVersion>
    <VersionSuffix Condition="'$(VersionSuffix)' != '' And '$(BuildNumber)' != ''">$(VersionSuffix)-$(BuildNumber)</VersionSuffix>
+    <!-- Even though we build from 'dev', this is set to 'master' to align with the dotnet-CLI channel name for their 'dev' branch. -->
+    <SharedFxCliBlobChannel>master</SharedFxCliBlobChannel>
  </PropertyGroup>
 </Project>