Create compressed delta lzma for antares

- The delta lzma is packaged as a zip
- Only new content that's not present in the LKG lzma is included
- Xmls are removed

build/PackageArchive.targets

@@ -2,10 +2,18 @@
   <PropertyGroup>
     <_TemplatesDirectory>$(MSBuildThisFileDirectory)tools\templates\</_TemplatesDirectory>
     <_WorkRoot>$(RepositoryRoot).w\</_WorkRoot>
+    <PreviousArchivePropsFile>$(MSBuildThisFileDirectory)PreviousArchiveManifest.props</PreviousArchivePropsFile>
     <MetapackageRestoreSource>$(RepositoryRoot).deps\Signed\Packages\</MetapackageRestoreSource>
     <LzmaOutputPackageName>nuGetPackagesArchive-$(PackageVersion)</LzmaOutputPackageName>
+    <FallbackStagingDir>$(_WorkRoot)obj\$(LzmaOutputPackageName)\</FallbackStagingDir>
+    <FallbackOutputDir>$(ArtifactsDir)lzma\</FallbackOutputDir>
+    <FallbackOutputPath>$(FallbackOutputDir)$(LzmaOutputPackageName).lzma</FallbackOutputPath>
+    <FallbackAntaresZipOutputPath>$(FallbackOutputDir)$(LzmaOutputPackageName)-antares.zip</FallbackAntaresZipOutputPath>
+    <GeneratedFallbackRestoreSourcesPropsPath>$(_WorkRoot)restoresources.$(LzmaOutputPackageName).props</GeneratedFallbackRestoreSourcesPropsPath>
   </PropertyGroup>
 
+  <Import Project="$(PreviousArchivePropsFile)" Condition="Exists($(PreviousArchivePropsFile))" />
+
   <Target Name="BuildFallbackArchive" DependsOnTargets="ResolveRepoInfo">
     <!-- Clear the directories -->
     <RemoveDir Directories="$(_WorkRoot)" />
@@ -21,13 +29,6 @@
       ExternalDependencies="@(ExternalDependency)"
       MetapackageVersion="$(PackageVersion)" />
 
-    <PropertyGroup>
-      <FallbackStagingDir>$(_WorkRoot)obj\$(LzmaOutputPackageName)</FallbackStagingDir>
-      <FallbackOutputDir>$(ArtifactsDir)lzma\</FallbackOutputDir>
-      <FallbackOutputPath>$(FallbackOutputDir)$(LzmaOutputPackageName).lzma</FallbackOutputPath>
-      <GeneratedFallbackRestoreSourcesPropsPath>$(_WorkRoot)restoresources.$(LzmaOutputPackageName).props</GeneratedFallbackRestoreSourcesPropsPath>
-    </PropertyGroup>
-
     <ItemGroup>
       <_FallbackArchiveRestoreSources Include="$(RestoreSources)" />
       <_FallbackArchiveRestoreSources Include="$(BuildDir)" Condition="Exists($(BuildDir))" />
@@ -50,4 +51,75 @@
     <!-- Create the archive -->
     <RepoTasks.CreateLzma OutputPath="$(FallbackOutputPath)" Sources="$(FallbackStagingDir)" />
   </Target>
+
+  <Target Name="UpdatePreviousArchiveManifest">
+    <Error Text="Please only specify one of PreviousLzmaUrl or PreviousLzmaFile but not both" Condition="'$(PreviousLzmaUrl)' != '' AND '$(PreviousLzmaFile)' != ''" />
+
+    <PropertyGroup>
+      <PreviousLzmaLayout>$(_WorkRoot)previous\</PreviousLzmaLayout>
+      <PreviousLzmaFile Condition="'$(PreviousLzmaFile)' == ''">$(RepositoryRoot).deps\nuGetPackagesArchive.previous.lzma</PreviousLzmaFile>
+    </PropertyGroup>
+
+    <!-- Clear the directories -->
+    <RemoveDir Directories="$(PreviousLzmaLayout)" />
+
+    <!-- Download and extract the previous LZMA if specified -->
+    <KoreBuild.Tasks.DownloadFile
+      Uri="$(PreviousLzmaUrl)"
+      DestinationPath="$(PreviousLzmaFile)"
+      Condition="'$(PreviousLzmaUrl)' != ''" />
+    <RepoTasks.ExtractLzma
+      InputArchive="$(PreviousLzmaFile)"
+      OutputPath="$(PreviousLzmaLayout)"
+      Condition="Exists($(PreviousLzmaFile))" />
+
+    <ItemGroup>
+      <PreviousLzmaContents Remove="@(PreviousLzmaContents)" />
+      <PreviousLzmaContents Include="$(PreviousLzmaLayout)**\*.*" />
+    </ItemGroup>
+
+    <!-- Create the archive -->
+    <RepoTasks.UpdatePreviousArchiveManifest OutputPath="$(PreviousArchivePropsFile)" Contents="@(PreviousLzmaContents)" />
+  </Target>
+
+  <Target Name="BuildAntaresFallbackZip">
+    <Error Text="Please only specify one of InputLzmaUrl or InputLzmaFile but not both" Condition="'$(InputLzmaUrl)' != '' AND '$(InputLzmaFile)' != ''" />
+
+    <PropertyGroup>
+      <InputLzmaLayout>$(_WorkRoot)input\</InputLzmaLayout>
+      <InputLzmaFile Condition="'$(InputLzmaFile)' == ''">$(RepositoryRoot).deps\nuGetPackagesArchive.input.lzma</InputLzmaFile>
+    </PropertyGroup>
+
+    <!-- Clear the directories -->
+    <RemoveDir Directories="$(InputLzmaLayout)" />
+
+    <!-- Download and extract the input LZMA if specified -->
+    <KoreBuild.Tasks.DownloadFile
+      Uri="$(InputLzmaUrl)"
+      DestinationPath="$(InputLzmaFile)"
+      Condition="'$(InputLzmaUrl)' != ''" />
+    <RepoTasks.ExtractLzma
+      InputArchive="$(InputLzmaFile)"
+      OutputPath="$(InputLzmaLayout)"
+      Condition="Exists($(InputLzmaFile))" />
+
+    <ItemGroup>
+      <InputLzmaContents Include="$(InputLzmaLayout)**\*.*"  Exclude="$(InputLzmaLayout)**\*.nupkg" />
+    </ItemGroup>
+
+    <Error Text="No lzma content found. Please specify a lzma via InputLzmaUrl or InputLzmaFile or run the BuildFallbackArchive target." Condition="'@(InputLzmaContents)' == ''" />
+
+    <!-- Filter lzma content to remove existing content and .xml files -->
+    <RepoTasks.FilterLzmaContent PreviousLzmaContent="@(PreviousLzmaContents)" InputLzmaContent="@(InputLzmaContents)">
+      <Output TaskParameter="FilteredLzmaContent" ItemName="AntaresLzmaContents" />
+    </RepoTasks.FilterLzmaContent>
+
+    <!-- Create antares zip -->
+    <ZipArchive
+      File="$(FallbackAntaresZipOutputPath)"
+      SourceFiles="@(AntaresLzmaContents)"
+      WorkingDirectory="$(InputLzmaLayout)"
+      Overwrite="true"
+      Condition="'@(AntaresLzmaContents)' != ''"/>
+  </Target>
 </Project>

build/tasks/ExtractLzma.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Microsoft.DotNet.Archive;
+
+namespace RepoTasks
+{
+    public class ExtractLzma : Task
+    {
+        [Required]
+        public string InputArchive { get; set; }
+
+        [Required]
+        public string OutputPath { get; set; }
+
+        public override bool Execute()
+        {
+            Log.LogMessage(MessageImportance.High, $"Extracting LZMA: {InputArchive}");
+            new IndexedArchive().Extract(InputArchive, OutputPath, new ConsoleProgressReport());
+            return true;
+        }
+    }
+}

build/tasks/FilterLzmaContent.cs

@@ -0,0 +1,90 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using RepoTasks.Utilities;
+
+namespace RepoTasks
+{
+    public class FilterLzmaContent : Task
+    {
+        public ITaskItem[] PreviousLzmaContent { get; set; }
+
+        [Required]
+        public ITaskItem[] InputLzmaContent { get; set; }
+
+        [Output]
+        public ITaskItem[] FilteredLzmaContent { get; set; }
+
+
+        public override bool Execute()
+        {
+            var previousContent = new HashSet<ITaskItem>(new ITaskItemComparer());
+            var inputContent = new HashSet<ITaskItem>(new ITaskItemComparer());
+            // Keeping dlls separate for use to trim xml
+            var inputDlls = new HashSet<ITaskItem>(new ITaskItemComparer());
+            var newContent = new List<ITaskItem>();
+
+            if (PreviousLzmaContent != null)
+            {
+                foreach (var item in PreviousLzmaContent)
+                {
+                    // To round trip correctly, overwrite RecursiveDir with RelativeDir
+                    item.SetMetadata("RecursiveDir", item.GetMetadata("RelativeDir"));
+                    previousContent.Add(item);
+                }
+            }
+            foreach (var item in InputLzmaContent)
+            {
+                var extension = item.GetExtension();
+
+                inputContent.Add(item);
+                if (string.Equals(".dll", extension, StringComparison.OrdinalIgnoreCase))
+                {
+                    inputDlls.Add(item);
+                }
+            }
+
+            foreach (var item in inputContent)
+            {
+                // skip if contained in the previous LZMA
+                if (previousContent.Contains(item))
+                {
+                    continue;
+                }
+
+                // skip if the file is an .xml that is matched with a .dll file
+                if (string.Equals(".xml", item.GetExtension(), StringComparison.OrdinalIgnoreCase)
+                    && inputDlls.Any(dll =>
+                        // Match by filename
+                        string.Equals(item.GetFileName(), dll.GetFileName(), StringComparison.OrdinalIgnoreCase)
+                        // Match by folder structure (.xml must be under .dll's folder)
+                        && item.GetRecursiveDir().StartsWith(dll.GetRecursiveDir())))
+                {
+                    continue;
+                }
+
+                newContent.Add(item);
+            }
+
+            FilteredLzmaContent = newContent.ToArray();
+
+            return true;
+        }
+
+        private class ITaskItemComparer : IEqualityComparer<ITaskItem>
+        {
+            public bool Equals(ITaskItem x, ITaskItem y)
+                => string.Equals(x.GetRecursiveDir(), y.GetRecursiveDir(), StringComparison.OrdinalIgnoreCase)
+                    && string.Equals(x.GetFileName(), y.GetFileName(), StringComparison.OrdinalIgnoreCase)
+                    && string.Equals(x.GetExtension(), y.GetExtension(), StringComparison.OrdinalIgnoreCase);
+
+            public int GetHashCode(ITaskItem obj)
+                => $"{obj.GetRecursiveDir()}{obj.GetFileName()}{obj.GetExtension()}".GetHashCode();
+        }
+    }
+}

build/tasks/RepoTasks.tasks

@@ -10,6 +10,8 @@
   <UsingTask TaskName="RepoTasks.CheckRepoGraph" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.CheckVersionOverrides" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.CreateLzma" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.ExtractLzma" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.FilterLzmaContent" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.GenerateRestoreSourcesPropsFile" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.GenerateSubmoduleGraph" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.JoinItems" AssemblyFile="$(_RepoTaskAssembly)" />
@@ -18,6 +20,7 @@
   <UsingTask TaskName="RepoTasks.PublishToAzureBlob" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.ResolveSymbolsRecursivePath" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.TrimDeps" AssemblyFile="$(_RepoTaskAssembly)" />
+  <UsingTask TaskName="RepoTasks.UpdatePreviousArchiveManifest" AssemblyFile="$(_RepoTaskAssembly)" />
   <UsingTask TaskName="RepoTasks.VerifyCoherentVersions" AssemblyFile="$(_RepoTaskAssembly)" />
 
   <!-- tools from dotnet-buildtools -->

build/tasks/UpdatePreviousArchiveManifest.cs

@@ -0,0 +1,50 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Xml;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using NuGet.Versioning;
+using RepoTasks.Utilities;
+
+namespace RepoTasks
+{
+    public class UpdatePreviousArchiveManifest : Task
+    {
+        [Required]
+        public string OutputPath { get; set; }
+
+        [Required]
+        public ITaskItem[] Contents { get; set; }
+
+        public override bool Execute()
+        {
+            var xmlDoc = new XmlDocument();
+
+            // Project
+            var projectElement = xmlDoc.CreateElement("Project");
+
+            // Items
+            var itemGroupElement = xmlDoc.CreateElement("ItemGroup");
+
+            foreach (var content in Contents)
+            {
+                var contentElement = xmlDoc.CreateElement("PreviousLzmaContents");
+                contentElement.SetAttribute("Include", $"{content.GetRecursiveDir()}{content.GetFileName()}{content.GetExtension()}");
+                itemGroupElement.AppendChild(contentElement);
+                // Recursive will be lost during round tripping using a props file. To fix this, set the RecursiveDir to RelativeDir.
+                // This can only be done in a task as MSBuild prevents overwritting reserved metadata.
+            }
+
+            projectElement.AppendChild(itemGroupElement);
+
+            // Save updated file
+            xmlDoc.AppendChild(projectElement);
+            xmlDoc.Save(OutputPath);
+
+            return true;
+        }
+    }
+}

build/tasks/Utilities/ITaskItemExtensions.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Build.Framework;
+
+namespace RepoTasks.Utilities
+{
+    public static class ITaskItemExtensions
+    {
+        public static string GetRecursiveDir(this ITaskItem item)
+            => item.GetMetadata("RecursiveDir");
+        public static string GetFileName(this ITaskItem item)
+            => item.GetMetadata("Filename");
+        public static string GetExtension(this ITaskItem item)
+            => item.GetMetadata("Extension");
+    }
+}