huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Create compressed delta lzma for antares 

- The delta lzma is packaged as a zip
- Only new content that's not present in the LKG lzma is included
- Xmls are removed
This commit is contained in:
John Luo 2018-05-22 23:53:54 -07:00
parent cf24fb23e7
commit 25bbb466db
7 changed files with 10345 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
build/PackageArchive.targets
View File

@ -2,10 +2,18 @@
<PropertyGroup>
<_TemplatesDirectory>$(MSBuildThisFileDirectory)tools\templates\</_TemplatesDirectory>
<_WorkRoot>$(RepositoryRoot).w\</_WorkRoot>
<PreviousArchivePropsFile>$(MSBuildThisFileDirectory)PreviousArchiveManifest.props</PreviousArchivePropsFile>
<MetapackageRestoreSource>$(RepositoryRoot).deps\Signed\Packages\</MetapackageRestoreSource>
<LzmaOutputPackageName>nuGetPackagesArchive-$(PackageVersion)</LzmaOutputPackageName>
<FallbackStagingDir>$(_WorkRoot)obj\$(LzmaOutputPackageName)\</FallbackStagingDir>
<FallbackOutputDir>$(ArtifactsDir)lzma\</FallbackOutputDir>
<FallbackOutputPath>$(FallbackOutputDir)$(LzmaOutputPackageName).lzma</FallbackOutputPath>
<FallbackAntaresZipOutputPath>$(FallbackOutputDir)$(LzmaOutputPackageName)-antares.zip</FallbackAntaresZipOutputPath>
<GeneratedFallbackRestoreSourcesPropsPath>$(_WorkRoot)restoresources.$(LzmaOutputPackageName).props</GeneratedFallbackRestoreSourcesPropsPath>
</PropertyGroup>
<Import Project="$(PreviousArchivePropsFile)" Condition="Exists($(PreviousArchivePropsFile))" />
<Target Name="BuildFallbackArchive" DependsOnTargets="ResolveRepoInfo">
<!-- Clear the directories -->
<RemoveDir Directories="$(_WorkRoot)" />
@ -21,13 +29,6 @@
ExternalDependencies="@(ExternalDependency)"
MetapackageVersion="$(PackageVersion)" />
<PropertyGroup>
<FallbackStagingDir>$(_WorkRoot)obj\$(LzmaOutputPackageName)</FallbackStagingDir>
<FallbackOutputDir>$(ArtifactsDir)lzma\</FallbackOutputDir>
<FallbackOutputPath>$(FallbackOutputDir)$(LzmaOutputPackageName).lzma</FallbackOutputPath>
<GeneratedFallbackRestoreSourcesPropsPath>$(_WorkRoot)restoresources.$(LzmaOutputPackageName).props</GeneratedFallbackRestoreSourcesPropsPath>
</PropertyGroup>
<ItemGroup>
<_FallbackArchiveRestoreSources Include="$(RestoreSources)" />
<_FallbackArchiveRestoreSources Include="$(BuildDir)" Condition="Exists($(BuildDir))" />
@ -50,4 +51,75 @@
<!-- Create the archive -->
<RepoTasks.CreateLzma OutputPath="$(FallbackOutputPath)" Sources="$(FallbackStagingDir)" />
</Target>
<Target Name="UpdatePreviousArchiveManifest">
<Error Text="Please only specify one of PreviousLzmaUrl or PreviousLzmaFile but not both" Condition="'$(PreviousLzmaUrl)' != '' AND '$(PreviousLzmaFile)' != ''" />
<PropertyGroup>
<PreviousLzmaLayout>$(_WorkRoot)previous\</PreviousLzmaLayout>
<PreviousLzmaFile Condition="'$(PreviousLzmaFile)' == ''">$(RepositoryRoot).deps\nuGetPackagesArchive.previous.lzma</PreviousLzmaFile>
</PropertyGroup>
<!-- Clear the directories -->
<RemoveDir Directories="$(PreviousLzmaLayout)" />
<!-- Download and extract the previous LZMA if specified -->
<KoreBuild.Tasks.DownloadFile
Uri="$(PreviousLzmaUrl)"
DestinationPath="$(PreviousLzmaFile)"
Condition="'$(PreviousLzmaUrl)' != ''" />
<RepoTasks.ExtractLzma
InputArchive="$(PreviousLzmaFile)"
OutputPath="$(PreviousLzmaLayout)"
Condition="Exists($(PreviousLzmaFile))" />
<ItemGroup>
<PreviousLzmaContents Remove="@(PreviousLzmaContents)" />
<PreviousLzmaContents Include="$(PreviousLzmaLayout)**\*.*" />
</ItemGroup>
<!-- Create the archive -->
<RepoTasks.UpdatePreviousArchiveManifest OutputPath="$(PreviousArchivePropsFile)" Contents="@(PreviousLzmaContents)" />
</Target>
<Target Name="BuildAntaresFallbackZip">
<Error Text="Please only specify one of InputLzmaUrl or InputLzmaFile but not both" Condition="'$(InputLzmaUrl)' != '' AND '$(InputLzmaFile)' != ''" />
<PropertyGroup>
<InputLzmaLayout>$(_WorkRoot)input\</InputLzmaLayout>
<InputLzmaFile Condition="'$(InputLzmaFile)' == ''">$(RepositoryRoot).deps\nuGetPackagesArchive.input.lzma</InputLzmaFile>
</PropertyGroup>
<!-- Clear the directories -->
<RemoveDir Directories="$(InputLzmaLayout)" />
<!-- Download and extract the input LZMA if specified -->
<KoreBuild.Tasks.DownloadFile
Uri="$(InputLzmaUrl)"
DestinationPath="$(InputLzmaFile)"
Condition="'$(InputLzmaUrl)' != ''" />
<RepoTasks.ExtractLzma
InputArchive="$(InputLzmaFile)"
OutputPath="$(InputLzmaLayout)"
Condition="Exists($(InputLzmaFile))" />
<ItemGroup>
<InputLzmaContents Include="$(InputLzmaLayout)**\*.*" Exclude="$(InputLzmaLayout)**\*.nupkg" />
</ItemGroup>
<Error Text="No lzma content found. Please specify a lzma via InputLzmaUrl or InputLzmaFile or run the BuildFallbackArchive target." Condition="'@(InputLzmaContents)' == ''" />
<!-- Filter lzma content to remove existing content and .xml files -->
<RepoTasks.FilterLzmaContent PreviousLzmaContent="@(PreviousLzmaContents)" InputLzmaContent="@(InputLzmaContents)">
<Output TaskParameter="FilteredLzmaContent" ItemName="AntaresLzmaContents" />
</RepoTasks.FilterLzmaContent>
<!-- Create antares zip -->
<ZipArchive
File="$(FallbackAntaresZipOutputPath)"
SourceFiles="@(AntaresLzmaContents)"
WorkingDirectory="$(InputLzmaLayout)"
Overwrite="true"
Condition="'@(AntaresLzmaContents)' != ''"/>
</Target>
</Project>

10081
build/PreviousArchiveManifest.props Normal file
View File

File diff suppressed because it is too large Load Diff

25
build/tasks/ExtractLzma.cs Normal file
View File

@ -0,0 +1,25 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using Microsoft.DotNet.Archive;
namespace RepoTasks
{
public class ExtractLzma : Task
{
[Required]
public string InputArchive { get; set; }
[Required]
public string OutputPath { get; set; }
public override bool Execute()
{
Log.LogMessage(MessageImportance.High, $"Extracting LZMA: {InputArchive}");
new IndexedArchive().Extract(InputArchive, OutputPath, new ConsoleProgressReport());
return true;
}
}
}

90
build/tasks/FilterLzmaContent.cs Normal file
View File

@ -0,0 +1,90 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Linq;
using System.Collections.Generic;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using RepoTasks.Utilities;
namespace RepoTasks
{
public class FilterLzmaContent : Task
{
public ITaskItem[] PreviousLzmaContent { get; set; }
[Required]
public ITaskItem[] InputLzmaContent { get; set; }
[Output]
public ITaskItem[] FilteredLzmaContent { get; set; }
public override bool Execute()
{
var previousContent = new HashSet<ITaskItem>(new ITaskItemComparer());
var inputContent = new HashSet<ITaskItem>(new ITaskItemComparer());
// Keeping dlls separate for use to trim xml
var inputDlls = new HashSet<ITaskItem>(new ITaskItemComparer());
var newContent = new List<ITaskItem>();
if (PreviousLzmaContent != null)
{
foreach (var item in PreviousLzmaContent)
{
// To round trip correctly, overwrite RecursiveDir with RelativeDir
item.SetMetadata("RecursiveDir", item.GetMetadata("RelativeDir"));
previousContent.Add(item);
}
}
foreach (var item in InputLzmaContent)
{
var extension = item.GetExtension();
inputContent.Add(item);
if (string.Equals(".dll", extension, StringComparison.OrdinalIgnoreCase))
{
inputDlls.Add(item);
}
}
foreach (var item in inputContent)
{
// skip if contained in the previous LZMA
if (previousContent.Contains(item))
{
continue;
}
// skip if the file is an .xml that is matched with a .dll file
if (string.Equals(".xml", item.GetExtension(), StringComparison.OrdinalIgnoreCase)
&& inputDlls.Any(dll =>
// Match by filename
string.Equals(item.GetFileName(), dll.GetFileName(), StringComparison.OrdinalIgnoreCase)
// Match by folder structure (.xml must be under .dll's folder)
&& item.GetRecursiveDir().StartsWith(dll.GetRecursiveDir())))
{
continue;
}
newContent.Add(item);
}
FilteredLzmaContent = newContent.ToArray();
return true;
}
private class ITaskItemComparer : IEqualityComparer<ITaskItem>
{
public bool Equals(ITaskItem x, ITaskItem y)
=> string.Equals(x.GetRecursiveDir(), y.GetRecursiveDir(), StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.GetFileName(), y.GetFileName(), StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.GetExtension(), y.GetExtension(), StringComparison.OrdinalIgnoreCase);
public int GetHashCode(ITaskItem obj)
=> $"{obj.GetRecursiveDir()}{obj.GetFileName()}{obj.GetExtension()}".GetHashCode();
}
}
}

3
build/tasks/RepoTasks.tasks
View File

@ -10,6 +10,8 @@
<UsingTask TaskName="RepoTasks.CheckRepoGraph" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.CheckVersionOverrides" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.CreateLzma" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.ExtractLzma" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.FilterLzmaContent" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.GenerateRestoreSourcesPropsFile" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.GenerateSubmoduleGraph" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.JoinItems" AssemblyFile="$(_RepoTaskAssembly)" />
@ -18,6 +20,7 @@
<UsingTask TaskName="RepoTasks.PublishToAzureBlob" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.ResolveSymbolsRecursivePath" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.TrimDeps" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.UpdatePreviousArchiveManifest" AssemblyFile="$(_RepoTaskAssembly)" />
<UsingTask TaskName="RepoTasks.VerifyCoherentVersions" AssemblyFile="$(_RepoTaskAssembly)" />
<!-- tools from dotnet-buildtools -->

50
build/tasks/UpdatePreviousArchiveManifest.cs Normal file
View File

@ -0,0 +1,50 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Linq;
using System.Xml;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using NuGet.Versioning;
using RepoTasks.Utilities;
namespace RepoTasks
{
public class UpdatePreviousArchiveManifest : Task
{
[Required]
public string OutputPath { get; set; }
[Required]
public ITaskItem[] Contents { get; set; }
public override bool Execute()
{
var xmlDoc = new XmlDocument();
// Project
var projectElement = xmlDoc.CreateElement("Project");
// Items
var itemGroupElement = xmlDoc.CreateElement("ItemGroup");
foreach (var content in Contents)
{
var contentElement = xmlDoc.CreateElement("PreviousLzmaContents");
contentElement.SetAttribute("Include", $"{content.GetRecursiveDir()}{content.GetFileName()}{content.GetExtension()}");
itemGroupElement.AppendChild(contentElement);
// Recursive will be lost during round tripping using a props file. To fix this, set the RecursiveDir to RelativeDir.
// This can only be done in a task as MSBuild prevents overwritting reserved metadata.
}
projectElement.AppendChild(itemGroupElement);
// Save updated file
xmlDoc.AppendChild(projectElement);
xmlDoc.Save(OutputPath);
return true;
}
}
}

17
build/tasks/Utilities/ITaskItemExtensions.cs Normal file
View File

@ -0,0 +1,17 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.Build.Framework;
namespace RepoTasks.Utilities
{
public static class ITaskItemExtensions
{
public static string GetRecursiveDir(this ITaskItem item)
=> item.GetMetadata("RecursiveDir");
public static string GetFileName(this ITaskItem item)
=> item.GetMetadata("Filename");
public static string GetExtension(this ITaskItem item)
=> item.GetMetadata("Extension");
}
}