From 499a3bcdc1ae9c765afdaddf74edaa50a59d537d Mon Sep 17 00:00:00 2001 From: Pranav K Date: Fri, 3 Jul 2020 17:20:17 -0700 Subject: [PATCH] Add support for gzip compression during build and publish (#23611) * Add support for gzip compression during build and publish 3.2 shipped with gzip compression during build and publish. During the port to 5.0, the build and publish pipeline was different and ended up only during brotli compression during publish. However, during build the app size is now up to 20MB. Statically compressing runtime assets during build reduces the payload size to about 8.5 MB. This should help with faster initial boot ups and perception. * Quarantine test * More quarantine --- .../integrationtests/Assert.cs | 29 ++++++++ .../Wasm/WasmBuildIncrementalismTest.cs | 33 ++++++++- .../Wasm/WasmBuildIntegrationTest.cs | 8 +++ .../Wasm/WasmCompressionTests.cs | 5 +- .../Wasm/WasmPublishIntegrationTest.cs | 19 ++++- .../src/BrotliCompress.cs | 6 +- .../src/GZipCompress.cs | 70 +++++++++++++++++++ ...soft.NET.Sdk.Razor.Components.Wasm.targets | 63 ++++++++++++++++- 8 files changed, 223 insertions(+), 10 deletions(-) create mode 100644 src/Razor/Microsoft.NET.Sdk.Razor/src/GZipCompress.cs diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Assert.cs b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Assert.cs index 6583ce89d0..27c19d0748 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Assert.cs +++ b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Assert.cs @@ -319,6 +319,22 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests return filePath; } + public static string DirectoryExists(MSBuildResult result, params string[] paths) + { + if (result == null) + { + throw new ArgumentNullException(nameof(result)); + } + + var filePath = Path.Combine(result.Project.DirectoryPath, Path.Combine(paths)); + if (!Directory.Exists(filePath)) + { + throw new DirectoryMissingException(result, filePath); + } + + return filePath; + } + public static void FileCountEquals(MSBuildResult result, int expected, string directoryPath, string searchPattern, SearchOption searchOption = SearchOption.AllDirectories) { if (result == null) @@ -820,6 +836,19 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests protected override string Heading => $"File: '{FilePath}' was not found."; } + private class DirectoryMissingException : MSBuildXunitException + { + public DirectoryMissingException(MSBuildResult result, string directoryPath) + : base(result) + { + DirectoryPath = directoryPath; + } + + public string DirectoryPath { get; } + + protected override string Heading => $"Directory: '{DirectoryPath}' was not found."; + } + private class FileCountException : MSBuildXunitException { public FileCountException(MSBuildResult result, int expected, string directoryPath, string searchPattern, string[] files) diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIncrementalismTest.cs b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIncrementalismTest.cs index 301d688e40..f91dd03863 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIncrementalismTest.cs +++ b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIncrementalismTest.cs @@ -12,7 +12,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests public class WasmBuildIncrementalismTest { [Fact] - public async Task Build_WithLinker_IsIncremental() + public async Task Build_IsIncremental() { // Arrange using var project = ProjectDirectory.Create("blazorwasm", additionalProjects: new[] { "razorclasslibrary" }); @@ -40,6 +40,37 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests } } + [Fact] + public async Task Build_GzipCompression_IsIncremental() + { + // Arrange + using var project = ProjectDirectory.Create("blazorwasm", additionalProjects: new[] { "razorclasslibrary" }); + var result = await MSBuildProcessManager.DotnetMSBuild(project); + + Assert.BuildPassed(result); + + var gzipCompressionDirectory = Path.Combine(project.IntermediateOutputDirectory, "build-gz"); + + Assert.DirectoryExists(result, gzipCompressionDirectory); + + // Act + var thumbPrint = FileThumbPrint.CreateFolderThumbprint(project, gzipCompressionDirectory); + + // Assert + for (var i = 0; i < 3; i++) + { + result = await MSBuildProcessManager.DotnetMSBuild(project); + Assert.BuildPassed(result); + + var newThumbPrint = FileThumbPrint.CreateFolderThumbprint(project, gzipCompressionDirectory); + Assert.Equal(thumbPrint.Count, newThumbPrint.Count); + for (var j = 0; j < thumbPrint.Count; j++) + { + Assert.Equal(thumbPrint[j], newThumbPrint[j]); + } + } + } + [Fact] public async Task Build_SatelliteAssembliesFileIsPreserved() { diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIntegrationTest.cs b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIntegrationTest.cs index 8788d9ce17..12fbbd0380 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIntegrationTest.cs +++ b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmBuildIntegrationTest.cs @@ -28,10 +28,14 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazor.boot.json"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazor.webassembly.js"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "dotnet.wasm"); + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "dotnet.wasm.gz"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", DotNetJsFileName); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazorwasm.dll"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "RazorClassLibrary.dll"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.Text.Json.dll"); // Verify dependencies are part of the output. + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.Text.Json.dll.gz"); // Verify dependencies are part of the output. + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.dll"); + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.dll.gz"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazorwasm.pdb"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "RazorClassLibrary.pdb"); @@ -55,10 +59,14 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazor.boot.json"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazor.webassembly.js"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "dotnet.wasm"); + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "dotnet.wasm.gz"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", DotNetJsFileName); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazorwasm.dll"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "RazorClassLibrary.dll"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.Text.Json.dll"); // Verify dependencies are part of the output. + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.Text.Json.dll.gz"); + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.dll"); + Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "System.dll.gz"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "blazorwasm.pdb"); Assert.FileExists(result, buildOutputDirectory, "wwwroot", "_framework", "RazorClassLibrary.pdb"); diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmCompressionTests.cs b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmCompressionTests.cs index e308d78274..2814308b10 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmCompressionTests.cs +++ b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmCompressionTests.cs @@ -90,7 +90,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests var buildOutputDirectory = project.BuildOutputDirectory; // Act - var compressedFilesFolder = Path.Combine("..", "blazorwasm", project.IntermediateOutputDirectory, "brotli"); + var compressedFilesFolder = Path.Combine("..", "blazorwasm", project.IntermediateOutputDirectory, "compress"); var thumbPrint = FileThumbPrint.CreateFolderThumbprint(project, compressedFilesFolder); // Assert @@ -120,7 +120,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests var buildOutputDirectory = project.BuildOutputDirectory; // Act - var compressedFilesFolder = Path.Combine("..", "blazorwasm", project.IntermediateOutputDirectory, "brotli"); + var compressedFilesFolder = Path.Combine("..", "blazorwasm", project.IntermediateOutputDirectory, "compress"); var thumbPrint = FileThumbPrint.CreateFolderThumbprint(project, compressedFilesFolder); // Assert @@ -157,6 +157,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests var extension = Path.GetExtension(file); if (extension != ".br" && extension != ".gz") { + Assert.FileExists(result, file + ".gz"); Assert.FileExists(result, file + ".br"); } } diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmPublishIntegrationTest.cs b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmPublishIntegrationTest.cs index fdcb60b864..adbe0c18f4 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmPublishIntegrationTest.cs +++ b/src/Razor/Microsoft.NET.Sdk.Razor/integrationtests/Wasm/WasmPublishIntegrationTest.cs @@ -208,7 +208,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests // Verify compression works Assert.FileExists(result, blazorPublishDirectory, "_framework", "dotnet.wasm.br"); - Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.br"); // + Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.br"); // // Verify static assets are in the publish directory Assert.FileExists(result, blazorPublishDirectory, "index.html"); @@ -311,6 +311,11 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests Assert.FileExists(result, blazorPublishDirectory, "_framework", "RazorClassLibrary.dll.br"); Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.br"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "dotnet.wasm.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "blazorwasm.dll.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "RazorClassLibrary.dll.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.gz"); + VerifyServiceWorkerFiles(result, blazorPublishDirectory, serviceWorkerPath: Path.Combine("serviceworkers", "my-service-worker.js"), serviceWorkerContent: "// This is the production service worker", @@ -352,6 +357,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests } [Fact] + [QuarantinedTest] public async Task Publish_HostedApp_WithSatelliteAssemblies() { // Arrange @@ -444,6 +450,11 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests Assert.FileExists(result, blazorPublishDirectory, "_framework", "RazorClassLibrary.dll.br"); Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.br"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "dotnet.wasm.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "blazorwasm.dll.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "RazorClassLibrary.dll.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.gz"); + VerifyServiceWorkerFiles(result, blazorPublishDirectory, serviceWorkerPath: Path.Combine("serviceworkers", "my-service-worker.js"), serviceWorkerContent: "// This is the production service worker", @@ -539,6 +550,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests // Regression test to verify satellite assemblies from the blazor app are copied to the published app's wwwroot output directory as // part of publishing in VS [Fact] + [QuarantinedTest] public async Task Publish_HostedApp_VisualStudio_WithSatelliteAssemblies() { // Simulates publishing the same way VS does by setting BuildProjectReferences=false. @@ -643,6 +655,11 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests Assert.FileExists(result, blazorPublishDirectory, "_framework", "RazorClassLibrary.dll.br"); Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.br"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "dotnet.wasm.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "blazorwasm.dll.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "RazorClassLibrary.dll.gz"); + Assert.FileExists(result, blazorPublishDirectory, "_framework", "System.Text.Json.dll.gz"); + VerifyServiceWorkerFiles(result, blazorPublishDirectory, serviceWorkerPath: Path.Combine("serviceworkers", "my-service-worker.js"), serviceWorkerContent: "// This is the production service worker", diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/src/BrotliCompress.cs b/src/Razor/Microsoft.NET.Sdk.Razor/src/BrotliCompress.cs index 9019f0520f..1ac1c343d7 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/src/BrotliCompress.cs +++ b/src/Razor/Microsoft.NET.Sdk.Razor/src/BrotliCompress.cs @@ -49,7 +49,7 @@ namespace Microsoft.AspNetCore.Razor.Tasks var input = FilesToCompress[i]; var inputFullPath = input.GetMetadata("FullPath"); var relativePath = input.GetMetadata("RelativePath"); - var outputRelativePath = Path.Combine(OutputDirectory, CalculateTargetPath(relativePath)); + var outputRelativePath = Path.Combine(OutputDirectory, CalculateTargetPath(relativePath, ".br")); var outputItem = new TaskItem(outputRelativePath); input.CopyMetadataTo(outputItem); @@ -75,7 +75,7 @@ namespace Microsoft.AspNetCore.Razor.Tasks return builder.ToString(); } - private static string CalculateTargetPath(string relativePath) + internal static string CalculateTargetPath(string relativePath, string extension) { // RelativePath can be long and if used as-is to write the output, might result in long path issues on Windows. // Instead we'll calculate a fixed length path by hashing the input file name. This uses SHA1 similar to the Hash task in MSBuild @@ -92,7 +92,7 @@ namespace Microsoft.AspNetCore.Razor.Tasks builder.Append(InvalidPathChars.Contains(c) ? '+' : c); } - builder.Append(".br"); + builder.Append(extension); return builder.ToString(); } } diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/src/GZipCompress.cs b/src/Razor/Microsoft.NET.Sdk.Razor/src/GZipCompress.cs new file mode 100644 index 0000000000..6649ee5d9c --- /dev/null +++ b/src/Razor/Microsoft.NET.Sdk.Razor/src/GZipCompress.cs @@ -0,0 +1,70 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.AspNetCore.Razor.Tasks +{ + public class GZipCompress : Task + { + [Required] + public ITaskItem[] FilesToCompress { get; set; } + + [Output] + public ITaskItem[] CompressedFiles { get; set; } + + [Required] + public string OutputDirectory { get; set; } + + public override bool Execute() + { + CompressedFiles = new ITaskItem[FilesToCompress.Length]; + + Directory.CreateDirectory(OutputDirectory); + + System.Threading.Tasks.Parallel.For(0, FilesToCompress.Length, i => + { + var file = FilesToCompress[i]; + var inputPath = file.ItemSpec; + var relativePath = file.GetMetadata("RelativePath"); + var outputRelativePath = Path.Combine( + OutputDirectory, + BrotliCompress.CalculateTargetPath(relativePath, ".gz")); + + var outputItem = new TaskItem(outputRelativePath); + outputItem.SetMetadata("RelativePath", relativePath + ".gz"); + CompressedFiles[i] = outputItem; + + if (File.Exists(outputRelativePath) && File.GetLastWriteTimeUtc(inputPath) < File.GetLastWriteTimeUtc(outputRelativePath)) + { + // Incrementalism. If input source doesn't exist or it exists and is not newer than the expected output, do nothing. + Log.LogMessage(MessageImportance.Low, $"Skipping '{inputPath}' because '{outputRelativePath}' is newer than '{inputPath}'."); + return; + } + + try + { + using var sourceStream = File.OpenRead(inputPath); + using var fileStream = File.Create(outputRelativePath); + using var stream = new GZipStream(fileStream, CompressionLevel.Optimal); + + sourceStream.CopyTo(stream); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + return; + } + }); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/src/Razor/Microsoft.NET.Sdk.Razor/src/build/netstandard2.0/Microsoft.NET.Sdk.Razor.Components.Wasm.targets b/src/Razor/Microsoft.NET.Sdk.Razor/src/build/netstandard2.0/Microsoft.NET.Sdk.Razor.Components.Wasm.targets index 9c06ac8210..37994fdc6b 100644 --- a/src/Razor/Microsoft.NET.Sdk.Razor/src/build/netstandard2.0/Microsoft.NET.Sdk.Razor.Components.Wasm.targets +++ b/src/Razor/Microsoft.NET.Sdk.Razor/src/build/netstandard2.0/Microsoft.NET.Sdk.Razor.Components.Wasm.targets @@ -15,6 +15,7 @@ Copyright (c) .NET Foundation. All rights reserved. + @@ -151,6 +152,30 @@ Copyright (c) .NET Foundation. All rights reserved. + + <_BlazorBuildGZipCompressDirectory>$(IntermediateOutputPath)build-gz\ + + + + + <_GzipFileToCompressForBuild + Include="@(ReferenceCopyLocalPaths)" + RelativePath="$(_BlazorOutputPath)%(ReferenceCopyLocalPaths.DestinationSubDirectory)%(FileName)%(Extension)" + Condition="'%(Extension)' == '.dll' or '%(ReferenceCopyLocalPaths.AssetType)' == 'native'" /> + + + + + + + + <_BlazorWriteSatelliteAssembly Include="@(_BlazorOutputWithTargetPath->HasMetadata('Culture'))" /> @@ -238,6 +263,15 @@ Copyright (c) .NET Foundation. All rights reserved. Never + <_BlazorWebAssemblyStaticWebAsset Include="@(_BlazorBuildGZipCompressedFile)"> + $(PackageId) + + $([MSBuild]::NormalizeDirectory('$(TargetDir)wwwroot\')) + $(StaticWebAssetBasePath) + $([System.String]::Copy('%(_BlazorBuildGZipCompressedFile.RelativePath)').Replace('\','/').Substring(8)) + Never + + <_ExternalStaticWebAsset Include="@(_BlazorWebAssemblyStaticWebAsset)" SourceType="Generated" /> @@ -305,6 +339,20 @@ Copyright (c) .NET Foundation. All rights reserved. + + + + + @@ -411,12 +459,12 @@ Copyright (c) .NET Foundation. All rights reserved. - <_CompressedFileOutputPath>$(IntermediateOutputPath)brotli\ + <_CompressedFileOutputPath>$(IntermediateOutputPath)compress\ <_BlazorWebAssemblyBrotliIncremental>true - <_BrotliFileToCompress + <_FileToCompress Include="@(ResolvedFileToPublish)" Condition="$([System.String]::Copy('%(ResolvedFileToPublish.RelativePath)').Replace('\','/').StartsWith('wwwroot/'))" /> @@ -427,7 +475,7 @@ Copyright (c) .NET Foundation. All rights reserved. @@ -436,8 +484,17 @@ Copyright (c) .NET Foundation. All rights reserved. + + + + + + +