From 1e62df1a2da79792092eba5dfa2f3a86a87e3c5e Mon Sep 17 00:00:00 2001 From: Pranav K Date: Thu, 9 Jan 2020 19:02:28 -0800 Subject: [PATCH] Try self-contained --- .../Driver/BenchmarkResultsStartup.cs | 6 ++- .../Wasm.Performance/Driver/Program.cs | 39 ++++++++++++++----- .../Wasm.Performance/Driver/Selenium.cs | 30 ++++++++++++-- .../benchmarkapps/Wasm.Performance/README.md | 8 ++-- .../TestApp/wwwroot/benchmarks/index.js | 5 ++- .../benchmarks/lib/minibench/minibench.js | 2 +- .../Wasm.Performance/benchmarks.compose.json | 17 +++++--- .../benchmarkapps/Wasm.Performance/dockerfile | 4 +- .../Wasm.Performance/local.dockerfile | 4 +- .../test/E2ETest/Tests/PerformanceTest.cs | 6 +-- 10 files changed, 85 insertions(+), 36 deletions(-) diff --git a/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs b/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs index 1fa1f067a0..7a4af028df 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs +++ b/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Text.Json; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -22,12 +23,13 @@ namespace Wasm.Performance.Driver { app.UseCors(); - app.Run(async request => + app.Run(async context => { - var result = await JsonSerializer.DeserializeAsync>(request.Request.Body, new JsonSerializerOptions + var result = await JsonSerializer.DeserializeAsync>(context.Request.Body, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }); + await context.Response.WriteAsync("OK"); Program.SetBenchmarkResult(result); }); } diff --git a/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs b/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs index 99ddc97420..cfaa9cef0f 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs +++ b/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.ExceptionServices; @@ -40,8 +39,10 @@ namespace Wasm.Performance.Driver // This write is required for the benchmarking infrastructure. Console.WriteLine("Application started."); - using var browser = await Selenium.CreateBrowser(seleniumPort); + var cancellationToken = new CancellationTokenSource(Timeout); + cancellationToken.Token.Register(() => benchmarkResult.TrySetException(new TimeoutException($"Timed out after {Timeout}"))); + using var browser = await Selenium.CreateBrowser(seleniumPort, cancellationToken.Token); using var testApp = StartTestApp(); using var benchmarkReceiver = StartBenchmarkResultReceiver(); @@ -54,9 +55,6 @@ namespace Wasm.Performance.Driver browser.Url = launchUrl; browser.Navigate(); - var cancellationToken = new CancellationTokenSource(Timeout); - cancellationToken.Token.Register(() => benchmarkResult.TrySetException(new TimeoutException($"Timed out after {Timeout}"))); - var results = await benchmarkResult.Task; FormatAsBenchmarksOutput(results); @@ -96,24 +94,45 @@ namespace Wasm.Performance.Driver output.Metadata.Add(new BenchmarkMetadata { Source = "BlazorWasm", - Name = "Publish size (linked)", - ShortDescription = "Publish size - linked app (MB)", - LongDescription = "Publish size - linked app (MB)", + Name = "Publish size", + ShortDescription = "Publish size (KB)", + LongDescription = "Publish size (KB)", Format = "n2", }); var testAssembly = typeof(TestApp.Startup).Assembly; + var testAssemblyLocation = new FileInfo(testAssembly.Location); var testApp = new DirectoryInfo(Path.Combine( - Path.GetDirectoryName(testAssembly.Location), + testAssemblyLocation.Directory.FullName, testAssembly.GetName().Name)); output.Measurements.Add(new BenchmarkMeasurement { Timestamp = DateTime.UtcNow, - Name = "Publish size (linked)", + Name = "Publish size", Value = GetDirectorySize(testApp) / 1024, }); + output.Metadata.Add(new BenchmarkMetadata + { + Source = "BlazorWasm", + Name = "Publish size (compressed)", + ShortDescription = "Publish size compressed app (KB)", + LongDescription = "Publish size - compressed app (KB)", + Format = "n2", + }); + + var gzip = new FileInfo(Path.Combine( + testAssemblyLocation.Directory.FullName, + $"{testAssembly.GetName().Name}.gzip")); + + output.Measurements.Add(new BenchmarkMeasurement + { + Timestamp = DateTime.UtcNow, + Name = "Publish size (compressed)", + Value = (gzip.Exists ? gzip.Length : 0) / 1024, + }); + Console.WriteLine("#StartJobStatistics"); Console.WriteLine(JsonSerializer.Serialize(output)); Console.WriteLine("#EndJobStatistics"); diff --git a/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs b/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs index 169d3735a1..1c30e69e20 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs +++ b/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs @@ -2,8 +2,10 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Linq; using System.Net; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; @@ -14,8 +16,9 @@ namespace Wasm.Performance.Driver class Selenium { static bool RunHeadlessBrowser = true; + static bool PoolForBrowserLogs = true; - private static async ValueTask WaitForServerAsync(int port) + private static async ValueTask WaitForServerAsync(int port, CancellationToken cancellationToken) { var uri = new UriBuilder("http", "localhost", port, "/wd/hub/").Uri; var httpClient = new HttpClient @@ -34,7 +37,7 @@ namespace Wasm.Performance.Driver retries++; try { - var response = (await httpClient.GetAsync("status")).EnsureSuccessStatusCode(); + var response = (await httpClient.GetAsync("status", cancellationToken)).EnsureSuccessStatusCode(); Console.WriteLine("Connected to Selenium"); return uri; } @@ -52,9 +55,9 @@ namespace Wasm.Performance.Driver throw new Exception($"Unable to connect to selenium-server at {uri}"); } - public static async Task CreateBrowser(int port) + public static async Task CreateBrowser(int port, CancellationToken cancellationToken) { - var uri = await WaitForServerAsync(port); + var uri = await WaitForServerAsync(port, cancellationToken); var options = new ChromeOptions(); @@ -82,6 +85,25 @@ namespace Wasm.Performance.Driver driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(1); + if (PoolForBrowserLogs) + { + // Run in background. + var logs = new RemoteLogs(driver); + _ = Task.Run(async () => + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(TimeSpan.FromSeconds(3)); + + var consoleLogs = logs.GetLog(LogType.Browser); + foreach (var entry in consoleLogs) + { + Console.WriteLine($"[Browser Log]: {entry.Timestamp}: {entry.Message}"); + } + } + }); + } + return driver; } catch (Exception ex) diff --git a/src/Components/benchmarkapps/Wasm.Performance/README.md b/src/Components/benchmarkapps/Wasm.Performance/README.md index ac0f59e570..9522ecc502 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/README.md +++ b/src/Components/benchmarkapps/Wasm.Performance/README.md @@ -5,7 +5,9 @@ See https://github.com/aspnet/Benchmarks#benchmarks for usage guidance on using ### Running the benchmarks -The TestApp is a regular BlazorWASM project and can be run using `dotnet run`. The Driver is an app that speaks the Benchmark protocol. You generally do not need to run the Driver locally, but if you were to do so, you require docker. Here are the commands you would need to run it locally: +The TestApp is a regular BlazorWASM project and can be run using `dotnet run`. The Driver is an app that connects against an existing Selenium server, and speaks the Benchmark protocol. You generally do not need to run the Driver locally, but if you were to do so, you can either start a selenium-server instance and run using `dotnet run []` or run it inside a Linux-based docker container. + +Here are the commands you would need to run it locally inside docker: 1. `dotnet publish -c Release -r linux-x64 Driver/Wasm.Performance.Driver.csproj` 2. `docker build -t blazor-local -f ./local.dockerfile . ` @@ -14,5 +16,5 @@ The TestApp is a regular BlazorWASM project and can be run using `dotnet run`. T To run the benchmark app in the Benchmark server, run ``` -dotnet run -- --config aspnetcore/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json --services.blazorwasmbenchmark.endpoints -``` \ No newline at end of file +dotnet run -- --config aspnetcore/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json application.endpoints --scenario blazorwasmbenchmark +``` diff --git a/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/index.js b/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/index.js index 6a1ba57f82..c1690cfac8 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/index.js +++ b/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/index.js @@ -1,7 +1,7 @@ import { groups, BenchmarkEvent, onBenchmarkEvent } from './lib/minibench/minibench.js'; import { HtmlUI } from './lib/minibench/minibench.ui.js'; -// import './appStartup.js'; -// import './renderList.js'; +import './appStartup.js'; +import './renderList.js'; import './jsonHandling.js'; new HtmlUI('E2E Performance', '#display'); @@ -21,6 +21,7 @@ if (location.href.indexOf('#automated') !== -1) { break; case BenchmarkEvent.benchmarkCompleted: case BenchmarkEvent.benchmarkError: + console.log(`Completed benchmark ${args.name}`); benchmarksResults.push(args); break; case BenchmarkEvent.runCompleted: diff --git a/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/lib/minibench/minibench.js b/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/lib/minibench/minibench.js index 6d13c56577..241721ceeb 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/lib/minibench/minibench.js +++ b/src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/lib/minibench/minibench.js @@ -302,7 +302,7 @@ class Group extends EventEmitter { } const groups = []; -let reportBenchmarkEvent; +let reportBenchmarkEvent = () => {}; function group(name, configure) { groups.push(new Group(name)); diff --git a/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json b/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json index 9c1aceac94..81607364dc 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json +++ b/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json @@ -1,16 +1,21 @@ { - "dependencies": [ - "blazorwasmbenchmark" - ], - "services": { + "$schema": "https://raw.githubusercontent.com/aspnet/Benchmarks/master/src/BenchmarksDriver2/benchmarks.schema.json", + "scenarios": { + "blazorwasmbenchmark": { + "application": { + "job": "blazorwasmbenchmark" + } + } + }, + "jobs": { "blazorwasmbenchmark": { "source": { "repository": "https://github.com/dotnet/AspNetCore.git", - "branchOrCommit": "prkrishn/blazor-benchmarking", + "branchOrCommit": "blazor-wasm", "dockerfile": "src/Components/benchmarkapps/Wasm.Performance/dockerfile" }, "waitForExit": true, "readyStateText": "Application started." } } -} \ No newline at end of file +} diff --git a/src/Components/benchmarkapps/Wasm.Performance/dockerfile b/src/Components/benchmarkapps/Wasm.Performance/dockerfile index 126c172b9a..69f27a9212 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/dockerfile +++ b/src/Components/benchmarkapps/Wasm.Performance/dockerfile @@ -11,7 +11,7 @@ RUN apt-get update \ nodejs \ git -ARG gitBranch=prkrishn/blazor-benchmarking +ARG gitBranch=blazor-wasm WORKDIR /src ADD https://api.github.com/repos/dotnet/aspnetcore/git/ref/heads/${gitBranch} /aspnetcore.commit @@ -29,4 +29,4 @@ FROM selenium/standalone-chrome:3.141.59-mercury as final COPY --from=build ./app ./ COPY ./exec.sh ./ -ENTRYPOINT [ "bash", "./exec.sh" ] \ No newline at end of file +ENTRYPOINT [ "bash", "./exec.sh" ] diff --git a/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile b/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile index b5c7578be8..188bc5dc5a 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile +++ b/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile @@ -2,6 +2,6 @@ FROM selenium/standalone-chrome:3.141.59-mercury as final WORKDIR /app COPY ./Driver/bin/Release/netcoreapp3.1/linux-x64/publish ./ -COPY ./exec.sh ./exec.sh +COPY ./exec.sh ./ -ENTRYPOINT [ "bash", "./exec.sh" ] \ No newline at end of file +ENTRYPOINT [ "bash", "./exec.sh" ] diff --git a/src/Components/test/E2ETest/Tests/PerformanceTest.cs b/src/Components/test/E2ETest/Tests/PerformanceTest.cs index d5cb01b77c..f7187a4557 100644 --- a/src/Components/test/E2ETest/Tests/PerformanceTest.cs +++ b/src/Components/test/E2ETest/Tests/PerformanceTest.cs @@ -52,10 +52,8 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests () => runAllButton.Displayed || Browser.FindElements(By.CssSelector(".benchmark-error")).Any(), TimeSpan.FromSeconds(60)); - var finishedBenchmarks = Browser.FindElements(By.CssSelector(".benchmark-idle")); - var failedBenchmarks = Browser.FindElements(By.CssSelector(".benchmark-error")); - Assert.NotEmpty(finishedBenchmarks); - Assert.Empty(failedBenchmarks); + Browser.DoesNotExist(By.CssSelector(".benchmark-error")); // no failures + Browser.Exists(By.CssSelector(".benchmark-idle")); // everything's done } } }