From 4741da1451990be0123ca232c2344215f87a49ad Mon Sep 17 00:00:00 2001 From: Pranav K Date: Mon, 9 Mar 2020 15:21:30 -0700 Subject: [PATCH] Allow BlazorWasm benchmark driver to perform stress runs (#19619) * Allow BlazorWasm benchmark driver to perform stress runs Adds an option that allows running the driver in a loop until timeout expires --- .../Driver/BenchmarkResultsStartup.cs | 2 +- .../Wasm.Performance/Driver/Program.cs | 87 +++++++++++++------ .../Wasm.Performance/Driver/Selenium.cs | 7 +- .../benchmarkapps/Wasm.Performance/README.md | 6 ++ .../benchmarkapps/Wasm.Performance/dockerfile | 1 + .../benchmarkapps/Wasm.Performance/exec.sh | 2 +- .../Wasm.Performance/local.dockerfile | 2 + 7 files changed, 77 insertions(+), 30 deletions(-) diff --git a/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs b/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs index 326530df8e..4f4454dc89 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs +++ b/src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs @@ -29,7 +29,7 @@ namespace Wasm.Performance.Driver PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }); await context.Response.WriteAsync("OK"); - Program.SetBenchmarkResult(result); + Program.BenchmarkResultTask.TrySetResult(result); }); } } diff --git a/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs b/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs index 1d291e1fb9..c21957588a 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs +++ b/src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs @@ -2,9 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; using System.Reflection; using System.Runtime.ExceptionServices; @@ -23,52 +21,77 @@ namespace Wasm.Performance.Driver { public class Program { - static readonly TimeSpan Timeout = TimeSpan.FromMinutes(3); - static TaskCompletionSource benchmarkResult = new TaskCompletionSource(); + internal static TaskCompletionSource BenchmarkResultTask; public static async Task Main(string[] args) { - var seleniumPort = 4444; + // This cancellation token manages the timeout for the stress run. + // By default the driver executes and reports a single Benchmark run. For stress runs, + // we'll pass in the duration to execute the runs in milliseconds. This will cause this driver + // to repeat executions for the duration specified. + var stressRunCancellation = CancellationToken.None; + var isStressRun = false; if (args.Length > 0) { - if (!int.TryParse(args[0], out seleniumPort)) + if (!int.TryParse(args[0], out var stressRunSeconds)) { - Console.Error.WriteLine("Usage Driver "); + Console.Error.WriteLine("Usage Driver "); return 1; } + + if (stressRunSeconds > 0) + { + isStressRun = true; + + var stressRunDuration = TimeSpan.FromSeconds(stressRunSeconds); + Console.WriteLine($"Stress run duration: {stressRunDuration}."); + stressRunCancellation = new CancellationTokenSource(stressRunDuration).Token; + } } // This write is required for the benchmarking infrastructure. Console.WriteLine("Application started."); - var cancellationToken = new CancellationTokenSource(Timeout); - cancellationToken.Token.Register(() => benchmarkResult.TrySetException(new TimeoutException($"Timed out after {Timeout}"))); - - using var browser = await Selenium.CreateBrowser(seleniumPort, cancellationToken.Token); + using var browser = await Selenium.CreateBrowser(default); using var testApp = StartTestApp(); using var benchmarkReceiver = StartBenchmarkResultReceiver(); - var testAppUrl = GetListeningUrl(testApp); var receiverUrl = GetListeningUrl(benchmarkReceiver); - Console.WriteLine($"Test app listening at {testAppUrl}."); - var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated"; - browser.Url = launchUrl; - browser.Navigate(); + var firstRun = true; + do + { + BenchmarkResultTask = new TaskCompletionSource(); + var timeForEachRun = TimeSpan.FromMinutes(3); + using var runCancellationToken = new CancellationTokenSource(timeForEachRun); + runCancellationToken.Token.Register(() => BenchmarkResultTask.TrySetException(new TimeoutException($"Timed out after {timeForEachRun}"))); - FormatAsBenchmarksOutput(benchmarkResult.Task.Result); + if (firstRun) + { + var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated"; + browser.Url = launchUrl; + browser.Navigate(); + } + else + { + browser.FindElementById("runAll").Click(); + } + + var results = await BenchmarkResultTask.Task; + + FormatAsBenchmarksOutput(results, + includeMetadata: firstRun, + isStressRun: isStressRun); + + firstRun = false; + } while (isStressRun && !stressRunCancellation.IsCancellationRequested); Console.WriteLine("Done executing benchmark"); return 0; } - internal static void SetBenchmarkResult(BenchmarkResult result) - { - benchmarkResult.TrySetResult(result); - } - - private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult) + private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult, bool includeMetadata, bool isStressRun) { // Sample of the the format: https://github.com/aspnet/Benchmarks/blob/e55f9e0312a7dd019d1268c1a547d1863f0c7237/src/Benchmarks/Program.cs#L51-L67 var output = new BenchmarkOutput(); @@ -126,6 +149,20 @@ namespace Wasm.Performance.Driver }); } + if (!includeMetadata) + { + output.Metadata.Clear(); + } + + if (isStressRun) + { + output.Measurements.Add(new BenchmarkMeasurement + { + Timestamp = DateTime.UtcNow, + Name = "$$Delimiter$$", + }); + } + Console.WriteLine("#StartJobStatistics"); Console.WriteLine(JsonSerializer.Serialize(output)); Console.WriteLine("#EndJobStatistics"); @@ -184,9 +221,9 @@ namespace Wasm.Performance.Driver isDone.Set(); }); - if (!isDone.WaitOne(Timeout)) + if (!isDone.WaitOne(TimeSpan.FromSeconds(30))) { - throw new TimeoutException("Timed out waiting for: " + action); + throw new TimeoutException("Timed out waiting to start the host"); } if (edi != null) diff --git a/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs b/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs index 1c30e69e20..057699b72e 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs +++ b/src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -15,6 +15,7 @@ namespace Wasm.Performance.Driver { class Selenium { + const int SeleniumPort = 4444; static bool RunHeadlessBrowser = true; static bool PoolForBrowserLogs = true; @@ -55,9 +56,9 @@ namespace Wasm.Performance.Driver throw new Exception($"Unable to connect to selenium-server at {uri}"); } - public static async Task CreateBrowser(int port, CancellationToken cancellationToken) + public static async Task CreateBrowser(CancellationToken cancellationToken) { - var uri = await WaitForServerAsync(port, cancellationToken); + var uri = await WaitForServerAsync(SeleniumPort, cancellationToken); var options = new ChromeOptions(); diff --git a/src/Components/benchmarkapps/Wasm.Performance/README.md b/src/Components/benchmarkapps/Wasm.Performance/README.md index 9522ecc502..6110956bfc 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/README.md +++ b/src/Components/benchmarkapps/Wasm.Performance/README.md @@ -18,3 +18,9 @@ To run the benchmark app in the Benchmark server, run ``` dotnet run -- --config aspnetcore/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json application.endpoints --scenario blazorwasmbenchmark ``` + +If you have local changes that you'd like to benchmark, the easiest way is to push your local changes and tell the server to use your branch: + +``` +dotnet run -- --config aspnetcore/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json application.endpoints --scenario blazorwasmbenchmark --application.buildArguments "gitBranch=mylocalchanges" +``` \ No newline at end of file diff --git a/src/Components/benchmarkapps/Wasm.Performance/dockerfile b/src/Components/benchmarkapps/Wasm.Performance/dockerfile index 410556f21a..843a0eed21 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/dockerfile +++ b/src/Components/benchmarkapps/Wasm.Performance/dockerfile @@ -1,5 +1,6 @@ FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build +ENV StressRunDuration=0 ARG DEBIAN_FRONTEND=noninteractive # Setup for nodejs diff --git a/src/Components/benchmarkapps/Wasm.Performance/exec.sh b/src/Components/benchmarkapps/Wasm.Performance/exec.sh index bae38ae1e1..be913a8576 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/exec.sh +++ b/src/Components/benchmarkapps/Wasm.Performance/exec.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash /opt/bin/start-selenium-standalone.sh& -./Wasm.Performance.Driver +./Wasm.Performance.Driver $StressRunDuration diff --git a/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile b/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile index 188bc5dc5a..bccdc856f8 100644 --- a/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile +++ b/src/Components/benchmarkapps/Wasm.Performance/local.dockerfile @@ -1,5 +1,7 @@ FROM selenium/standalone-chrome:3.141.59-mercury as final +ENV StressRunDuration=0 + WORKDIR /app COPY ./Driver/bin/Release/netcoreapp3.1/linux-x64/publish ./ COPY ./exec.sh ./