From fff3e1ebd0bf063c764eedb641db227a6dde3a5d Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Mon, 23 Oct 2017 09:51:00 -0700 Subject: [PATCH] Validate benchmarks (#2126) --- .../DotSegmentRemovalBenchmark.cs | 2 +- ...Http1ConnectionParsingOverheadBenchmark.cs | 24 ++++++++++++------- .../Http1WritingBenchmark.cs | 2 +- .../HttpParserBenchmark.cs | 2 +- .../HttpProtocolFeatureCollection.cs | 24 +++++++++++++------ .../Kestrel.Performance.csproj | 4 ++-- .../KnownStringsBenchmark.cs | 1 + .../PipeThroughputBenchmark.cs | 2 +- benchmarks/Kestrel.Performance/Program.cs | 16 ------------- .../RequestParsingBenchmark.cs | 24 ++++++++++++------- .../ResponseHeaderCollectionBenchmark.cs | 23 +++++++++++------- .../ResponseHeadersWritingBenchmark.cs | 2 +- .../StringUtilitiesBenchmark.cs | 2 +- .../Kestrel.Performance/configs/CoreConfig.cs | 23 +++++++++++------- build/repo.props | 6 ++--- 15 files changed, 90 insertions(+), 67 deletions(-) delete mode 100644 benchmarks/Kestrel.Performance/Program.cs diff --git a/benchmarks/Kestrel.Performance/DotSegmentRemovalBenchmark.cs b/benchmarks/Kestrel.Performance/DotSegmentRemovalBenchmark.cs index 5e65f9aa03..caa43f2b58 100644 --- a/benchmarks/Kestrel.Performance/DotSegmentRemovalBenchmark.cs +++ b/benchmarks/Kestrel.Performance/DotSegmentRemovalBenchmark.cs @@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class DotSegmentRemovalBenchmark { // Immutable diff --git a/benchmarks/Kestrel.Performance/Http1ConnectionParsingOverheadBenchmark.cs b/benchmarks/Kestrel.Performance/Http1ConnectionParsingOverheadBenchmark.cs index b582379697..208d9eca15 100644 --- a/benchmarks/Kestrel.Performance/Http1ConnectionParsingOverheadBenchmark.cs +++ b/benchmarks/Kestrel.Performance/Http1ConnectionParsingOverheadBenchmark.cs @@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Performance.Mocks; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class Http1ConnectionParsingOverheadBenchmark { private const int InnerLoopCount = 512; @@ -22,20 +22,28 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance [IterationSetup] public void Setup() { + var pipeFactory = new PipeFactory(); + var pair = pipeFactory.CreateConnectionPair(); + var serviceContext = new ServiceContext { - HttpParserFactory = _ => NullParser.Instance, - ServerOptions = new KestrelServerOptions() + ServerOptions = new KestrelServerOptions(), + HttpParserFactory = f => NullParser.Instance }; - var http1ConnectionContext = new Http1ConnectionContext + + var http1Connection = new Http1Connection(application: null, context: new Http1ConnectionContext { ServiceContext = serviceContext, ConnectionFeatures = new FeatureCollection(), - PipeFactory = new PipeFactory(), - TimeoutControl = new MockTimeoutControl() - }; + PipeFactory = pipeFactory, + TimeoutControl = new MockTimeoutControl(), + Application = pair.Application, + Transport = pair.Transport + }); - _http1Connection = new Http1Connection(application: null, context: http1ConnectionContext); + http1Connection.Reset(); + + _http1Connection = http1Connection; } [Benchmark(Baseline = true, OperationsPerInvoke = InnerLoopCount)] diff --git a/benchmarks/Kestrel.Performance/Http1WritingBenchmark.cs b/benchmarks/Kestrel.Performance/Http1WritingBenchmark.cs index a1a491a7dd..c216433f50 100644 --- a/benchmarks/Kestrel.Performance/Http1WritingBenchmark.cs +++ b/benchmarks/Kestrel.Performance/Http1WritingBenchmark.cs @@ -15,7 +15,7 @@ using Microsoft.AspNetCore.Testing; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class Http1WritingBenchmark { // Standard completed task diff --git a/benchmarks/Kestrel.Performance/HttpParserBenchmark.cs b/benchmarks/Kestrel.Performance/HttpParserBenchmark.cs index 934b543594..0bf1a6fca4 100644 --- a/benchmarks/Kestrel.Performance/HttpParserBenchmark.cs +++ b/benchmarks/Kestrel.Performance/HttpParserBenchmark.cs @@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class HttpParserBenchmark : IHttpRequestLineHandler, IHttpHeadersHandler { diff --git a/benchmarks/Kestrel.Performance/HttpProtocolFeatureCollection.cs b/benchmarks/Kestrel.Performance/HttpProtocolFeatureCollection.cs index f637269c3c..f461fe470b 100644 --- a/benchmarks/Kestrel.Performance/HttpProtocolFeatureCollection.cs +++ b/benchmarks/Kestrel.Performance/HttpProtocolFeatureCollection.cs @@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class HttpProtocolFeatureCollection { private readonly Http1Connection _http1Connection; @@ -77,19 +77,29 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance public HttpProtocolFeatureCollection() { + var pipeFactory = new PipeFactory(); + var pair = pipeFactory.CreateConnectionPair(); + var serviceContext = new ServiceContext { - HttpParserFactory = _ => NullParser.Instance, - ServerOptions = new KestrelServerOptions() + DateHeaderValueManager = new DateHeaderValueManager(), + ServerOptions = new KestrelServerOptions(), + Log = new MockTrace(), + HttpParserFactory = f => new HttpParser() }; - var http1ConnectionContext = new Http1ConnectionContext + + var http1Connection = new Http1Connection(application: null, context: new Http1ConnectionContext { ServiceContext = serviceContext, ConnectionFeatures = new FeatureCollection(), - PipeFactory = new PipeFactory() - }; + PipeFactory = pipeFactory, + Application = pair.Application, + Transport = pair.Transport + }); - _http1Connection = new Http1Connection(application: null, context: http1ConnectionContext); + http1Connection.Reset(); + + _http1Connection = http1Connection; } [IterationSetup] diff --git a/benchmarks/Kestrel.Performance/Kestrel.Performance.csproj b/benchmarks/Kestrel.Performance/Kestrel.Performance.csproj index 24548de865..4797141dcb 100644 --- a/benchmarks/Kestrel.Performance/Kestrel.Performance.csproj +++ b/benchmarks/Kestrel.Performance/Kestrel.Performance.csproj @@ -4,8 +4,7 @@ Kestrel.Performance Microsoft.AspNetCore.Server.Kestrel.Performance - netcoreapp2.0;net461 - netcoreapp2.0 + netcoreapp2.0 Exe true true @@ -24,6 +23,7 @@ + diff --git a/benchmarks/Kestrel.Performance/KnownStringsBenchmark.cs b/benchmarks/Kestrel.Performance/KnownStringsBenchmark.cs index 69bf2d5adb..095d568360 100644 --- a/benchmarks/Kestrel.Performance/KnownStringsBenchmark.cs +++ b/benchmarks/Kestrel.Performance/KnownStringsBenchmark.cs @@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { + [ParameterizedJobConfig(typeof(CoreConfig))] public class KnownStringsBenchmark { static byte[] _methodConnect = Encoding.ASCII.GetBytes("CONNECT "); diff --git a/benchmarks/Kestrel.Performance/PipeThroughputBenchmark.cs b/benchmarks/Kestrel.Performance/PipeThroughputBenchmark.cs index 7af128c11d..15ea0c4cae 100644 --- a/benchmarks/Kestrel.Performance/PipeThroughputBenchmark.cs +++ b/benchmarks/Kestrel.Performance/PipeThroughputBenchmark.cs @@ -7,7 +7,7 @@ using BenchmarkDotNet.Attributes; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class PipeThroughputBenchmark { private const int _writeLenght = 57; diff --git a/benchmarks/Kestrel.Performance/Program.cs b/benchmarks/Kestrel.Performance/Program.cs deleted file mode 100644 index 2c73b19ef8..0000000000 --- a/benchmarks/Kestrel.Performance/Program.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Reflection; -using BenchmarkDotNet.Running; - -namespace Microsoft.AspNetCore.Server.Kestrel.Performance -{ - public class Program - { - public static void Main(string[] args) - { - BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args); - } - } -} diff --git a/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs b/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs index f7355007cc..7f5c44c437 100644 --- a/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs +++ b/benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs @@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Performance.Mocks; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class RequestParsingBenchmark { public IPipe Pipe { get; set; } @@ -23,23 +23,31 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance [IterationSetup] public void Setup() { - PipeFactory = new PipeFactory(); - Pipe = PipeFactory.Create(); + var pipeFactory = new PipeFactory(); + var pair = pipeFactory.CreateConnectionPair(); var serviceContext = new ServiceContext { - HttpParserFactory = f => new HttpParser(), + DateHeaderValueManager = new DateHeaderValueManager(), ServerOptions = new KestrelServerOptions(), + Log = new MockTrace(), + HttpParserFactory = f => new HttpParser() }; - var http1ConnectionContext = new Http1ConnectionContext + + var http1Connection = new Http1Connection(application: null, context: new Http1ConnectionContext { ServiceContext = serviceContext, ConnectionFeatures = new FeatureCollection(), - PipeFactory = PipeFactory, + PipeFactory = pipeFactory, + Application = pair.Application, + Transport = pair.Transport, TimeoutControl = new MockTimeoutControl() - }; + }); - Http1Connection = new Http1Connection(application: null, context: http1ConnectionContext); + http1Connection.Reset(); + + Http1Connection = http1Connection; + Pipe = pipeFactory.Create(); } [Benchmark(Baseline = true, OperationsPerInvoke = RequestParsingData.InnerLoopCount)] diff --git a/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs b/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs index b2ed0ae549..36207e4296 100644 --- a/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs +++ b/benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs @@ -14,7 +14,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class ResponseHeaderCollectionBenchmark { private const int InnerLoopCount = 512; @@ -170,21 +170,28 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance [IterationSetup] public void Setup() { + var pipeFactory = new PipeFactory(); + var pair = pipeFactory.CreateConnectionPair(); + var serviceContext = new ServiceContext { - HttpParserFactory = f => new HttpParser(), - ServerOptions = new KestrelServerOptions() + DateHeaderValueManager = new DateHeaderValueManager(), + ServerOptions = new KestrelServerOptions(), + Log = new MockTrace(), + HttpParserFactory = f => new HttpParser() }; - var http1ConnectionContext = new Http1ConnectionContext + + var http1Connection = new Http1Connection(application: null, context: new Http1ConnectionContext { ServiceContext = serviceContext, ConnectionFeatures = new FeatureCollection(), - PipeFactory = new PipeFactory() - }; - - var http1Connection = new Http1Connection(application: null, context: http1ConnectionContext); + PipeFactory = pipeFactory, + Application = pair.Application, + Transport = pair.Transport + }); http1Connection.Reset(); + _responseHeadersDirect = (HttpResponseHeaders)http1Connection.ResponseHeaders; var context = new DefaultHttpContext(http1Connection); _response = new DefaultHttpResponse(context); diff --git a/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs b/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs index 6604d7afbb..8443e13a5b 100644 --- a/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs +++ b/benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs @@ -16,7 +16,7 @@ using Microsoft.AspNetCore.Testing; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class ResponseHeadersWritingBenchmark { private static readonly byte[] _helloWorldPayload = Encoding.ASCII.GetBytes("Hello, World!"); diff --git a/benchmarks/Kestrel.Performance/StringUtilitiesBenchmark.cs b/benchmarks/Kestrel.Performance/StringUtilitiesBenchmark.cs index 8097a8b744..3d1b8aee01 100644 --- a/benchmarks/Kestrel.Performance/StringUtilitiesBenchmark.cs +++ b/benchmarks/Kestrel.Performance/StringUtilitiesBenchmark.cs @@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { - [Config(typeof(CoreConfig))] + [ParameterizedJobConfig(typeof(CoreConfig))] public class StringUtilitiesBenchmark { private const int Iterations = 500_000; diff --git a/benchmarks/Kestrel.Performance/configs/CoreConfig.cs b/benchmarks/Kestrel.Performance/configs/CoreConfig.cs index 49897ae207..98f1ab03fd 100644 --- a/benchmarks/Kestrel.Performance/configs/CoreConfig.cs +++ b/benchmarks/Kestrel.Performance/configs/CoreConfig.cs @@ -12,20 +12,25 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance { public class CoreConfig : ManualConfig { - public CoreConfig() - { - Add(JitOptimizationsValidator.FailOnError); - Add(MemoryDiagnoser.Default); - Add(StatisticColumn.OperationsPerSecond); - - Add(Job.Default - .With(BenchmarkDotNet.Environments.Runtime.Core) + public CoreConfig() : this(Job.Core .WithRemoveOutliers(false) .With(new GcMode() { Server = true }) .With(RunStrategy.Throughput) .WithLaunchCount(3) .WithWarmupCount(5) - .WithTargetCount(10)); + .WithTargetCount(10)) + { + Add(JitOptimizationsValidator.FailOnError); + } + + public CoreConfig(Job job) + { + Add(DefaultConfig.Instance); + + Add(MemoryDiagnoser.Default); + Add(StatisticColumn.OperationsPerSecond); + + Add(job); } } } diff --git a/build/repo.props b/build/repo.props index de0440bdd2..eadaa28976 100644 --- a/build/repo.props +++ b/build/repo.props @@ -1,8 +1,8 @@ + + true + - - -