Validate benchmarks (#2126)

benchmarks/Kestrel.Performance/DotSegmentRemovalBenchmark.cs

@@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class DotSegmentRemovalBenchmark
     {
         // Immutable

benchmarks/Kestrel.Performance/Http1ConnectionParsingOverheadBenchmark.cs

@@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Performance.Mocks;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class Http1ConnectionParsingOverheadBenchmark
     {
         private const int InnerLoopCount = 512;
@@ -22,20 +22,28 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
         [IterationSetup]
         public void Setup()
         {
+            var pipeFactory = new PipeFactory();
+            var pair = pipeFactory.CreateConnectionPair();
+
             var serviceContext = new ServiceContext
             {
-                HttpParserFactory = _ => NullParser<Http1ParsingHandler>.Instance,
+                ServerOptions = new KestrelServerOptions(),
-                ServerOptions = new KestrelServerOptions()
+                HttpParserFactory = f => NullParser<Http1ParsingHandler>.Instance
             };
-            var http1ConnectionContext = new Http1ConnectionContext
+
+            var http1Connection = new Http1Connection<object>(application: null, context: new Http1ConnectionContext
             {
                 ServiceContext = serviceContext,
                 ConnectionFeatures = new FeatureCollection(),
-                PipeFactory = new PipeFactory(),
+                PipeFactory = pipeFactory,
-                TimeoutControl = new MockTimeoutControl()
+                TimeoutControl = new MockTimeoutControl(),
-            };
+                Application = pair.Application,
+                Transport = pair.Transport
+            });
 
-            _http1Connection = new Http1Connection<object>(application: null, context: http1ConnectionContext);
+            http1Connection.Reset();
+
+            _http1Connection = http1Connection;
         }
 
         [Benchmark(Baseline = true, OperationsPerInvoke = InnerLoopCount)]

benchmarks/Kestrel.Performance/Http1WritingBenchmark.cs

@@ -15,7 +15,7 @@ using Microsoft.AspNetCore.Testing;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class Http1WritingBenchmark
     {
         // Standard completed task

benchmarks/Kestrel.Performance/HttpParserBenchmark.cs

@@ -8,7 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
 
     public class HttpParserBenchmark : IHttpRequestLineHandler, IHttpHeadersHandler
     {

benchmarks/Kestrel.Performance/HttpProtocolFeatureCollection.cs

@@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class HttpProtocolFeatureCollection
     {
         private readonly Http1Connection<object> _http1Connection;
@@ -77,19 +77,29 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 
         public HttpProtocolFeatureCollection()
         {
+            var pipeFactory = new PipeFactory();
+            var pair = pipeFactory.CreateConnectionPair();
+
             var serviceContext = new ServiceContext
             {
-                HttpParserFactory = _ => NullParser<Http1ParsingHandler>.Instance,
+                DateHeaderValueManager = new DateHeaderValueManager(),
-                ServerOptions = new KestrelServerOptions()
+                ServerOptions = new KestrelServerOptions(),
+                Log = new MockTrace(),
+                HttpParserFactory = f => new HttpParser<Http1ParsingHandler>()
             };
-            var http1ConnectionContext = new Http1ConnectionContext
+
+            var http1Connection = new Http1Connection<object>(application: null, context: new Http1ConnectionContext
             {
                 ServiceContext = serviceContext,
                 ConnectionFeatures = new FeatureCollection(),
-                PipeFactory = new PipeFactory()
+                PipeFactory = pipeFactory,
-            };
+                Application = pair.Application,
+                Transport = pair.Transport
+            });
 
-            _http1Connection = new Http1Connection<object>(application: null, context: http1ConnectionContext);
+            http1Connection.Reset();
+
+            _http1Connection = http1Connection;
         }
 
         [IterationSetup]

benchmarks/Kestrel.Performance/Kestrel.Performance.csproj

@@ -4,8 +4,7 @@
     <!-- Using shorter assembly name instead of Microsoft.AspNetCore.Server.Kestrel.Performance because https://github.com/dotnet/BenchmarkDotNet/issues/498 -->
     <AssemblyName>Kestrel.Performance</AssemblyName>
     <RootNamespace>Microsoft.AspNetCore.Server.Kestrel.Performance</RootNamespace>
-    <TargetFrameworks>netcoreapp2.0;net461</TargetFrameworks>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
-    <TargetFrameworks Condition=" '$(OS)' != 'Windows_NT' ">netcoreapp2.0</TargetFrameworks>
     <OutputType>Exe</OutputType>
     <ServerGarbageCollection>true</ServerGarbageCollection>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -24,6 +23,7 @@
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.10.9" NoWarn="KRB4002" />
+    <PackageReference Include="Microsoft.AspNetCore.BenchmarkRunner.Sources" PrivateAssets="All" />
   </ItemGroup>
 
 </Project>

benchmarks/Kestrel.Performance/KnownStringsBenchmark.cs

@@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class KnownStringsBenchmark
     {
         static byte[] _methodConnect = Encoding.ASCII.GetBytes("CONNECT ");

benchmarks/Kestrel.Performance/PipeThroughputBenchmark.cs

@@ -7,7 +7,7 @@ using BenchmarkDotNet.Attributes;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class PipeThroughputBenchmark
     {
         private const int _writeLenght = 57;

benchmarks/Kestrel.Performance/Program.cs

@@ -1,16 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System.Reflection;
-using BenchmarkDotNet.Running;
-
-namespace Microsoft.AspNetCore.Server.Kestrel.Performance
-{
-    public class Program
-    {
-        public static void Main(string[] args)
-        {
-            BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args);
-        }
-    }
-}

benchmarks/Kestrel.Performance/RequestParsingBenchmark.cs

@@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Performance.Mocks;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class RequestParsingBenchmark
     {
         public IPipe Pipe { get; set; }
@@ -23,23 +23,31 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
         [IterationSetup]
         public void Setup()
         {
-            PipeFactory = new PipeFactory();
+            var pipeFactory = new PipeFactory();
-            Pipe = PipeFactory.Create();
+            var pair = pipeFactory.CreateConnectionPair();
 
             var serviceContext = new ServiceContext
             {
-                HttpParserFactory = f => new HttpParser<Http1ParsingHandler>(),
+                DateHeaderValueManager = new DateHeaderValueManager(),
                 ServerOptions = new KestrelServerOptions(),
+                Log = new MockTrace(),
+                HttpParserFactory = f => new HttpParser<Http1ParsingHandler>()
             };
-            var http1ConnectionContext = new Http1ConnectionContext
+
+            var http1Connection = new Http1Connection<object>(application: null, context: new Http1ConnectionContext
             {
                 ServiceContext = serviceContext,
                 ConnectionFeatures = new FeatureCollection(),
-                PipeFactory = PipeFactory,
+                PipeFactory = pipeFactory,
+                Application = pair.Application,
+                Transport = pair.Transport,
                 TimeoutControl = new MockTimeoutControl()
-            };
+            });
 
-            Http1Connection = new Http1Connection<object>(application: null, context: http1ConnectionContext);
+            http1Connection.Reset();
+
+            Http1Connection = http1Connection;
+            Pipe = pipeFactory.Create();
         }
 
         [Benchmark(Baseline = true, OperationsPerInvoke = RequestParsingData.InnerLoopCount)]

benchmarks/Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs

@@ -14,7 +14,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class ResponseHeaderCollectionBenchmark
     {
         private const int InnerLoopCount = 512;
@@ -170,21 +170,28 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
         [IterationSetup]
         public void Setup()
         {
+            var pipeFactory = new PipeFactory();
+            var pair = pipeFactory.CreateConnectionPair();
+
             var serviceContext = new ServiceContext
             {
-                HttpParserFactory = f => new HttpParser<Http1ParsingHandler>(),
+                DateHeaderValueManager = new DateHeaderValueManager(),
-                ServerOptions = new KestrelServerOptions()
+                ServerOptions = new KestrelServerOptions(),
+                Log = new MockTrace(),
+                HttpParserFactory = f => new HttpParser<Http1ParsingHandler>()
             };
-            var http1ConnectionContext = new Http1ConnectionContext
+
+            var http1Connection = new Http1Connection<object>(application: null, context: new Http1ConnectionContext
             {
                 ServiceContext = serviceContext,
                 ConnectionFeatures = new FeatureCollection(),
-                PipeFactory = new PipeFactory()
+                PipeFactory = pipeFactory,
-            };
+                Application = pair.Application,
-
+                Transport = pair.Transport
-            var http1Connection = new Http1Connection<object>(application: null, context: http1ConnectionContext);
+            });
 
             http1Connection.Reset();
+
             _responseHeadersDirect = (HttpResponseHeaders)http1Connection.ResponseHeaders;
             var context = new DefaultHttpContext(http1Connection);
             _response = new DefaultHttpResponse(context);

benchmarks/Kestrel.Performance/ResponseHeadersWritingBenchmark.cs

@@ -16,7 +16,7 @@ using Microsoft.AspNetCore.Testing;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class ResponseHeadersWritingBenchmark
     {
         private static readonly byte[] _helloWorldPayload = Encoding.ASCII.GetBytes("Hello, World!");

benchmarks/Kestrel.Performance/StringUtilitiesBenchmark.cs

@@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
-    [Config(typeof(CoreConfig))]
+    [ParameterizedJobConfig(typeof(CoreConfig))]
     public class StringUtilitiesBenchmark
     {
         private const int Iterations = 500_000;

benchmarks/Kestrel.Performance/configs/CoreConfig.cs

@@ -12,20 +12,25 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Performance
 {
     public class CoreConfig : ManualConfig
     {
-        public CoreConfig()
+        public CoreConfig() : this(Job.Core
-        {
-            Add(JitOptimizationsValidator.FailOnError);
-            Add(MemoryDiagnoser.Default);
-            Add(StatisticColumn.OperationsPerSecond);
-
-            Add(Job.Default
-                .With(BenchmarkDotNet.Environments.Runtime.Core)
                 .WithRemoveOutliers(false)
                 .With(new GcMode() { Server = true })
                 .With(RunStrategy.Throughput)
                 .WithLaunchCount(3)
                 .WithWarmupCount(5)
-                .WithTargetCount(10));
+                .WithTargetCount(10))
+        {
+            Add(JitOptimizationsValidator.FailOnError);
+        }
+
+        public CoreConfig(Job job)
+        {
+            Add(DefaultConfig.Instance);
+
+            Add(MemoryDiagnoser.Default);
+            Add(StatisticColumn.OperationsPerSecond);
+
+            Add(job);
         }
     }
 }

build/repo.props

@@ -1,8 +1,8 @@
 <Project>
+  <PropertyGroup>
+    <EnableBenchmarkValidation>true</EnableBenchmarkValidation>
+  </PropertyGroup>
   <ItemGroup>
-    <!-- required for the custom version of csc -->
-    <DotNetCoreRuntime Include="1.1.2" />
-
     <PackageLineup Include="Internal.AspNetCore.Universe.Lineup" Version="2.1.0-*" />
     <PackageLineup Include="Internal.AspNetCore.Partners.Lineup" Version="2.1.0-*" />
   </ItemGroup>