From 61e16e2b3108b6fc4dc9f7e690dd1e55a0cfe3b5 Mon Sep 17 00:00:00 2001 From: "Chris Ross (ASP.NET)" Date: Thu, 7 Jun 2018 12:20:22 -0700 Subject: [PATCH] Automation for h2spec functional tests #2640 --- build/dependencies.props | 69 +++--- samples/Http2SampleApp/Program.cs | 16 ++ src/Kestrel.Core/CoreStrings.resx | 3 + .../Internal/Http2/Http2Connection.cs | 2 +- .../Properties/CoreStrings.Designer.cs | 14 ++ .../Http2/H2SpecTests.cs | 135 +++++++++++ .../TestHelpers/H2SpecCommands.cs | 222 ++++++++++++++++++ ...rel.Transport.Libuv.FunctionalTests.csproj | 1 + ...l.Transport.Sockets.FunctionalTests.csproj | 1 + 9 files changed, 428 insertions(+), 35 deletions(-) create mode 100644 test/Kestrel.FunctionalTests/Http2/H2SpecTests.cs create mode 100644 test/Kestrel.FunctionalTests/TestHelpers/H2SpecCommands.cs diff --git a/build/dependencies.props b/build/dependencies.props index b31aece010..96b506f966 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -5,48 +5,49 @@ 0.10.13 - 2.2.0-preview1-34484 + 2.2.0-preview1-34492 2.2.0-preview1-17087 1.10.0 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 - 2.2.0-preview1-34484 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.2.0-preview1-34492 + 2.1.1 2.0.0 2.1.0 - 2.2.0-preview1-26614-02 - 2.2.0-preview1-34484 + 2.2.0-preview1-26618-02 + 2.2.0-preview1-34492 15.6.1 4.7.49 2.0.3 11.0.2 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 - 4.6.0-preview1-26613-07 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 + 4.6.0-preview1-26617-01 1.3.7 0.8.0 2.3.1 diff --git a/samples/Http2SampleApp/Program.cs b/samples/Http2SampleApp/Program.cs index ec75b652d1..acf22dc01c 100644 --- a/samples/Http2SampleApp/Program.cs +++ b/samples/Http2SampleApp/Program.cs @@ -31,13 +31,29 @@ namespace Http2SampleApp // Run callbacks on the transport thread options.ApplicationSchedulingMode = SchedulingMode.Inline; + // Http/1.1 endpoint for comparison options.Listen(IPAddress.Any, basePort, listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http1; + listenOptions.UseConnectionLogging(); + }); + + // TLS Http/1.1 or HTTP/2 endpoint negotiated via ALPN + options.Listen(IPAddress.Any, basePort + 1, listenOptions => { listenOptions.Protocols = HttpProtocols.Http1AndHttp2; listenOptions.UseHttps("testCert.pfx", "testPassword"); listenOptions.UseConnectionLogging(); listenOptions.ConnectionAdapters.Add(new TlsFilterAdapter()); }); + + // Prior knowledge, no TLS handshake. WARNING: Not supported by browsers + // but useful for the h2spec tests + options.Listen(IPAddress.Any, basePort + 5, listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http2; + listenOptions.UseConnectionLogging(); + }); }) .UseContentRoot(Directory.GetCurrentDirectory()) .UseStartup(); diff --git a/src/Kestrel.Core/CoreStrings.resx b/src/Kestrel.Core/CoreStrings.resx index d0448aa7d4..0530d599a8 100644 --- a/src/Kestrel.Core/CoreStrings.resx +++ b/src/Kestrel.Core/CoreStrings.resx @@ -521,4 +521,7 @@ For more information on configuring HTTPS see https://go.microsoft.com/fwlink/?l Tls 1.2 or later must be used for HTTP/2. {protocol} was negotiated. + + Invalid HTTP/2 connection preface. + \ No newline at end of file diff --git a/src/Kestrel.Core/Internal/Http2/Http2Connection.cs b/src/Kestrel.Core/Internal/Http2/Http2Connection.cs index dedac244ba..053388fb53 100644 --- a/src/Kestrel.Core/Internal/Http2/Http2Connection.cs +++ b/src/Kestrel.Core/Internal/Http2/Http2Connection.cs @@ -272,7 +272,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 { if (ClientPreface[i] != span[i]) { - throw new Exception("Invalid HTTP/2 connection preface."); + throw new Http2ConnectionErrorException(CoreStrings.Http2ErrorInvalidPreface, Http2ErrorCode.PROTOCOL_ERROR); } } diff --git a/src/Kestrel.Core/Properties/CoreStrings.Designer.cs b/src/Kestrel.Core/Properties/CoreStrings.Designer.cs index 070fe9c45d..29d9ac9bb3 100644 --- a/src/Kestrel.Core/Properties/CoreStrings.Designer.cs +++ b/src/Kestrel.Core/Properties/CoreStrings.Designer.cs @@ -1890,6 +1890,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core internal static string FormatHttp2ErrorMinTlsVersion(object protocol) => string.Format(CultureInfo.CurrentCulture, GetString("Http2ErrorMinTlsVersion", "protocol"), protocol); + /// + /// Invalid HTTP/2 connection preface. + /// + internal static string Http2ErrorInvalidPreface + { + get => GetString("Http2ErrorInvalidPreface"); + } + + /// + /// Invalid HTTP/2 connection preface. + /// + internal static string FormatHttp2ErrorInvalidPreface() + => GetString("Http2ErrorInvalidPreface"); + private static string GetString(string name, params string[] formatterNames) { var value = _resourceManager.GetString(name); diff --git a/test/Kestrel.FunctionalTests/Http2/H2SpecTests.cs b/test/Kestrel.FunctionalTests/Http2/H2SpecTests.cs new file mode 100644 index 0000000000..ca916cc4ae --- /dev/null +++ b/test/Kestrel.FunctionalTests/Http2/H2SpecTests.cs @@ -0,0 +1,135 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#if NETCOREAPP2_2 + +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Testing.xunit; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests.Http2 +{ + [OSSkipCondition(OperatingSystems.MacOSX, SkipReason = "Missing SslStream ALPN support: https://github.com/dotnet/corefx/issues/30492")] + [MinimumOSVersion(OperatingSystems.Windows, WindowsVersions.Win81, + SkipReason = "Missing Windows ALPN support: https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation#Support")] + public class H2SpecTests : TestApplicationErrorLoggerLoggedTest + { + [ConditionalTheory] + [MemberData(nameof(H2SpecTestCases))] + public async Task RunIndividualTestCase(H2SpecTestCase testCase) + { + var hostBuilder = TransportSelector.GetWebHostBuilder() + .UseKestrel(options => + { + options.Listen(IPAddress.Loopback, 0, listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http2; + if (testCase.Https) + { + listenOptions.UseHttps(TestResources.TestCertificatePath, "testPassword"); + } + }); + }) + .ConfigureServices(AddTestLogging) + .Configure(ConfigureHelloWorld); + + using (var host = hostBuilder.Build()) + { + await host.StartAsync(); + + H2SpecCommands.RunTest(testCase.Id, host.GetPort(), testCase.Https, Logger); + } + } + + public static TheoryData H2SpecTestCases + { + get + { + var dataset = new TheoryData(); + var toSkip = new[] { "hpack/4.2/1", "http2/5.1/8", "http2/6.9.1/2", "http2/6.9.1/3", "http2/8.1.2.3/1", + "http2/8.1.2.6/1", "http2/8.1.2.6/2" }; + + foreach (var testcase in H2SpecCommands.EnumerateTestCases()) + { + string skip = null; + if (toSkip.Contains(testcase.Item1)) + { + skip = "https://github.com/aspnet/KestrelHttpServer/issues/2154"; + } + + dataset.Add(new H2SpecTestCase() + { + Id = testcase.Item1, + Description = testcase.Item2, + Https = false, + Skip = skip, + }); + + dataset.Add(new H2SpecTestCase() + { + Id = testcase.Item1, + Description = testcase.Item2, + Https = true, + Skip = skip, + }); + } + + return dataset; + } + } + + public class H2SpecTestCase : IXunitSerializable + { + // For the serializer + public H2SpecTestCase() + { + } + + public string Id { get; set; } + public string Description { get; set; } + public bool Https { get; set; } + public string Skip { get; set; } + + public void Deserialize(IXunitSerializationInfo info) + { + Id = info.GetValue(nameof(Id)); + Description = info.GetValue(nameof(Description)); + Https = info.GetValue(nameof(Https)); + Skip = info.GetValue(nameof(Skip)); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(Id), Id, typeof(string)); + info.AddValue(nameof(Description), Description, typeof(string)); + info.AddValue(nameof(Https), Https, typeof(bool)); + info.AddValue(nameof(Skip), Skip, typeof(string)); + } + + public override string ToString() + { + return $"{Id}, HTTPS:{Https}, {Description}"; + } + } + + private void ConfigureHelloWorld(IApplicationBuilder app) + { + app.Run(context => + { + return context.Request.Body.CopyToAsync(context.Response.Body); + }); + } + } +} +#elif NET461 // HTTP/2 is not supported +#else +#error TFMs need updating +#endif diff --git a/test/Kestrel.FunctionalTests/TestHelpers/H2SpecCommands.cs b/test/Kestrel.FunctionalTests/TestHelpers/H2SpecCommands.cs new file mode 100644 index 0000000000..e299e9a0f2 --- /dev/null +++ b/test/Kestrel.FunctionalTests/TestHelpers/H2SpecCommands.cs @@ -0,0 +1,222 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; +using System.Xml; +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests +{ + public static class H2SpecCommands + { + private static string GetToolLocation() + { + var root = Path.Combine(Environment.CurrentDirectory, "h2spec"); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Path.Combine(root, "windows", "h2spec.exe"); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return Path.Combine(root, "linux", "h2spec"); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return Path.Combine(root, "darwin", "h2spec"); + } + throw new NotImplementedException("Invalid OS"); + } + + public static IList> EnumerateTestCases() + { + var testCases = new List>(); + var processOptions = new ProcessStartInfo() + { + FileName = GetToolLocation(), + RedirectStandardOutput = true, + Arguments = "--strict --dryrun", + WindowStyle = ProcessWindowStyle.Hidden, + CreateNoWindow = true, + }; + using (var process = Process.Start(processOptions)) + { + // https://github.com/summerwind/h2spec#running-a-specific-test-case + //Hypertext Transfer Protocol Version 2(HTTP / 2) + // 3.Starting HTTP / 2 + // 3.5.HTTP / 2 Connection Preface + // 1: Sends client connection preface + // 2: Sends invalid connection preface + //Generic tests for HTTP / 2 server + // 1.Starting HTTP / 2 + // 1: Sends a client connection preface + + // Expected output: "http2/3.5/1", "Sends client connection preface" + var groupName = string.Empty; // http2, generic, or hpack + var sectionId = string.Empty; // 3 or 3.5 + + var line = string.Empty; + while (line != null) + { + line = process.StandardOutput.ReadLine(); + + if (string.IsNullOrEmpty(line)) + { + continue; + } + + if (IsGroupLine(line, out var group)) + { + groupName = group; + continue; + } + + if (IsSectionLine(line, out var section)) + { + sectionId = section; + continue; + } + + if (IsTestLine(line, out var testNumber, out var description)) + { + testCases.Add(new Tuple($"{groupName}/{sectionId}/{testNumber}", description)); + continue; + } + + throw new InvalidOperationException("Unrecognized line: " + line); + } + } + return testCases; + } + + private static bool IsGroupLine(string line, out string groupName) + { + if (line.StartsWith(" ")) + { + groupName = null; + return false; + } + + if (line.StartsWith("Hypertext")) + { + groupName = "http2"; + return true; + } + if (line.StartsWith("Generic")) + { + groupName = "generic"; + return true; + } + if (line.StartsWith("HPACK")) + { + groupName = "hpack"; + return true; + } + throw new InvalidOperationException("Unrecognized line: " + line); + } + + // "8.1.2.1. Pseudo-Header Fields" + private static bool IsSectionLine(string line, out string section) + { + line = line.TrimStart(); + var firstSpace = line.IndexOf(" "); + if (firstSpace < 2) // Minimum: "8. description" + { + section = string.Empty; + return false; + } + + // As opposed to test cases that are marked with : + if (line[firstSpace - 1] == '.') + { + section = line.Substring(0, firstSpace - 1); // Drop the trailing dot. + return true; + } + + section = string.Empty; + return false; + } + + // "1: Sends a DATA frame" + private static bool IsTestLine(string line, out string testNumber, out string description) + { + line = line.TrimStart(); + var firstSpace = line.IndexOf(" "); + if (firstSpace < 2) // Minimum: "8: description" + { + testNumber = string.Empty; + description = string.Empty; + return false; + } + + // As opposed to test cases that are marked with : + if (line[firstSpace - 1] == ':') + { + testNumber = line.Substring(0, firstSpace - 1); // Drop the trailing colon. + description = line.Substring(firstSpace + 1); + return true; + } + + testNumber = string.Empty; + description = string.Empty; + return false; + } + + public static void RunTest(string testId, int port, bool https, ILogger logger) + { + var tempFile = Path.GetTempPath() + Guid.NewGuid() + ".xml"; + var processOptions = new ProcessStartInfo() + { + FileName = GetToolLocation(), + RedirectStandardOutput = true, + Arguments = $"{testId} -p {port.ToString(CultureInfo.InvariantCulture)} --strict -j {tempFile} --timeout 15" + + (https ? " --tls --insecure" : ""), + WindowStyle = ProcessWindowStyle.Hidden, + CreateNoWindow = true, + }; + + using (var process = Process.Start(processOptions)) + { + var data = process.StandardOutput.ReadToEnd(); + logger.LogDebug(data); + + var results = File.ReadAllText(tempFile); + File.Delete(tempFile); + + var xml = new XmlDocument(); + xml.LoadXml(results); + // + // + var foundTests = false; + var failures = new List(); + foreach (XmlNode node in xml.GetElementsByTagName("testsuite")) + { + if (node.Attributes["errors"].Value != "0") + { + // This does not list the individual sub-tests in each section + failures.Add("Test failed: " + node.Attributes["package"].Value + "; " + node.Attributes["name"].Value); + } + if (node.Attributes["tests"].Value != "0") + { + foundTests = true; + } + } + + if (failures.Count > 0) + { + throw new Exception(string.Join(Environment.NewLine, failures)); + } + + if (!foundTests) + { + logger.LogDebug(results); + throw new InvalidOperationException("No test case results found."); + } + } + } + } +} \ No newline at end of file diff --git a/test/Kestrel.Transport.Libuv.FunctionalTests/Kestrel.Transport.Libuv.FunctionalTests.csproj b/test/Kestrel.Transport.Libuv.FunctionalTests/Kestrel.Transport.Libuv.FunctionalTests.csproj index 3f87ee3ccd..8a25878b94 100644 --- a/test/Kestrel.Transport.Libuv.FunctionalTests/Kestrel.Transport.Libuv.FunctionalTests.csproj +++ b/test/Kestrel.Transport.Libuv.FunctionalTests/Kestrel.Transport.Libuv.FunctionalTests.csproj @@ -26,6 +26,7 @@ + diff --git a/test/Kestrel.Transport.Sockets.FunctionalTests/Kestrel.Transport.Sockets.FunctionalTests.csproj b/test/Kestrel.Transport.Sockets.FunctionalTests/Kestrel.Transport.Sockets.FunctionalTests.csproj index 82a8cab1e3..b57b0c3f93 100644 --- a/test/Kestrel.Transport.Sockets.FunctionalTests/Kestrel.Transport.Sockets.FunctionalTests.csproj +++ b/test/Kestrel.Transport.Sockets.FunctionalTests/Kestrel.Transport.Sockets.FunctionalTests.csproj @@ -25,6 +25,7 @@ +