diff --git a/src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj b/src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj index 06d5ad8b82..d8f520d0d9 100644 --- a/src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj +++ b/src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Servers/IIS/IIS/src/Core/IISHttpContext.cs b/src/Servers/IIS/IIS/src/Core/IISHttpContext.cs index 3f7d3c53ce..cb34c49db3 100644 --- a/src/Servers/IIS/IIS/src/Core/IISHttpContext.cs +++ b/src/Servers/IIS/IIS/src/Core/IISHttpContext.cs @@ -74,8 +74,9 @@ namespace Microsoft.AspNetCore.Server.IIS.Core IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, - ILogger logger) - : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.HttpGetRawRequest(pInProcessHandler)) + ILogger logger, + bool useLatin1) + : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.HttpGetRawRequest(pInProcessHandler), useLatin1: useLatin1) { _memoryPool = memoryPool; _pInProcessHandler = pInProcessHandler; diff --git a/src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs b/src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs index 372eadfa71..635ed097ea 100644 --- a/src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs +++ b/src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs @@ -3,6 +3,7 @@ using System; using System.Buffers; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; @@ -15,8 +16,8 @@ namespace Microsoft.AspNetCore.Server.IIS.Core { private readonly IHttpApplication _application; - public IISHttpContextOfT(MemoryPool memoryPool, IHttpApplication application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger) - : base(memoryPool, pInProcessHandler, options, server, logger) + public IISHttpContextOfT(MemoryPool memoryPool, IHttpApplication application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger, bool useLatin1) + : base(memoryPool, pInProcessHandler, options, server, logger, useLatin1) { _application = application; } diff --git a/src/Servers/IIS/IIS/src/Core/IISHttpServer.cs b/src/Servers/IIS/IIS/src/Core/IISHttpServer.cs index ef4c3aabb8..7724f18300 100644 --- a/src/Servers/IIS/IIS/src/Core/IISHttpServer.cs +++ b/src/Servers/IIS/IIS/src/Core/IISHttpServer.cs @@ -214,11 +214,14 @@ namespace Microsoft.AspNetCore.Server.IIS.Core private class IISContextFactory : IISContextFactory { + private const string Latin1Suppport = "Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders"; + private readonly IHttpApplication _application; private readonly MemoryPool _memoryPool; private readonly IISServerOptions _options; private readonly IISHttpServer _server; private readonly ILogger _logger; + private readonly bool _useLatin1; public IISContextFactory(MemoryPool memoryPool, IHttpApplication application, IISServerOptions options, IISHttpServer server, ILogger logger) { @@ -227,11 +230,12 @@ namespace Microsoft.AspNetCore.Server.IIS.Core _options = options; _server = server; _logger = logger; + AppContext.TryGetSwitch(Latin1Suppport, out _useLatin1); } public IISHttpContext CreateHttpContext(IntPtr pInProcessHandler) { - return new IISHttpContextOfT(_memoryPool, _application, pInProcessHandler, _options, _server, _logger); + return new IISHttpContextOfT(_memoryPool, _application, pInProcessHandler, _options, _server, _logger, _useLatin1); } } } diff --git a/src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj b/src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj index c645b12741..4f17fcae7a 100644 --- a/src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj +++ b/src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj @@ -19,6 +19,7 @@ + diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/Latin1Tests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/Latin1Tests.cs new file mode 100644 index 0000000000..646b9c9bd9 --- /dev/null +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/Latin1Tests.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Server.IIS.FunctionalTests.Utilities; +using Microsoft.AspNetCore.Server.IntegrationTesting; +using Microsoft.AspNetCore.Server.IntegrationTesting.IIS; +using Microsoft.AspNetCore.Testing; +using Xunit; + +namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess +{ + [Collection(PublishedSitesCollection.Name)] + public class Latin1Tests : IISFunctionalTestBase + { + public Latin1Tests(PublishedSitesFixture fixture) : base(fixture) + { + } + + [ConditionalFact] + [RequiresNewHandler] + public async Task Latin1Works() + { + var deploymentParameters = Fixture.GetBaseDeploymentParameters(); + deploymentParameters.TransformArguments((a, _) => $"{a} AddLatin1"); + + var deploymentResult = await DeployAsync(deploymentParameters); + + var client = new HttpClient(new LoggingHandler(new WinHttpHandler() { SendTimeout = TimeSpan.FromMinutes(3) }, deploymentResult.Logger)); + + var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{deploymentResult.ApplicationBaseUri}Latin1"); + requestMessage.Headers.Add("foo", "£"); + + var result = await client.SendAsync(requestMessage); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + } + + [ConditionalFact] + [RequiresNewHandler] + public async Task Latin1ReplacedWithoutAppContextSwitch() + { + var deploymentParameters = Fixture.GetBaseDeploymentParameters(); + deploymentParameters.TransformArguments((a, _) => $"{a}"); + + var deploymentResult = await DeployAsync(deploymentParameters); + + var client = new HttpClient(new LoggingHandler(new WinHttpHandler() { SendTimeout = TimeSpan.FromMinutes(3) }, deploymentResult.Logger)); + + var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{deploymentResult.ApplicationBaseUri}InvalidCharacter"); + requestMessage.Headers.Add("foo", "£"); + + var result = await client.SendAsync(requestMessage); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + } + + [ConditionalFact] + [RequiresNewHandler] + public async Task Latin1InvalidCharacters_HttpSysRejects() + { + var deploymentParameters = Fixture.GetBaseDeploymentParameters(); + deploymentParameters.TransformArguments((a, _) => $"{a} AddLatin1"); + + var deploymentResult = await DeployAsync(deploymentParameters); + + using (var connection = new TestConnection(deploymentResult.HttpClient.BaseAddress.Port)) + { + await connection.Send( + "GET /ReadAndFlushEcho HTTP/1.1", + "Host: localhost", + "Connection: close", + "foo: £\0a", + "", + ""); + + await connection.ReceiveStartsWith("HTTP/1.1 400 Bad Request"); + } + } + } +} diff --git a/src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj index a7858c63b2..202138392d 100644 --- a/src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj +++ b/src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj @@ -1,4 +1,4 @@ - + @@ -27,6 +27,7 @@ + diff --git a/src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj index 9a74ecd31d..7c4f552cc3 100644 --- a/src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj +++ b/src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj @@ -1,4 +1,4 @@ - + $(DefaultNetCoreTargetFramework) @@ -34,6 +34,7 @@ + diff --git a/src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj index 03adce1d09..56f95d8ec1 100644 --- a/src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj +++ b/src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj @@ -1,4 +1,4 @@ - + $(DefaultNetCoreTargetFramework) @@ -28,6 +28,7 @@ + diff --git a/src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj b/src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj index 23a5ecdf71..8615b70c3d 100644 --- a/src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj +++ b/src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj @@ -1,4 +1,4 @@ - + @@ -30,6 +30,7 @@ + diff --git a/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs b/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs index 4b9889a141..0132efc2e7 100644 --- a/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs +++ b/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs @@ -145,9 +145,9 @@ namespace TestSite } return 0; +#if !FORWARDCOMPAT case "ThrowInStartupGenericHost": { -#if !FORWARDCOMPAT var host = new HostBuilder().ConfigureWebHost((c) => { c.ConfigureLogging((_, factory) => @@ -161,9 +161,26 @@ namespace TestSite host.Build().Run(); -#endif + return 0; } - return 0; + case "AddLatin1": + { + AppContext.SetSwitch("Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders", isEnabled: true); + var host = new HostBuilder().ConfigureWebHost((c) => + { + c.ConfigureLogging((_, factory) => + { + factory.AddConsole(); + factory.AddFilter("Console", level => level >= LogLevel.Information); + }) + .UseIIS() + .UseStartup(); + }); + + host.Build().Run(); + return 0; + } +#endif default: return StartServer(); diff --git a/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs b/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs index 7336dda665..ef9f8758b5 100644 --- a/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs +++ b/src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs @@ -1005,5 +1005,19 @@ namespace TestSite await context.Response.WriteAsync(httpsPort.HasValue ? httpsPort.Value.ToString() : "NOVALUE"); } + + public Task Latin1(HttpContext context) + { + var value = context.Request.Headers["foo"]; + Assert.Equal("£", value); + return Task.CompletedTask; + } + + public Task InvalidCharacter(HttpContext context) + { + var value = context.Request.Headers["foo"]; + Assert.Equal("�", value); + return Task.CompletedTask; + } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs index 04fd9711d7..121be08041 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs @@ -130,67 +130,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure return asciiString; } - private static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span span) - { - if (span.IsEmpty) - { - return string.Empty; - } - - var resultString = new string('\0', span.Length); - - fixed (char* output = resultString) - fixed (byte* buffer = span) - { - // StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters - // in the string - if (!StringUtilities.TryGetAsciiString(buffer, output, span.Length)) - { - // null characters are considered invalid - if (span.IndexOf((byte)0) != -1) - { - throw new InvalidOperationException(); - } - - try - { - resultString = HeaderValueEncoding.GetString(buffer, span.Length); - } - catch (DecoderFallbackException) - { - throw new InvalidOperationException(); - } - } - } - - return resultString; - } - - private static unsafe string GetLatin1StringNonNullCharacters(this Span span) - { - if (span.IsEmpty) - { - return string.Empty; - } - - var resultString = new string('\0', span.Length); - - fixed (char* output = resultString) - fixed (byte* buffer = span) - { - // This returns false if there are any null (0 byte) characters in the string. - if (!StringUtilities.TryGetLatin1String(buffer, output, span.Length)) - { - // null characters are considered invalid - throw new InvalidOperationException(); - } - } - - return resultString; - } - public static string GetRequestHeaderStringNonNullCharacters(this Span span, bool useLatin1) => - useLatin1 ? GetLatin1StringNonNullCharacters(span) : GetAsciiOrUTF8StringNonNullCharacters(span); + useLatin1 ? span.GetLatin1StringNonNullCharacters() : span.GetAsciiOrUTF8StringNonNullCharacters(HeaderValueEncoding); public static string GetAsciiStringEscaped(this Span span, int maxChars) { diff --git a/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj b/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj index 6c18f1a078..98283fec77 100644 --- a/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj +++ b/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj @@ -15,6 +15,7 @@ + diff --git a/src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs b/src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs index 991571904b..085ed7d7ed 100644 --- a/src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs +++ b/src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs @@ -1,29 +1,26 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Text; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.HttpSys.Internal { internal static class HeaderEncoding { - // It should just be ASCII or ANSI, but they break badly with un-expected values. We use UTF-8 because it's the same for - // ASCII, and because some old client would send UTF8 Host headers and expect UTF8 Location responses - // (e.g. IE and HttpWebRequest on intranets). - private static Encoding Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false); + private static readonly Encoding Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false); - internal static unsafe string GetString(byte* pBytes, int byteCount) + internal static unsafe string GetString(byte* pBytes, int byteCount, bool useLatin1) { - // net451: return new string(pBytes, 0, byteCount, Encoding); - - var charCount = Encoding.GetCharCount(pBytes, byteCount); - var chars = new char[charCount]; - fixed (char* pChars = chars) + if (useLatin1) { - var count = Encoding.GetChars(pBytes, byteCount, pChars, charCount); - System.Diagnostics.Debug.Assert(count == charCount); + return new Span(pBytes, byteCount).GetLatin1StringNonNullCharacters(); + } + else + { + return new Span(pBytes, byteCount).GetAsciiOrUTF8StringNonNullCharacters(Encoding); } - return new string(chars); } internal static byte[] GetBytes(string myString) diff --git a/src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs b/src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs index 4108d901e2..f44fa0c4ec 100644 --- a/src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs +++ b/src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs @@ -18,6 +18,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal { private const int AlignmentPadding = 8; private IntPtr _originalBufferAddress; + private bool _useLatin1; private HttpApiTypes.HTTP_REQUEST* _nativeRequest; private byte[] _backingBuffer; private int _bufferAlignment; @@ -39,8 +40,9 @@ namespace Microsoft.AspNetCore.HttpSys.Internal } // To be used by IIS Integration. - internal NativeRequestContext(HttpApiTypes.HTTP_REQUEST* request) + internal NativeRequestContext(HttpApiTypes.HTTP_REQUEST* request, bool useLatin1) { + _useLatin1 = useLatin1; _nativeRequest = request; _bufferAlignment = 0; _permanentlyPinned = true; @@ -125,7 +127,8 @@ namespace Microsoft.AspNetCore.HttpSys.Internal } else if (verb == HttpApiTypes.HTTP_VERB.HttpVerbUnknown && NativeRequest->pUnknownVerb != null) { - return HeaderEncoding.GetString(NativeRequest->pUnknownVerb, NativeRequest->UnknownVerbLength); + // Never use Latin1 for the VERB + return HeaderEncoding.GetString(NativeRequest->pUnknownVerb, NativeRequest->UnknownVerbLength, useLatin1: false); } return null; @@ -291,7 +294,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal // pRawValue will point to empty string ("\0") if (pKnownHeader->RawValueLength > 0) { - value = HeaderEncoding.GetString(pKnownHeader->pRawValue + fixup, pKnownHeader->RawValueLength); + value = HeaderEncoding.GetString(pKnownHeader->pRawValue + fixup, pKnownHeader->RawValueLength, _useLatin1); } return value; @@ -329,11 +332,11 @@ namespace Microsoft.AspNetCore.HttpSys.Internal // pRawValue will be null. if (pUnknownHeader->pName != null && pUnknownHeader->NameLength > 0) { - var headerName = HeaderEncoding.GetString(pUnknownHeader->pName + fixup, pUnknownHeader->NameLength); + var headerName = HeaderEncoding.GetString(pUnknownHeader->pName + fixup, pUnknownHeader->NameLength, _useLatin1); string headerValue; if (pUnknownHeader->pRawValue != null && pUnknownHeader->RawValueLength > 0) { - headerValue = HeaderEncoding.GetString(pUnknownHeader->pRawValue + fixup, pUnknownHeader->RawValueLength); + headerValue = HeaderEncoding.GetString(pUnknownHeader->pRawValue + fixup, pUnknownHeader->RawValueLength, _useLatin1); } else { diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/StringUtilities.cs b/src/Shared/ServerInfrastructure/StringUtilities.cs similarity index 92% rename from src/Servers/Kestrel/Core/src/Internal/Infrastructure/StringUtilities.cs rename to src/Shared/ServerInfrastructure/StringUtilities.cs index 268d77a0e4..0dfdf45ade 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/StringUtilities.cs +++ b/src/Shared/ServerInfrastructure/StringUtilities.cs @@ -11,8 +11,67 @@ using System.Text; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure { - internal class StringUtilities + internal static class StringUtilities { + public static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span span, Encoding defaultEncoding) + { + if (span.IsEmpty) + { + return string.Empty; + } + + var resultString = new string('\0', span.Length); + + fixed (char* output = resultString) + fixed (byte* buffer = span) + { + // StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters + // in the string + if (!TryGetAsciiString(buffer, output, span.Length)) + { + // null characters are considered invalid + if (span.IndexOf((byte)0) != -1) + { + throw new InvalidOperationException(); + } + + try + { + resultString = defaultEncoding.GetString(buffer, span.Length); + } + catch (DecoderFallbackException) + { + throw new InvalidOperationException(); + } + } + } + + return resultString; + } + + public static unsafe string GetLatin1StringNonNullCharacters(this Span span) + { + if (span.IsEmpty) + { + return string.Empty; + } + + var resultString = new string('\0', span.Length); + + fixed (char* output = resultString) + fixed (byte* buffer = span) + { + // This returns false if there are any null (0 byte) characters in the string. + if (!TryGetLatin1String(buffer, output, span.Length)) + { + // null characters are considered invalid + throw new InvalidOperationException(); + } + } + + return resultString; + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] public static unsafe bool TryGetAsciiString(byte* input, char* output, int count) { @@ -472,7 +531,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true).GetByteCount(value); return !value.Contains('\0'); } - catch (DecoderFallbackException) { + catch (DecoderFallbackException) + { return false; } } diff --git a/src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj b/src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj index 83fe4babb7..fde501c508 100644 --- a/src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj +++ b/src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj @@ -16,6 +16,7 @@ +