[3.1] Add latin1 support to IIS (#22798)

Co-authored-by: Chris Ross <Tratcher@Outlook.com>

src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj

@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <Compile Include="$(SharedSourceRoot)HttpSys\**\*.cs" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\*.cs" LinkBase="ServerInfrastructure" />
   </ItemGroup>
 
   <ItemGroup>

src/Servers/IIS/IIS/src/Core/IISHttpContext.cs

@@ -74,8 +74,9 @@ namespace Microsoft.AspNetCore.Server.IIS.Core
             IntPtr pInProcessHandler,
             IISServerOptions options,
             IISHttpServer server,
-            ILogger logger)
+            ILogger logger,
-            : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.HttpGetRawRequest(pInProcessHandler))
+            bool useLatin1)
+            : base((HttpApiTypes.HTTP_REQUEST*)NativeMethods.HttpGetRawRequest(pInProcessHandler), useLatin1: useLatin1)
         {
             _memoryPool = memoryPool;
             _pInProcessHandler = pInProcessHandler;

src/Servers/IIS/IIS/src/Core/IISHttpContextOfT.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Buffers;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
@@ -15,8 +16,8 @@ namespace Microsoft.AspNetCore.Server.IIS.Core
     {
         private readonly IHttpApplication<TContext> _application;
 
-        public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger)
+        public IISHttpContextOfT(MemoryPool<byte> memoryPool, IHttpApplication<TContext> application, IntPtr pInProcessHandler, IISServerOptions options, IISHttpServer server, ILogger logger, bool useLatin1)
-            : base(memoryPool, pInProcessHandler, options, server, logger)
+            : base(memoryPool, pInProcessHandler, options, server, logger, useLatin1)
         {
             _application = application;
         }

src/Servers/IIS/IIS/src/Core/IISHttpServer.cs

@@ -214,11 +214,14 @@ namespace Microsoft.AspNetCore.Server.IIS.Core
 
         private class IISContextFactory<T> : IISContextFactory
         {
+            private const string Latin1Suppport = "Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders";
+
             private readonly IHttpApplication<T> _application;
             private readonly MemoryPool<byte> _memoryPool;
             private readonly IISServerOptions _options;
             private readonly IISHttpServer _server;
             private readonly ILogger _logger;
+            private readonly bool _useLatin1;
 
             public IISContextFactory(MemoryPool<byte> memoryPool, IHttpApplication<T> application, IISServerOptions options, IISHttpServer server, ILogger logger)
             {
@@ -227,11 +230,12 @@ namespace Microsoft.AspNetCore.Server.IIS.Core
                 _options = options;
                 _server = server;
                 _logger = logger;
+                AppContext.TryGetSwitch(Latin1Suppport, out _useLatin1);
             }
 
             public IISHttpContext CreateHttpContext(IntPtr pInProcessHandler)
             {
-                return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server, _logger);
+                return new IISHttpContextOfT<T>(_memoryPool, _application, pInProcessHandler, _options, _server, _logger, _useLatin1);
             }
         }
     }

src/Servers/IIS/IIS/src/Microsoft.AspNetCore.Server.IIS.csproj

@@ -19,6 +19,7 @@
     <Compile Include="$(SharedSourceRoot)StackTrace\**\*.cs" LinkBase="Shared\" />
     <Compile Include="$(SharedSourceRoot)RazorViews\*.cs" LinkBase="Shared\" />
     <Compile Include="$(SharedSourceRoot)ErrorPage\*.cs" LinkBase="Shared\" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\*.cs" LinkBase="Shared\" />
   </ItemGroup>
 
   <Target Name="ValidateNativeComponentsBuilt" AfterTargets="Build" >

src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/Latin1Tests.cs

@@ -0,0 +1,82 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Server.IIS.FunctionalTests.Utilities;
+using Microsoft.AspNetCore.Server.IntegrationTesting;
+using Microsoft.AspNetCore.Server.IntegrationTesting.IIS;
+using Microsoft.AspNetCore.Testing;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess
+{
+    [Collection(PublishedSitesCollection.Name)]
+    public class Latin1Tests : IISFunctionalTestBase
+    {
+        public Latin1Tests(PublishedSitesFixture fixture) : base(fixture)
+        {
+        }
+
+        [ConditionalFact]
+        [RequiresNewHandler]
+        public async Task Latin1Works()
+        {
+            var deploymentParameters = Fixture.GetBaseDeploymentParameters();
+            deploymentParameters.TransformArguments((a, _) => $"{a} AddLatin1");
+
+            var deploymentResult = await DeployAsync(deploymentParameters);
+
+            var client = new HttpClient(new LoggingHandler(new WinHttpHandler() { SendTimeout = TimeSpan.FromMinutes(3) }, deploymentResult.Logger));
+
+            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{deploymentResult.ApplicationBaseUri}Latin1");
+            requestMessage.Headers.Add("foo", "£");
+
+            var result = await client.SendAsync(requestMessage);
+            Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+        }
+
+        [ConditionalFact]
+        [RequiresNewHandler]
+        public async Task Latin1ReplacedWithoutAppContextSwitch()
+        {
+            var deploymentParameters = Fixture.GetBaseDeploymentParameters();
+            deploymentParameters.TransformArguments((a, _) => $"{a}");
+
+            var deploymentResult = await DeployAsync(deploymentParameters);
+
+            var client = new HttpClient(new LoggingHandler(new WinHttpHandler() { SendTimeout = TimeSpan.FromMinutes(3) }, deploymentResult.Logger));
+
+            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{deploymentResult.ApplicationBaseUri}InvalidCharacter");
+            requestMessage.Headers.Add("foo", "£");
+
+            var result = await client.SendAsync(requestMessage);
+            Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+        }
+
+        [ConditionalFact]
+        [RequiresNewHandler]
+        public async Task Latin1InvalidCharacters_HttpSysRejects()
+        {
+            var deploymentParameters = Fixture.GetBaseDeploymentParameters();
+            deploymentParameters.TransformArguments((a, _) => $"{a} AddLatin1");
+
+            var deploymentResult = await DeployAsync(deploymentParameters);
+
+            using (var connection = new TestConnection(deploymentResult.HttpClient.BaseAddress.Port))
+            {
+                await connection.Send(
+                    "GET /ReadAndFlushEcho HTTP/1.1",
+                    "Host: localhost",
+                    "Connection: close",
+                    "foo: £\0a",
+                    "",
+                    "");
+
+                await connection.ReceiveStartsWith("HTTP/1.1 400 Bad Request");
+            }
+        }
+    }
+}

src/Servers/IIS/IIS/test/IIS.FunctionalTests/IIS.FunctionalTests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 
   <PropertyGroup>
@@ -27,6 +27,7 @@
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="System.Diagnostics.EventLog" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>

src/Servers/IIS/IIS/test/IIS.NewHandler.FunctionalTests/IIS.NewHandler.FunctionalTests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
@@ -34,6 +34,7 @@
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="System.Diagnostics.EventLog" />
     <Reference Include="System.Net.WebSockets.WebSocketProtocol" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>

src/Servers/IIS/IIS/test/IIS.NewShim.FunctionalTests/IIS.NewShim.FunctionalTests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
@@ -28,6 +28,7 @@
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
     <Reference Include="System.Diagnostics.EventLog" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>

src/Servers/IIS/IIS/test/IISExpress.FunctionalTests/IISExpress.FunctionalTests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <Import Project="../FunctionalTest.props" />
 
@@ -30,6 +30,7 @@
     <Reference Include="Microsoft.Extensions.Logging" />
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
     <Reference Include="System.Diagnostics.EventLog" />
+    <Reference Include="System.Net.Http.WinHttpHandler" />
   </ItemGroup>
 
 </Project>

src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Program.cs

@@ -145,9 +145,9 @@ namespace TestSite
                     }
 
                     return 0;
+#if !FORWARDCOMPAT
                 case "ThrowInStartupGenericHost":
                     {
-#if !FORWARDCOMPAT
                         var host = new HostBuilder().ConfigureWebHost((c) =>
                         {
                             c.ConfigureLogging((_, factory) =>
@@ -161,9 +161,26 @@ namespace TestSite
                                    
 
                         host.Build().Run();
-#endif
+                        return 0;
                     }
-                    return 0;
+                case "AddLatin1":
+                    {
+                        AppContext.SetSwitch("Microsoft.AspNetCore.Server.IIS.Latin1RequestHeaders", isEnabled: true);
+                        var host = new HostBuilder().ConfigureWebHost((c) =>
+                        {
+                            c.ConfigureLogging((_, factory) =>
+                            {
+                                factory.AddConsole();
+                                factory.AddFilter("Console", level => level >= LogLevel.Information);
+                            })
+                            .UseIIS()
+                            .UseStartup<Startup>();
+                        });
+
+                        host.Build().Run();
+                        return 0;
+                    }
+#endif
                 default:
                     return StartServer();
 

src/Servers/IIS/IIS/test/testassets/InProcessWebSite/Startup.cs

@@ -1005,5 +1005,19 @@ namespace TestSite
 
             await context.Response.WriteAsync(httpsPort.HasValue ? httpsPort.Value.ToString() : "NOVALUE");
         }
+
+        public Task Latin1(HttpContext context)
+        {
+            var value = context.Request.Headers["foo"];
+            Assert.Equal("£", value);
+            return Task.CompletedTask;
+        }
+
+        public Task InvalidCharacter(HttpContext context)
+        {
+            var value = context.Request.Headers["foo"];
+            Assert.Equal("<22>", value);
+            return Task.CompletedTask;
+        }
     }
 }

src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs

@@ -130,67 +130,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
             return asciiString;
         }
 
-        private static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte> span)
-        {
-            if (span.IsEmpty)
-            {
-                return string.Empty;
-            }
-
-            var resultString = new string('\0', span.Length);
-
-            fixed (char* output = resultString)
-            fixed (byte* buffer = span)
-            {
-                // StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters
-                // in the string
-                if (!StringUtilities.TryGetAsciiString(buffer, output, span.Length))
-                {
-                    // null characters are considered invalid
-                    if (span.IndexOf((byte)0) != -1)
-                    {
-                        throw new InvalidOperationException();
-                    }
-
-                    try
-                    {
-                        resultString = HeaderValueEncoding.GetString(buffer, span.Length);
-                    }
-                    catch (DecoderFallbackException)
-                    {
-                        throw new InvalidOperationException();
-                    }
-                }
-            }
-
-            return resultString;
-        }
-
-        private static unsafe string GetLatin1StringNonNullCharacters(this Span<byte> span)
-        {
-            if (span.IsEmpty)
-            {
-                return string.Empty;
-            }
-
-            var resultString = new string('\0', span.Length);
-
-            fixed (char* output = resultString)
-            fixed (byte* buffer = span)
-            {
-                // This returns false if there are any null (0 byte) characters in the string.
-                if (!StringUtilities.TryGetLatin1String(buffer, output, span.Length))
-                {
-                    // null characters are considered invalid
-                    throw new InvalidOperationException();
-                }
-            }
-
-            return resultString;
-        }
-
         public static string GetRequestHeaderStringNonNullCharacters(this Span<byte> span, bool useLatin1) =>
-            useLatin1 ? GetLatin1StringNonNullCharacters(span) : GetAsciiOrUTF8StringNonNullCharacters(span);
+            useLatin1 ? span.GetLatin1StringNonNullCharacters() : span.GetAsciiOrUTF8StringNonNullCharacters(HeaderValueEncoding);
 
         public static string GetAsciiStringEscaped(this Span<byte> span, int maxChars)
         {

src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj

@@ -15,6 +15,7 @@
     <Compile Include="$(SharedSourceRoot)CertificateGeneration\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)ValueTaskExtensions\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)UrlDecoder\**\*.cs" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\**\*.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/Shared/HttpSys/RequestProcessing/HeaderEncoding.cs

@@ -1,29 +1,26 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Text;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
 namespace Microsoft.AspNetCore.HttpSys.Internal
 {
     internal static class HeaderEncoding
     {
-        // It should just be ASCII or ANSI, but they break badly with un-expected values. We use UTF-8 because it's the same for
+        private static readonly Encoding Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false);
-        // ASCII, and because some old client would send UTF8 Host headers and expect UTF8 Location responses
-        // (e.g. IE and HttpWebRequest on intranets).
-        private static Encoding Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false);
 
-        internal static unsafe string GetString(byte* pBytes, int byteCount)
+        internal static unsafe string GetString(byte* pBytes, int byteCount, bool useLatin1)
         {
-            // net451: return new string(pBytes, 0, byteCount, Encoding);
+            if (useLatin1)
-
-            var charCount = Encoding.GetCharCount(pBytes, byteCount);
-            var chars = new char[charCount];
-            fixed (char* pChars = chars)
             {
-                var count = Encoding.GetChars(pBytes, byteCount, pChars, charCount);
+                return new Span<byte>(pBytes, byteCount).GetLatin1StringNonNullCharacters();
-                System.Diagnostics.Debug.Assert(count == charCount);
+            }
+            else
+            {
+                return new Span<byte>(pBytes, byteCount).GetAsciiOrUTF8StringNonNullCharacters(Encoding);
             }
-            return new string(chars);
         }
 
         internal static byte[] GetBytes(string myString)

src/Shared/HttpSys/RequestProcessing/NativeRequestContext.cs

@@ -18,6 +18,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
     {
         private const int AlignmentPadding = 8;
         private IntPtr _originalBufferAddress;
+        private bool _useLatin1;
         private HttpApiTypes.HTTP_REQUEST* _nativeRequest;
         private byte[] _backingBuffer;
         private int _bufferAlignment;
@@ -39,8 +40,9 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
         }
 
         // To be used by IIS Integration.
-        internal NativeRequestContext(HttpApiTypes.HTTP_REQUEST* request)
+        internal NativeRequestContext(HttpApiTypes.HTTP_REQUEST* request, bool useLatin1)
         {
+            _useLatin1 = useLatin1;
             _nativeRequest = request;
             _bufferAlignment = 0;
             _permanentlyPinned = true;
@@ -125,7 +127,8 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
             }
             else if (verb == HttpApiTypes.HTTP_VERB.HttpVerbUnknown && NativeRequest->pUnknownVerb != null)
             {
-                return HeaderEncoding.GetString(NativeRequest->pUnknownVerb, NativeRequest->UnknownVerbLength);
+                // Never use Latin1 for the VERB
+                return HeaderEncoding.GetString(NativeRequest->pUnknownVerb, NativeRequest->UnknownVerbLength, useLatin1: false);
             }
 
             return null;
@@ -291,7 +294,7 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
             // pRawValue will point to empty string ("\0")
             if (pKnownHeader->RawValueLength > 0)
             {
-                value = HeaderEncoding.GetString(pKnownHeader->pRawValue + fixup, pKnownHeader->RawValueLength);
+                value = HeaderEncoding.GetString(pKnownHeader->pRawValue + fixup, pKnownHeader->RawValueLength, _useLatin1);
             }
 
             return value;
@@ -329,11 +332,11 @@ namespace Microsoft.AspNetCore.HttpSys.Internal
                     // pRawValue will be null.
                     if (pUnknownHeader->pName != null && pUnknownHeader->NameLength > 0)
                     {
-                        var headerName = HeaderEncoding.GetString(pUnknownHeader->pName + fixup, pUnknownHeader->NameLength);
+                        var headerName = HeaderEncoding.GetString(pUnknownHeader->pName + fixup, pUnknownHeader->NameLength, _useLatin1);
                         string headerValue;
                         if (pUnknownHeader->pRawValue != null && pUnknownHeader->RawValueLength > 0)
                         {
-                            headerValue = HeaderEncoding.GetString(pUnknownHeader->pRawValue + fixup, pUnknownHeader->RawValueLength);
+                            headerValue = HeaderEncoding.GetString(pUnknownHeader->pRawValue + fixup, pUnknownHeader->RawValueLength, _useLatin1);
                         }
                         else
                         {

src/Shared/ServerInfrastructure/StringUtilities.cs

@@ -11,8 +11,67 @@ using System.Text;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
 {
-    internal class StringUtilities
+    internal static class StringUtilities
     {
+        public static unsafe string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte> span, Encoding defaultEncoding)
+        {
+            if (span.IsEmpty)
+            {
+                return string.Empty;
+            }
+
+            var resultString = new string('\0', span.Length);
+
+            fixed (char* output = resultString)
+            fixed (byte* buffer = span)
+            {
+                // StringUtilities.TryGetAsciiString returns null if there are any null (0 byte) characters
+                // in the string
+                if (!TryGetAsciiString(buffer, output, span.Length))
+                {
+                    // null characters are considered invalid
+                    if (span.IndexOf((byte)0) != -1)
+                    {
+                        throw new InvalidOperationException();
+                    }
+
+                    try
+                    {
+                        resultString = defaultEncoding.GetString(buffer, span.Length);
+                    }
+                    catch (DecoderFallbackException)
+                    {
+                        throw new InvalidOperationException();
+                    }
+                }
+            }
+
+            return resultString;
+        }
+
+        public static unsafe string GetLatin1StringNonNullCharacters(this Span<byte> span)
+        {
+            if (span.IsEmpty)
+            {
+                return string.Empty;
+            }
+
+            var resultString = new string('\0', span.Length);
+
+            fixed (char* output = resultString)
+            fixed (byte* buffer = span)
+            {
+                // This returns false if there are any null (0 byte) characters in the string.
+                if (!TryGetLatin1String(buffer, output, span.Length))
+                {
+                    // null characters are considered invalid
+                    throw new InvalidOperationException();
+                }
+            }
+
+            return resultString;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe bool TryGetAsciiString(byte* input, char* output, int count)
         {
@@ -472,7 +531,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
                 new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true).GetByteCount(value);
                 return !value.Contains('\0');
             }
-            catch (DecoderFallbackException) {
+            catch (DecoderFallbackException)
+            {
                 return false;
             }
         }

src/Shared/test/Shared.Tests/Microsoft.AspNetCore.Shared.Tests.csproj

@@ -16,6 +16,7 @@
     <Compile Include="$(SharedSourceRoot)SecurityHelper\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)StackTrace\StackFrame\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)WebEncoders\**\*.cs" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\**\*.cs" />
   </ItemGroup>
 
   <ItemGroup>