Fix a write past the end of buffer when IntegerEncoder encodes to multi-bytes and does not have sufficient room to do so. (#10914)

Add some basic asserts to constrain parameters to HPACK spec.
Add some additional tests.

src/Servers/Kestrel/Core/src/Internal/Http2/HPack/IntegerEncoder.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Diagnostics;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.HPack
 {
@@ -9,6 +10,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.HPack
     {
         public static bool Encode(int i, int n, Span<byte> buffer, out int length)
         {
+            Debug.Assert(i >= 0);
+            Debug.Assert(n >= 1 && n <= 8);
+
             var j = 0;
             length = 0;
 
@@ -37,7 +41,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.HPack
                 {
                     buffer[j++] = (byte)(i % 128 + 128);
 
-                    if (j > buffer.Length)
+                    if (j >= buffer.Length)
                     {
                         return false;
                     }

src/Servers/Kestrel/Core/test/HPackIntegerTests.cs

@@ -1,6 +1,8 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.HPack;
 using Xunit;
@@ -35,5 +37,55 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
                 }
             }
         }
+
+        [Theory]
+        [MemberData(nameof(IntegerCodecSamples))]
+        public void EncodeSamples(int value, int bits, byte[] expectedResult)
+        {
+            Span<byte> actualResult = new byte[64];
+            bool success = IntegerEncoder.Encode(value, bits, actualResult, out int bytesWritten);
+
+            Assert.True(success);
+            Assert.Equal(expectedResult.Length, bytesWritten);
+            Assert.True(actualResult.Slice(0, bytesWritten).SequenceEqual(expectedResult));
+        }
+
+        [Theory]
+        [MemberData(nameof(IntegerCodecSamples))]
+        public void EncodeSamplesWithShortBuffer(int value, int bits, byte[] expectedResult)
+        {
+            Span<byte> actualResult = new byte[expectedResult.Length - 1];
+            bool success = IntegerEncoder.Encode(value, bits, actualResult, out int bytesWritten);
+
+            Assert.False(success);
+        }
+
+        [Theory]
+        [MemberData(nameof(IntegerCodecSamples))]
+        public void DecodeSamples(int expectedResult, int bits, byte[] encoded)
+        {
+            var integerDecoder = new IntegerDecoder();
+
+            bool finished = integerDecoder.BeginTryDecode(encoded[0], bits, out int actualResult);
+
+            int i = 1;
+            for (; !finished && i < encoded.Length; ++i)
+            {
+                finished = integerDecoder.TryDecode(encoded[i], out actualResult);
+            }
+
+            Assert.True(finished);
+            Assert.Equal(encoded.Length, i);
+
+            Assert.Equal(expectedResult, actualResult);
+        }
+
+        // integer, prefix length, encoded
+        public static IEnumerable<object[]> IntegerCodecSamples()
+        {
+            yield return new object[] { 10, 5, new byte[] { 0x0A } };
+            yield return new object[] { 1337, 5, new byte[] { 0x1F, 0x9A, 0x0A } };
+            yield return new object[] { 42, 8, new byte[] { 0x2A } };
+        }
     }
 }