From 4f7977d440fff5ea4ae60983fb2d20d008252c54 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 3 Oct 2016 07:58:24 +0100 Subject: [PATCH] Use PowerOfTwoToHighByte for FindFirstEqualByte --- .../Infrastructure/MemoryPoolIterator.cs | 81 +++++++------------ .../MemoryPoolIteratorTests.cs | 24 +----- 2 files changed, 31 insertions(+), 74 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 5f7e908c36..0ab1bf01ad 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -4,12 +4,14 @@ using System; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; using System.Threading; namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure { public struct MemoryPoolIterator { + private static readonly ulong _powerOfTwoToHighByte = PowerOfTwoToHighByte(); private static readonly int _vectorSpan = Vector.Count; private MemoryPoolBlock _block; @@ -307,7 +309,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(ref byte0Equals); var vectorBytesScanned = firstEqualByteIndex + 1; if (bytesScanned + vectorBytesScanned > limit) @@ -413,7 +415,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + var firstEqualByteIndex = LocateFirstFoundByte(ref byte0Equals); if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) { @@ -512,11 +514,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (!byte0Equals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); + byte0Index = LocateFirstFoundByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector.Zero)) { - byte1Index = FindFirstEqualByte(ref byte1Equals); + byte1Index = LocateFirstFoundByte(ref byte1Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) @@ -656,15 +658,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure if (!byte0Equals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); + byte0Index = LocateFirstFoundByte(ref byte0Equals); } if (!byte1Equals.Equals(Vector.Zero)) { - byte1Index = FindFirstEqualByte(ref byte1Equals); + byte1Index = LocateFirstFoundByte(ref byte1Equals); } if (!byte2Equals.Equals(Vector.Zero)) { - byte2Index = FindFirstEqualByte(ref byte2Equals); + byte2Index = LocateFirstFoundByte(ref byte2Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) @@ -761,60 +763,30 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure } /// - /// Find first byte + /// Locate the first of the found bytes /// /// /// The first index of the result vector - /// byteEquals = 0 - internal static int FindFirstEqualByte(ref Vector byteEquals) + // Force inlining (64 IL bytes, 91 bytes asm) Issue: https://github.com/dotnet/coreclr/issues/7386 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LocateFirstFoundByte(ref Vector byteEquals) { - if (!BitConverter.IsLittleEndian) return FindFirstEqualByteSlow(ref byteEquals); - - // Quasi-tree search var vector64 = Vector.AsVectorInt64(byteEquals); - for (var i = 0; i < Vector.Count; i++) + var i = 0; + long longValue = 0; + for (; i < Vector.Count; i++) { - var longValue = vector64[i]; + longValue = vector64[i]; if (longValue == 0) continue; - - return (i << 3) + - ((longValue & 0x00000000ffffffff) > 0 - ? (longValue & 0x000000000000ffff) > 0 - ? (longValue & 0x00000000000000ff) > 0 ? 0 : 1 - : (longValue & 0x0000000000ff0000) > 0 ? 2 : 3 - : (longValue & 0x0000ffff00000000) > 0 - ? (longValue & 0x000000ff00000000) > 0 ? 4 : 5 - : (longValue & 0x00ff000000000000) > 0 ? 6 : 7); + break; } - throw new InvalidOperationException(); - } - // Internal for testing - internal static int FindFirstEqualByteSlow(ref Vector byteEquals) - { - // Quasi-tree search - var vector64 = Vector.AsVectorInt64(byteEquals); - for (var i = 0; i < Vector.Count; i++) - { - var longValue = vector64[i]; - if (longValue == 0) continue; - - var shift = i << 1; - var offset = shift << 2; - var vector32 = Vector.AsVectorInt32(byteEquals); - if (vector32[shift] != 0) - { - if (byteEquals[offset] != 0) return offset; - if (byteEquals[offset + 1] != 0) return offset + 1; - if (byteEquals[offset + 2] != 0) return offset + 2; - return offset + 3; - } - if (byteEquals[offset + 4] != 0) return offset + 4; - if (byteEquals[offset + 5] != 0) return offset + 5; - if (byteEquals[offset + 6] != 0) return offset + 6; - return offset + 7; - } - throw new InvalidOperationException(); + // Flag least significant power of two bit + var powerOfTwoFlag = (ulong)(longValue & -longValue); + // Shift all powers of two into the high byte and extract + var foundByteIndex = (int)((powerOfTwoFlag * _powerOfTwoToHighByte) >> 61); + // Single LEA instruction with jitted const (using function result) + return i * 8 + foundByteIndex; } /// @@ -1059,5 +1031,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure _block = block; _index = blockIndex; } + + private static ulong PowerOfTwoToHighByte() + { + return BitConverter.IsLittleEndian ? 0x20406080A0C0E0ul : 0xE0C0A080604020ul; + } } } diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 454f66291f..8ac556ed55 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -32,7 +32,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests for (int i = 0; i < Vector.Count; i++) { Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByte(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(ref vector)); bytes[i] = 0; } @@ -40,27 +40,7 @@ namespace Microsoft.AspNetCore.Server.KestrelTests { bytes[i] = 1; Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByte(ref vector)); - bytes[i] = 0; - } - } - - [Fact] - public void TestFindFirstEqualByteSlow() - { - var bytes = Enumerable.Repeat(0xff, Vector.Count).ToArray(); - for (int i = 0; i < Vector.Count; i++) - { - Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByteSlow(ref vector)); - bytes[i] = 0; - } - - for (int i = 0; i < Vector.Count; i++) - { - bytes[i] = 1; - Vector vector = new Vector(bytes); - Assert.Equal(i, MemoryPoolIterator.FindFirstEqualByteSlow(ref vector)); + Assert.Equal(i, MemoryPoolIterator.LocateFirstFoundByte(ref vector)); bytes[i] = 0; } }