From 3511c8cef094188b9e781dce7430969ec254c0f1 Mon Sep 17 00:00:00 2001 From: Ryan Nowak Date: Mon, 27 Aug 2018 14:52:16 -0700 Subject: [PATCH 1/2] Add vectorized il-emit trie jump table Add new futuristic jump table. Remove old experimental jump tables since this is much much better. --- Directory.Build.props | 2 +- .../Matching/AsciiKeyedJumpTable.cs | 172 ----- .../Matching/CustomHashTableJumpTable.cs | 197 ------ .../Matching/DictionaryLookupJumpTable.cs | 112 ---- .../JumpTableMultipleEntryBenchmark.cs | 55 +- .../Matching/JumpTableSingleEntryBenchmark.cs | 259 +++++++- build/dependencies.props | 2 + .../Matching/ILEmitTrieFactory.cs | 588 ++++++++++++++++++ .../Matching/ILEmitTrieJumpTable.cs | 102 +++ .../Matching/JumpTableBuilder.cs | 50 +- .../Microsoft.AspNetCore.Routing.csproj | 16 +- .../Matching/ILEmitTrieJumpTableTest.cs | 228 +++++++ .../NonVectorizedILEmitTrieJumpTableTest.cs | 12 + .../VectorizedILEmitTrieJumpTableTest.cs | 14 + .../Microsoft.AspNetCore.Routing.Tests.csproj | 12 +- 15 files changed, 1291 insertions(+), 530 deletions(-) delete mode 100644 benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/AsciiKeyedJumpTable.cs delete mode 100644 benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/CustomHashTableJumpTable.cs delete mode 100644 benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/DictionaryLookupJumpTable.cs create mode 100644 src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs create mode 100644 src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs create mode 100644 test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieJumpTableTest.cs create mode 100644 test/Microsoft.AspNetCore.Routing.Tests/Matching/NonVectorizedILEmitTrieJumpTableTest.cs create mode 100644 test/Microsoft.AspNetCore.Routing.Tests/Matching/VectorizedILEmitTrieJumpTableTest.cs diff --git a/Directory.Build.props b/Directory.Build.props index cc459eb3d3..42b13bf610 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -18,6 +18,6 @@ MicrosoftNuGet true true - 7.2 + 7.3 diff --git a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/AsciiKeyedJumpTable.cs b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/AsciiKeyedJumpTable.cs deleted file mode 100644 index 2e99bd9faa..0000000000 --- a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/AsciiKeyedJumpTable.cs +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; - -namespace Microsoft.AspNetCore.Routing.Matching -{ - // An optimized jump table that trades a small amount of additional memory for - // hash-table like performance. - // - // The optimization here is to use the first character of the known entries - // as a 'key' in the hash table in the space of A-Z. This gives us a maximum - // of 26 buckets (hence the reduced memory) - internal class AsciiKeyedJumpTable : JumpTable - { - public static bool TryCreate( - int defaultDestination, - int exitDestination, - List<(string text, int destination)> entries, - out JumpTable result) - { - result = null; - - // First we group string by their uppercase letter. If we see a string - // that starts with a non-ASCII letter - var map = new Dictionary>(); - - for (var i = 0; i < entries.Count; i++) - { - if (entries[i].text.Length == 0) - { - return false; - } - - if (!IsAscii(entries[i].text)) - { - return false; - } - - var first = ToUpperAscii(entries[i].text[0]); - if (first < 'A' || first > 'Z') - { - // Not a letter - return false; - } - - if (!map.TryGetValue(first, out var matches)) - { - matches = new List<(string text, int destination)>(); - map.Add(first, matches); - } - - matches.Add(entries[i]); - } - - var next = 0; - var ordered = new(string text, int destination)[entries.Count]; - var indexes = new int[26 * 2]; - for (var i = 0; i < 26; i++) - { - indexes[i * 2] = next; - - var length = 0; - if (map.TryGetValue((char)('A' + i), out var matches)) - { - length += matches.Count; - for (var j = 0; j < matches.Count; j++) - { - ordered[next++] = matches[j]; - } - } - - indexes[i * 2 + 1] = length; - } - - result = new AsciiKeyedJumpTable(defaultDestination, exitDestination, ordered, indexes); - return true; - } - - private readonly int _defaultDestination; - private readonly int _exitDestination; - private readonly (string text, int destination)[] _entries; - private readonly int[] _indexes; - - private AsciiKeyedJumpTable( - int defaultDestination, - int exitDestination, - (string text, int destination)[] entries, - int[] indexes) - { - _defaultDestination = defaultDestination; - _exitDestination = exitDestination; - _entries = entries; - _indexes = indexes; - } - - public override int GetDestination(string path, PathSegment segment) - { - if (segment.Length == 0) - { - return _exitDestination; - } - - var c = path[segment.Start]; - if (!IsAscii(c)) - { - return _defaultDestination; - } - - c = ToUpperAscii(c); - if (c < 'A' || c > 'Z') - { - // Character is non-ASCII or not a letter. Since we know that all of the entries are ASCII - // and begin with a letter this is not a match. - return _defaultDestination; - } - - var offset = (c - 'A') * 2; - var start = _indexes[offset]; - var length = _indexes[offset + 1]; - - var entries = _entries; - for (var i = start; i < start + length; i++) - { - var text = entries[i].text; - if (segment.Length == text.Length && - string.Compare( - path, - segment.Start, - text, - 0, - segment.Length, - StringComparison.OrdinalIgnoreCase) == 0) - { - return entries[i].destination; - } - } - - return _defaultDestination; - } - - internal static bool IsAscii(char c) - { - // ~0x7F is a bit mask that checks for bits that won't be set in an ASCII character. - // ASCII only uses the lowest 7 bits. - return (c & ~0x7F) == 0; - } - - internal static bool IsAscii(string text) - { - for (var i = 0; i < text.Length; i++) - { - var c = text[i]; - if (!IsAscii(c)) - { - return false; - } - } - - return true; - } - - internal static char ToUpperAscii(char c) - { - // 0x5F can be used to convert a character to uppercase ascii (assuming it's a letter). - // This works because lowercase ASCII chars are exactly 32 less than their uppercase - // counterparts. - return (char)(c & 0x5F); - } - } -} \ No newline at end of file diff --git a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/CustomHashTableJumpTable.cs b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/CustomHashTableJumpTable.cs deleted file mode 100644 index d7bf028dbd..0000000000 --- a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/CustomHashTableJumpTable.cs +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; - -namespace Microsoft.AspNetCore.Routing.Matching -{ - internal class CustomHashTableJumpTable : JumpTable - { - // Similar to HashHelpers list of primes, but truncated. We don't expect - // incredibly large numbers to be useful here. - private static readonly int[] Primes = new int[] - { - 3, 7, 11, 17, 23, 29, 37, 47, 59, - 71, 89, 107, 131, 163, 197, 239, 293, - 353, 431, 521, 631, 761, 919, 1103, 1327, - 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, - 7013, 8419, 10103, - }; - - private readonly int _defaultDestination; - private readonly int _exitDestination; - - private readonly int _prime; - private readonly int[] _buckets; - private readonly Entry[] _entries; - - public CustomHashTableJumpTable( - int defaultDestination, - int exitDestination, - (string text, int destination)[] entries) - { - _defaultDestination = defaultDestination; - _exitDestination = exitDestination; - - var map = new Dictionary>(); - - for (var i = 0; i < entries.Length; i++) - { - var key = GetKey(entries[i].text, new PathSegment(0, entries[i].text.Length)); - if (!map.TryGetValue(key, out var matches)) - { - matches = new List<(string text, int destination)>(); - map.Add(key, matches); - } - - matches.Add(entries[i]); - } - - _prime = GetPrime(map.Count); - _buckets = new int[_prime + 1]; - _entries = new Entry[map.Sum(kvp => kvp.Value.Count)]; - - var next = 0; - foreach (var group in map.GroupBy(kvp => kvp.Key % _prime).OrderBy(g => g.Key)) - { - _buckets[group.Key] = next; - - foreach (var array in group) - { - for (var i = 0; i < array.Value.Count; i++) - { - _entries[next++] = new Entry(array.Value[i].text, array.Value[i].destination); - } - } - } - - Debug.Assert(next == _entries.Length); - _buckets[_prime] = next; - - var last = 0; - for (var i = 0; i < _buckets.Length; i++) - { - if (_buckets[i] == 0) - { - _buckets[i] = last; - } - else - { - last = _buckets[i]; - } - } - } - - public int Find(int key) - { - return key % _prime; - } - - private static int GetPrime(int capacity) - { - for (int i = 0; i < Primes.Length; i++) - { - int prime = Primes[i]; - if (prime >= capacity) - { - return prime; - } - } - - return Primes[Primes.Length - 1]; - } - - public override int GetDestination(string path, PathSegment segment) - { - if (segment.Length == 0) - { - return _exitDestination; - } - - var key = GetKey(path, segment); - var index = Find(key); - - var start = _buckets[index]; - var end = _buckets[index + 1]; - - var entries = _entries.AsSpan(start, end - start); - for (var i = 0; i < entries.Length; i++) - { - var text = entries[i].Text; - if (text.Length == segment.Length && - string.Compare( - path, - segment.Start, - text, - 0, - segment.Length, - StringComparison.OrdinalIgnoreCase) == 0) - { - return entries[i].Destination; - } - } - - return _defaultDestination; - } - - // Builds a hashcode from segment (first four characters converted to 8bit ASCII) - private static unsafe int GetKey(string path, PathSegment segment) - { - fixed (char* p = path) - { - switch (path.Length) - { - case 0: - { - return 0; - } - - case 1: - { - return - ((*(p + segment.Start + 0) & 0x5F) << (0 * 8)); - } - - case 2: - { - return - ((*(p + segment.Start + 0) & 0x5F) << (0 * 8)) | - ((*(p + segment.Start + 1) & 0x5F) << (1 * 8)); - } - - case 3: - { - return - ((*(p + segment.Start + 0) & 0x5F) << (0 * 8)) | - ((*(p + segment.Start + 1) & 0x5F) << (1 * 8)) | - ((*(p + segment.Start + 2) & 0x5F) << (2 * 8)); - } - - default: - { - return - ((*(p + segment.Start + 0) & 0x5F) << (0 * 8)) | - ((*(p + segment.Start + 1) & 0x5F) << (1 * 8)) | - ((*(p + segment.Start + 2) & 0x5F) << (2 * 8)) | - ((*(p + segment.Start + 3) & 0x5F) << (3 * 8)); - } - } - } - } - - private readonly struct Entry - { - public readonly string Text; - public readonly int Destination; - - public Entry(string text, int destination) - { - Text = text; - Destination = destination; - } - } - } -} diff --git a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/DictionaryLookupJumpTable.cs b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/DictionaryLookupJumpTable.cs deleted file mode 100644 index 847d44e4ac..0000000000 --- a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/DictionaryLookupJumpTable.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.AspNetCore.Routing.Matching -{ - internal class DictionaryLookupJumpTable : JumpTable - { - private readonly int _defaultDestination; - private readonly int _exitDestination; - private readonly Dictionary _store; - - public DictionaryLookupJumpTable( - int defaultDestination, - int exitDestination, - (string text, int destination)[] entries) - { - _defaultDestination = defaultDestination; - _exitDestination = exitDestination; - - var map = new Dictionary>(); - - for (var i = 0; i < entries.Length; i++) - { - var key = GetKey(entries[i].text.AsSpan()); - if (!map.TryGetValue(key, out var matches)) - { - matches = new List<(string text, int destination)>(); - map.Add(key, matches); - } - - matches.Add(entries[i]); - } - - _store = map.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToArray()); - } - - public override int GetDestination(string path, PathSegment segment) - { - if (segment.Length == 0) - { - return _exitDestination; - } - - var key = GetKey(path.AsSpan(segment.Start, segment.Length)); - if (_store.TryGetValue(key, out var entries)) - { - for (var i = 0; i < entries.Length; i++) - { - var text = entries[i].text; - if (text.Length == segment.Length && - string.Compare( - path, - segment.Start, - text, - 0, - segment.Length, - StringComparison.OrdinalIgnoreCase) == 0) - { - return entries[i].destination; - } - } - } - - return _defaultDestination; - } - - private static int GetKey(string path, PathSegment segment) - { - return GetKey(path.AsSpan(segment.Start, segment.Length)); - } - - /// builds a key from the last byte of length + first 3 characters of text (converted to ascii) - private static int GetKey(ReadOnlySpan span) - { - var length = (byte)(span.Length & 0xFF); - - byte c0, c1, c2; - switch (length) - { - case 0: - { - return 0; - } - - case 1: - { - c0 = (byte)(span[0] & 0x5F); - return (length << 24) | (c0 << 16); - } - - case 2: - { - c0 = (byte)(span[0] & 0x5F); - c1 = (byte)(span[1] & 0x5F); - return (length << 24) | (c0 << 16) | (c1 << 8); - } - - default: - { - c0 = (byte)(span[0] & 0x5F); - c1 = (byte)(span[1] & 0x5F); - c2 = (byte)(span[2] & 0x5F); - return (length << 24) | (c0 << 16) | (c1 << 8) | c2; - } - } - } - } -} diff --git a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableMultipleEntryBenchmark.cs b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableMultipleEntryBenchmark.cs index 410dbe85ab..b3a6855025 100644 --- a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableMultipleEntryBenchmark.cs +++ b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableMultipleEntryBenchmark.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using BenchmarkDotNet.Attributes; namespace Microsoft.AspNetCore.Routing.Matching @@ -15,12 +14,11 @@ namespace Microsoft.AspNetCore.Routing.Matching private JumpTable _linearSearch; private JumpTable _dictionary; - private JumpTable _ascii; - private JumpTable _dictionaryLookup; - private JumpTable _customHashTable; + private JumpTable _trie; + private JumpTable _vectorTrie; // All factors of 100 to support sampling - [Params(2, 4, 5, 10, 25)] + [Params(2, 5, 10, 25, 50, 100)] public int Count; [GlobalSetup] @@ -48,9 +46,8 @@ namespace Microsoft.AspNetCore.Routing.Matching _linearSearch = new LinearSearchJumpTable(0, -1, entries.ToArray()); _dictionary = new DictionaryJumpTable(0, -1, entries.ToArray()); - AsciiKeyedJumpTable.TryCreate(0, -1, entries, out _ascii); - _dictionaryLookup = new DictionaryLookupJumpTable(0, -1, entries.ToArray()); - _customHashTable = new CustomHashTableJumpTable(0, -1, entries.ToArray()); + _trie = new ILEmitTrieJumpTable(0, -1, entries.ToArray(), vectorize: false, _dictionary); + _vectorTrie = new ILEmitTrieJumpTable(0, -1, entries.ToArray(), vectorize: true, _dictionary); } // This baseline is similar to SingleEntryJumpTable. We just want @@ -67,15 +64,24 @@ namespace Microsoft.AspNetCore.Routing.Matching var @string = strings[i]; var segment = segments[i]; - destination = segment.Length == 0 ? -1 : - segment.Length != @string.Length ? 1 : - string.Compare( + if (segment.Length == 0) + { + destination = -1; + } + else if (segment.Length != @string.Length) + { + destination = 1; + } + else + { + destination = string.Compare( @string, segment.Start, @string, 0, - @string.Length, + segment.Length, StringComparison.OrdinalIgnoreCase); + } } return destination; @@ -112,7 +118,7 @@ namespace Microsoft.AspNetCore.Routing.Matching } [Benchmark(OperationsPerInvoke = 100)] - public int Ascii() + public int Trie() { var strings = _strings; var segments = _segments; @@ -120,14 +126,14 @@ namespace Microsoft.AspNetCore.Routing.Matching var destination = 0; for (var i = 0; i < strings.Length; i++) { - destination = _ascii.GetDestination(strings[i], segments[i]); + destination = _trie.GetDestination(strings[i], segments[i]); } return destination; } [Benchmark(OperationsPerInvoke = 100)] - public int DictionaryLookup() + public int VectorTrie() { var strings = _strings; var segments = _segments; @@ -135,22 +141,7 @@ namespace Microsoft.AspNetCore.Routing.Matching var destination = 0; for (var i = 0; i < strings.Length; i++) { - destination = _dictionaryLookup.GetDestination(strings[i], segments[i]); - } - - return destination; - } - - [Benchmark(OperationsPerInvoke = 100)] - public int CustomHashTable() - { - var strings = _strings; - var segments = _segments; - - var destination = 0; - for (var i = 0; i < strings.Length; i++) - { - destination = _customHashTable.GetDestination(strings[i], segments[i]); + destination = _vectorTrie.GetDestination(strings[i], segments[i]); } return destination; @@ -164,7 +155,7 @@ namespace Microsoft.AspNetCore.Routing.Matching var guid = Guid.NewGuid().ToString(); // Between 5 and 36 characters - var text = guid.Substring(0, Math.Max(5, Math.Min(count, 36))); + var text = guid.Substring(0, Math.Max(5, Math.Min(i, 36))); if (char.IsDigit(text[0])) { // Convert first character to a letter. diff --git a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs index ed59d796ed..e1e3a745e9 100644 --- a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs +++ b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs @@ -2,20 +2,30 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; namespace Microsoft.AspNetCore.Routing.Matching { public class JumpTableSingleEntryBenchmark { - private JumpTable _table; + private JumpTable _implementation; + private JumpTable _prototype; + private JumpTable _trie; + private JumpTable _vectorTrie; + private string[] _strings; private PathSegment[] _segments; [GlobalSetup] public void Setup() { - _table = new SingleEntryJumpTable(0, -1, "hello-world", 1); + _implementation = new SingleEntryJumpTable(0, -1, "hello-world", 1); + _prototype = new SingleEntryAsciiVectorizedJumpTable(0, -2, "hello-world", 1); + _trie = new ILEmitTrieJumpTable(0, -1, new [] { ("hello-world", 1), }, vectorize: false, _implementation); + _vectorTrie = new ILEmitTrieJumpTable(0, -1, new[] { ("hello-world", 1), }, vectorize: true, _implementation); + _strings = new string[] { "index/foo/2", @@ -40,21 +50,30 @@ namespace Microsoft.AspNetCore.Routing.Matching var strings = _strings; var segments = _segments; - var destination = 0; - for (var i = 0; i < strings.Length; i++) + int destination = 0; + for (int i = 0; i < strings.Length; i++) { var @string = strings[i]; var segment = segments[i]; - destination = segment.Length == 0 ? -1 : - segment.Length != 11 ? 1 : - string.Compare( + if (segment.Length == 0) + { + destination = -1; + } + else if (segment.Length != "hello-world".Length) + { + destination = 1; + } + else + { + destination = string.Compare( @string, segment.Start, "hello-world", 0, segment.Length, StringComparison.OrdinalIgnoreCase); + } } return destination; @@ -67,12 +86,234 @@ namespace Microsoft.AspNetCore.Routing.Matching var segments = _segments; var destination = 0; - for (var i = 0; i < strings.Length; i++) + for (int i = 0; i < strings.Length; i++) { - destination = _table.GetDestination(strings[i], segments[i]); + destination = _implementation.GetDestination(strings[i], segments[i]); } return destination; } + + [Benchmark(OperationsPerInvoke = 5)] + public int Prototype() + { + var strings = _strings; + var segments = _segments; + + var destination = 0; + for (int i = 0; i < strings.Length; i++) + { + destination = _prototype.GetDestination(strings[i], segments[i]); + } + + return destination; + } + + [Benchmark(OperationsPerInvoke = 5)] + public int Trie() + { + var strings = _strings; + var segments = _segments; + + var destination = 0; + for (int i = 0; i < strings.Length; i++) + { + destination = _trie.GetDestination(strings[i], segments[i]); + } + + return destination; + } + + [Benchmark(OperationsPerInvoke = 5)] + public int VectorTrie() + { + var strings = _strings; + var segments = _segments; + + var destination = 0; + for (int i = 0; i < strings.Length; i++) + { + destination = _vectorTrie.GetDestination(strings[i], segments[i]); + } + + return destination; + } + + private class SingleEntryAsciiVectorizedJumpTable : JumpTable + { + private readonly int _defaultDestination; + private readonly int _exitDestination; + private readonly string _text; + private readonly int _destination; + + private readonly ulong[] _values; + private readonly int _residue0Lower; + private readonly int _residue0Upper; + private readonly int _residue1Lower; + private readonly int _residue1Upper; + private readonly int _residue2Lower; + private readonly int _residue2Upper; + + public SingleEntryAsciiVectorizedJumpTable( + int defaultDestination, + int exitDestination, + string text, + int destination) + { + _defaultDestination = defaultDestination; + _exitDestination = exitDestination; + _text = text; + _destination = destination; + + int length = text.Length; + ReadOnlySpan span = text.ToLowerInvariant().AsSpan(); + ref byte p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + + _values = new ulong[length / 4]; + for (int i = 0; i < length / 4; i++) + { + _values[i] = Unsafe.ReadUnaligned(ref p); + p = Unsafe.Add(ref p, 64); + } + + switch (length % 4) + { + case 1: + { + var c = Unsafe.ReadUnaligned(ref p); + _residue0Lower = char.ToLowerInvariant(c); + _residue0Upper = char.ToUpperInvariant(c); + + break; + } + + case 2: + { + var c = Unsafe.ReadUnaligned(ref p); + _residue0Lower = char.ToLowerInvariant(c); + _residue0Upper = char.ToUpperInvariant(c); + + p = Unsafe.Add(ref p, 2); + c = Unsafe.ReadUnaligned(ref p); + _residue1Lower = char.ToLowerInvariant(c); + _residue1Upper = char.ToUpperInvariant(c); + + break; + } + + case 3: + { + var c = Unsafe.ReadUnaligned(ref p); + _residue0Lower = char.ToLowerInvariant(c); + _residue0Upper = char.ToUpperInvariant(c); + + p = Unsafe.Add(ref p, 2); + c = Unsafe.ReadUnaligned(ref p); + _residue1Lower = char.ToLowerInvariant(c); + _residue1Upper = char.ToUpperInvariant(c); + + p = Unsafe.Add(ref p, 2); + c = Unsafe.ReadUnaligned(ref p); + _residue2Lower = char.ToLowerInvariant(c); + _residue2Upper = char.ToUpperInvariant(c); + + break; + } + } + } + + public override int GetDestination(string path, PathSegment segment) + { + int length = segment.Length; + ReadOnlySpan span = path.AsSpan(segment.Start, length); + ref byte p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + + int i = 0; + while (length > 3) + { + var value = Unsafe.ReadUnaligned(ref p); + + if ((value & ~0x007F007F007F007FUL) == 0) + { + return _defaultDestination; + } + + ulong ulongLowerIndicator = value + (0x0080008000800080UL - 0x0041004100410041UL); + ulong ulongUpperIndicator = value + (0x0080008000800080UL - 0x005B005B005B005BUL); + ulong ulongCombinedIndicator = (ulongLowerIndicator ^ ulongUpperIndicator) & 0x0080008000800080UL; + ulong mask = (ulongCombinedIndicator) >> 2; + + value ^= mask; + + if (value != _values[i]) + { + return _defaultDestination; + } + + i++; + length -= 4; + p = ref Unsafe.Add(ref p, 64); + } + + switch (length) + { + case 1: + { + char c = Unsafe.ReadUnaligned(ref p); + if (c != _residue0Lower && c != _residue0Upper) + { + return _defaultDestination; + } + + break; + } + + case 2: + { + char c = Unsafe.ReadUnaligned(ref p); + if (c != _residue0Lower && c != _residue0Upper) + { + return _defaultDestination; + } + + p = ref Unsafe.Add(ref p, 2); + c = Unsafe.ReadUnaligned(ref p); + if (c != _residue1Lower && c != _residue1Upper) + { + return _defaultDestination; + } + + break; + } + + case 3: + { + char c = Unsafe.ReadUnaligned(ref p); + if (c != _residue0Lower && c != _residue0Upper) + { + return _defaultDestination; + } + + p = ref Unsafe.Add(ref p, 2); + c = Unsafe.ReadUnaligned(ref p); + if (c != _residue1Lower && c != _residue1Upper) + { + return _defaultDestination; + } + + p = ref Unsafe.Add(ref p, 2); + c = Unsafe.ReadUnaligned(ref p); + if (c != _residue2Lower && c != _residue2Upper) + { + return _defaultDestination; + } + + break; + } + } + + return _destination; + } + } } } diff --git a/build/dependencies.props b/build/dependencies.props index ab9fefc92d..1bb95172d2 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -38,6 +38,8 @@ 4.7.49 2.0.3 11.0.2 + 4.3.0 + 4.3.0 0.10.0 2.3.1 2.4.0 diff --git a/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs new file mode 100644 index 0000000000..f3400238c7 --- /dev/null +++ b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs @@ -0,0 +1,588 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#if IL_EMIT +using System; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Microsoft.AspNetCore.Routing.Matching +{ + internal static class ILEmitTrieFactory + { + // The algorthm we use only works for ASCII text. If we find non-ASCII text in the input + // we need to reject it and let is be processed with a fallback technique. + public const int NotAscii = Int32.MinValue; + + public static Func Create( + int defaultDestination, + int exitDestination, + (string text, int destination)[] entries, + bool? vectorize) + { + var method = new DynamicMethod( + "GetDestination", + typeof(int), + new[] { typeof(string), typeof(int), typeof(int), }); + + GenerateMethodBody(method.GetILGenerator(), defaultDestination, exitDestination, entries, vectorize); + +#if IL_EMIT_SAVE_ASSEMBLY + SaveAssembly(method.GetILGenerator(), defaultDestination, exitDestination, entries, vectorize); +#endif + + return (Func)method.CreateDelegate(typeof(Func)); + } + + private static void GenerateMethodBody( + ILGenerator il, + int defaultDestination, + int exitDestination, + (string text, int destination)[] entries, + bool? vectorize) + { + // There's no value in vectorizing the computation if we're on 32bit or + // if no string is long enough. We do the vectorized comparison with uint64 ulongs + // which isn't beneficial if they don't map to the native size of the CPU. The + // vectorized algorithm introduces additional overhead for casing. + // + // Vectorize by default on 64bit (allow override for testing) + vectorize = vectorize ?? (IntPtr.Size == 8); + + // Don't vectorize if all of the strings are small (prevents allocating unused locals) + vectorize &= entries.Any(e => e.text.Length >= 4); + + // See comments on Locals for details + var locals = new Locals(il, vectorize.Value); + + // See comments on Labels for details + var labels = new Labels() + { + ReturnDefault = il.DefineLabel(), + ReturnNotAscii = il.DefineLabel(), + }; + + // See comments on Methods for details + var methods = Methods.Instance; + + // Initializing top-level locals - this is similar to... + // ReadOnlySpan span = arg0.AsSpan(arg1, arg2); + // ref byte p = ref Unsafe.As(MemoryMarshal.GetReference(span)) + + // arg0.AsSpan(arg1, arg2) + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldarg_1); + il.Emit(OpCodes.Ldarg_2); + il.Emit(OpCodes.Call, methods.AsSpan); + + // ReadOnlySpan = ... + il.Emit(OpCodes.Stloc, locals.Span); + + // MemoryMarshal.GetReference(span) + il.Emit(OpCodes.Ldloc, locals.Span); + il.Emit(OpCodes.Call, methods.GetReference); + + // Unsafe.As(...) + il.Emit(OpCodes.Call, methods.As); + + // ref byte p = ... + il.Emit(OpCodes.Stloc_0, locals.P); + + var groups = entries.GroupBy(e => e.text.Length).ToArray(); + for (var i = 0; i < groups.Length; i++) + { + var group = groups[i]; + + // Similar to 'if (length != X) { ... } + var inside = il.DefineLabel(); + var next = il.DefineLabel(); + il.Emit(OpCodes.Ldarg_2); + il.Emit(OpCodes.Ldc_I4, group.Key); + il.Emit(OpCodes.Beq, inside); + il.Emit(OpCodes.Br, next); + + // Process the group + il.MarkLabel(inside); + EmitTable(il, group.ToArray(), 0, group.Key, locals, labels, methods); + il.MarkLabel(next); + } + + // Exit point - we end up here when the text doesn't match + il.MarkLabel(labels.ReturnDefault); + il.Emit(OpCodes.Ldc_I4, defaultDestination); + il.Emit(OpCodes.Ret); + + // Exit point - we end up here with the text contains non-ASCII text + il.MarkLabel(labels.ReturnNotAscii); + il.Emit(OpCodes.Ldc_I4, NotAscii); + il.Emit(OpCodes.Ret); + } + + private static void EmitTable( + ILGenerator il, + (string text, int destination)[] entries, + int index, + int length, + Locals locals, + Labels labels, + Methods methods) + { + // We've reached the end of the string. + if (index == length) + { + EmitReturnDestination(il, entries); + return; + } + + // If 4 or more characters remain, and we're vectorizing, we should process 4 characters at a time. + if (length - index >= 4 && locals.UInt64Value != null) + { + EmitVectorizedTable(il, entries, index, length, locals, labels, methods); + return; + } + + // Fall back to processing a character at a time. + EmitSingleCharacterTable(il, entries, index, length, locals, labels, methods); + } + + private static void EmitVectorizedTable( + ILGenerator il, + (string text, int destination)[] entries, + int index, + int length, + Locals locals, + Labels labels, + Methods methods) + { + // Emits code similar to: + // + // uint64Value = Unsafe.ReadUnaligned(ref p); + // p = ref Unsafe.Add(ref p, 8); + // + // if ((uint64Value & ~0x007F007F007F007FUL) == 0) + // { + // return NotAscii; + // } + // uint64LowerIndicator = value + (0x0080008000800080UL - 0x0041004100410041UL); + // uint64UpperIndicator = value + (0x0080008000800080UL - 0x005B005B005B005BUL); + // ulong temp1 = uint64LowerIndicator ^ uint64UpperIndicator + // ulong temp2 = temp1 & 0x0080008000800080UL; + // ulong temp3 = (temp2) >> 2; + // uint64Value = uint64Value ^ temp3; + // + // This is a vectorized non-branching technique for processing 4 utf16 characters + // at a time inside a single uint64. + // + // Similar to: + // https://github.com/GrabYourPitchforks/coreclr/commit/a3c1df25c4225995ffd6b18fd0fc39d6b81fd6a5#diff-d89b6ca07ea349899e45eed5f688a7ebR81 + // + // Basically we need to check if the text is non-ASCII first and bail if it is. + // The rest of the steps will convert the text to lowercase by checking all characters + // at a time to see if they are in the A-Z range, that's where 0x0041 and 0x005B come in. + + // IMPORTANT + // + // If you are modifying this code, be aware that the easiest way to make a mistake is by + // getting the set of casts wrong doing something like: + // + // il.Emit(OpCodes.Ldc_I8, ~0x007F007F007F007FUL); + // + // The IL Emit apis don't have overloads that accept ulong or ushort, and will resolve + // an overload that does an undesirable conversion (for instance convering ulong to float). + // + // IMPORTANT + + // Unsafe.ReadUnaligned(ref p) + il.Emit(OpCodes.Ldloc, locals.P); + il.Emit(OpCodes.Call, methods.ReadUnalignedUInt64); + + // uint64Value = ... + il.Emit(OpCodes.Stloc, locals.UInt64Value); + + // Unsafe.Add(ref p, 8) + il.Emit(OpCodes.Ldloc, locals.P); + il.Emit(OpCodes.Ldc_I4, 8); // 8 bytes were read + il.Emit(OpCodes.Call, methods.Add); + + // p = ref ... + il.Emit(OpCodes.Stloc, locals.P); + + // if ((uint64Value & ~0x007F007F007F007FUL) == 0) + // { + // goto: NotAscii; + // } + il.Emit(OpCodes.Ldloc, locals.UInt64Value); + il.Emit(OpCodes.Ldc_I8, unchecked((long)~0x007F007F007F007FUL)); + il.Emit(OpCodes.And); + il.Emit(OpCodes.Brtrue, labels.ReturnNotAscii); + + // uint64Value + (0x0080008000800080UL - 0x0041004100410041UL) + il.Emit(OpCodes.Ldloc, locals.UInt64Value); + il.Emit(OpCodes.Ldc_I8, unchecked((long)(0x0080008000800080UL - 0x0041004100410041UL))); + il.Emit(OpCodes.Add); + + // uint64LowerIndicator = ... + il.Emit(OpCodes.Stloc, locals.UInt64LowerIndicator); + + // value + (0x0080008000800080UL - 0x005B005B005B005BUL) + il.Emit(OpCodes.Ldloc, locals.UInt64Value); + il.Emit(OpCodes.Ldc_I8, unchecked((long)(0x0080008000800080UL - 0x005B005B005B005BUL))); + il.Emit(OpCodes.Add); + + // uint64UpperIndicator = ... + il.Emit(OpCodes.Stloc, locals.UInt64UpperIndicator); + + // ulongLowerIndicator ^ ulongUpperIndicator + il.Emit(OpCodes.Ldloc, locals.UInt64LowerIndicator); + il.Emit(OpCodes.Ldloc, locals.UInt64UpperIndicator); + il.Emit(OpCodes.Xor); + + // ... & 0x0080008000800080UL + il.Emit(OpCodes.Ldc_I8, unchecked((long)0x0080008000800080UL)); + il.Emit(OpCodes.And); + + // ... >> 2; + il.Emit(OpCodes.Ldc_I4, 2); + il.Emit(OpCodes.Shr_Un); + + // ... ^ uint64Value + il.Emit(OpCodes.Ldloc, locals.UInt64Value); + il.Emit(OpCodes.Xor); + + // uint64Value = ... + il.Emit(OpCodes.Stloc, locals.UInt64Value); + + // Now we generate an 'if' ladder with an entry for each of the unique 64 bit sections + // of the text. + var groups = entries.GroupBy(e => GetUInt64Key(e.text, index)); + foreach (var group in groups) + { + // if (uint64Value == 0x.....) { ... } + var next = il.DefineLabel(); + il.Emit(OpCodes.Ldloc, locals.UInt64Value); + il.Emit(OpCodes.Ldc_I8, unchecked((long)group.Key)); + il.Emit(OpCodes.Bne_Un, next); + + // Process the group + EmitTable(il, group.ToArray(), index + 4, length, locals, labels, methods); + il.MarkLabel(next); + } + + // goto: defaultDestination + il.Emit(OpCodes.Br, labels.ReturnDefault); + } + + private static void EmitSingleCharacterTable( + ILGenerator il, + (string text, int destination)[] entries, + int index, + int length, + Locals locals, + Labels labels, + Methods methods) + { + // See the vectorized code path for a much more thorough explanation. + + // IMPORTANT + // + // If you are modifying this code, be aware that the easiest way to make a mistake is by + // getting the set of casts wrong doing something like: + // + // il.Emit(OpCodes.Ldc_I4, ~0x007F); + // + // The IL Emit apis don't have overloads that accept ulong or ushort, and will resolve + // an overload that does an undesirable conversion (for instance convering ulong to float). + // + // IMPORTANT + + // Unsafe.ReadUnaligned(ref p) + il.Emit(OpCodes.Ldloc, locals.P); + il.Emit(OpCodes.Call, methods.ReadUnalignedUInt16); + + // uint16Value = ... + il.Emit(OpCodes.Stloc, locals.UInt16Value); + + // Unsafe.Add(ref p, 2) + il.Emit(OpCodes.Ldloc, locals.P); + il.Emit(OpCodes.Ldc_I4, 2); // 2 bytes were read + il.Emit(OpCodes.Call, methods.Add); + + // p = ref ... + il.Emit(OpCodes.Stloc, locals.P); + + // if ((uInt16Value & ~0x007FUL) == 0) + // { + // goto: NotAscii; + // } + il.Emit(OpCodes.Ldloc, locals.UInt16Value); + il.Emit(OpCodes.Ldc_I4, unchecked((int)((uint)~0x007F))); + il.Emit(OpCodes.And); + il.Emit(OpCodes.Brtrue, labels.ReturnNotAscii); + + // Since we're handling a single character at a time, it's easier to just + // generate an 'if' with two comparisons instead of doing complicated conversion + // logic. + + // Now we generate an 'if' ladder with an entry for each of the unique + // characters in the group. + var groups = entries.GroupBy(e => GetUInt16Key(e.text, index)); + foreach (var group in groups) + { + // if (uInt16Value == 'A' || uint16Value == 'a') { ... } + var next = il.DefineLabel(); + var inside = il.DefineLabel(); + il.Emit(OpCodes.Ldloc, locals.UInt16Value); + il.Emit(OpCodes.Ldc_I4, unchecked((int)((uint)group.Key))); + il.Emit(OpCodes.Beq, inside); + + var upper = (ushort)char.ToUpperInvariant((char)group.Key); + if (upper != group.Key) + { + il.Emit(OpCodes.Ldloc, locals.UInt16Value); + il.Emit(OpCodes.Ldc_I4, unchecked((int)((uint)upper))); + il.Emit(OpCodes.Beq, inside); + } + + il.Emit(OpCodes.Br, next); + + // Process the group + il.MarkLabel(inside); + EmitTable(il, group.ToArray(), index + 1, length, locals, labels, methods); + il.MarkLabel(next); + } + + // goto: defaultDestination + il.Emit(OpCodes.Br, labels.ReturnDefault); + } + + public static void EmitReturnDestination(ILGenerator il, (string text, int destination)[] entries) + { + Debug.Assert(entries.Length == 1, "We should have a single entry"); + il.Emit(OpCodes.Ldc_I4, entries[0].destination); + il.Emit(OpCodes.Ret); + } + + private static ulong GetUInt64Key(string text, int index) + { + Debug.Assert(index + 4 <= text.Length); + var span = text.ToLowerInvariant().AsSpan(index); + ref var p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + return Unsafe.ReadUnaligned(ref p); + } + + private static ushort GetUInt16Key(string text, int index) + { + Debug.Assert(index + 1 <= text.Length); + return (ushort)char.ToLowerInvariant(text[index]); + } + + // We require a special build-time define since this is a testing/debugging + // feature that will litter the app directory with assemblies. +#if IL_EMIT_SAVE_ASSEMBLY + private static void SaveAssembly( + int defaultDestination, + int exitDestination, + (string text, int destination)[] entries, + bool? vectorize) + { + var assemblyName = "Microsoft.AspNetCore.Routing.ILEmitTrie" + DateTime.Now.Ticks; + var fileName = assemblyName + ".dll"; + var assembly = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(assemblyName), AssemblyBuilderAccess.RunAndSave); + var module = assembly.DefineDynamicModule(assemblyName, fileName); + var type = module.DefineType("ILEmitTrie"); + var method = type.DefineMethod( + "GetDestination", + MethodAttributes.Public | MethodAttributes.Static, + CallingConventions.Standard, + typeof(int), + new [] { typeof(string), typeof(int), typeof(int), }; + + GenerateMethodBody(method.GetILGenerator(), defaultDestination, exitDestination, entries, vectorize); + + type.CreateTypeInfo(); + assembly.Save(fileName); + } +#endif + + private class Locals + { + public Locals(ILGenerator il, bool vectorize) + { + P = il.DeclareLocal(typeof(byte).MakeByRefType()); + Span = il.DeclareLocal(typeof(ReadOnlySpan)); + + UInt16Value = il.DeclareLocal(typeof(ushort)); + + if (vectorize) + { + UInt64Value = il.DeclareLocal(typeof(ulong)); + UInt64LowerIndicator = il.DeclareLocal(typeof(ulong)); + UInt64UpperIndicator = il.DeclareLocal(typeof(ulong)); + } + } + + /// + /// Holds current character when processing a character at a time. + /// + public LocalBuilder UInt16Value { get; set; } + + /// + /// Holds current character when processing 4 characters at a time. + /// + public LocalBuilder UInt64Value { get; set; } + + /// + /// Used to covert casing. See comments where it's used. + /// + public LocalBuilder UInt64LowerIndicator { get; set; } + + /// + /// Used to covert casing. See comments where it's used. + /// + public LocalBuilder UInt64UpperIndicator { get; set; } + + /// + /// Holds a 'ref byte' reference to the current character (in bytes). + /// + public LocalBuilder P { get; set; } + + /// + /// Holds the relevant portion of the path as a Span[byte]. + /// + public LocalBuilder Span { get; set; } + } + + private class Labels + { + /// + /// Label to goto that will return the default destination (not a match). + /// + public Label ReturnDefault { get; set; } + + /// + /// Label to goto that will return a sentinel value for non-ascii text. + /// + public Label ReturnNotAscii { get; set; } + } + + private class Methods + { + public static readonly Methods Instance = new Methods(); + + private Methods() + { + // Can't use GetMethod because the parameter is a generic method parameters. + Add = typeof(Unsafe) + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Unsafe.Add)) + .Where(m => m.GetGenericArguments().Length == 1) + .Where(m => m.GetParameters().Length == 2) + .FirstOrDefault() + ?.MakeGenericMethod(typeof(byte)); + if (Add == null) + { + throw new InvalidOperationException("Failed to find Unsafe.Add{T}(ref T, int)"); + } + + // Can't use GetMethod because the parameter is a generic method parameters. + As = typeof(Unsafe) + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Unsafe.As)) + .Where(m => m.GetGenericArguments().Length == 2) + .Where(m => m.GetParameters().Length == 1) + .FirstOrDefault() + ?.MakeGenericMethod(typeof(char), typeof(byte)); + if (Add == null) + { + throw new InvalidOperationException("Failed to find Unsafe.As{TFrom, TTo}(ref TFrom)"); + } + + AsSpan = typeof(MemoryExtensions).GetMethod( + nameof(MemoryExtensions.AsSpan), + BindingFlags.Public | BindingFlags.Static, + binder: null, + new[] { typeof(string), typeof(int), typeof(int), }, + modifiers: null); + if (AsSpan == null) + { + throw new InvalidOperationException("Failed to find MemoryExtensions.AsSpan(string, int, int)"); + } + + // Can't use GetMethod because the parameter is a generic method parameters. + GetReference = typeof(MemoryMarshal) + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(MemoryMarshal.GetReference)) + .Where(m => m.GetGenericArguments().Length == 1) + .Where(m => m.GetParameters().Length == 1) + // Disambiguate between ReadOnlySpan<> and Span<> - this method is overloaded. + .Where(m => m.GetParameters()[0].ParameterType.GetGenericTypeDefinition() == typeof(ReadOnlySpan<>)) + .FirstOrDefault() + ?.MakeGenericMethod(typeof(char)); + if (GetReference == null) + { + throw new InvalidOperationException("Failed to find MemoryMarshal.GetReference{T}(ReadOnlySpan{T})"); + } + + ReadUnalignedUInt64 = typeof(Unsafe).GetMethod( + nameof(Unsafe.ReadUnaligned), + BindingFlags.Public | BindingFlags.Static, + binder: null, + new[] { typeof(byte).MakeByRefType(), }, + modifiers: null) + .MakeGenericMethod(typeof(ulong)); + if (ReadUnalignedUInt64 == null) + { + throw new InvalidOperationException("Failed to find Unsafe.ReadUnaligned{T}(ref byte)"); + } + + ReadUnalignedUInt16 = typeof(Unsafe).GetMethod( + nameof(Unsafe.ReadUnaligned), + BindingFlags.Public | BindingFlags.Static, + binder: null, + new[] { typeof(byte).MakeByRefType(), }, + modifiers: null) + .MakeGenericMethod(typeof(ushort)); + if (ReadUnalignedUInt16 == null) + { + throw new InvalidOperationException("Failed to find Unsafe.ReadUnaligned{T}(ref byte)"); + } + } + + /// + /// - Add[ref byte] + /// + public MethodInfo Add { get; } + + /// + /// - As[char, byte] + /// + public MethodInfo As { get; } + + /// + /// + /// + public MethodInfo AsSpan { get; } + + /// + /// - GetReference[char] + /// + public MethodInfo GetReference { get; } + + /// + /// - ReadUnaligned[ulong] + /// + public MethodInfo ReadUnalignedUInt64 { get; } + + /// + /// - ReadUnaligned[ushort] + /// + public MethodInfo ReadUnalignedUInt16 { get; } + } + } +} + +#endif diff --git a/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs new file mode 100644 index 0000000000..2bd5951aca --- /dev/null +++ b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs @@ -0,0 +1,102 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +#if IL_EMIT + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Routing.Matching +{ + // Uses generated IL to implement the JumpTable contract. This approach requires + // a fallback jump table for two reasons: + // 1. We compute the IL lazily to avoid taking up significant time when processing a request + // 2. The generated IL only supports ASCII in the URL path + internal class ILEmitTrieJumpTable : JumpTable + { + private const int NotAscii = int.MinValue; + + private readonly int _defaultDestination; + private readonly int _exitDestination; + private readonly (string text, int destination)[] _entries; + + private readonly bool? _vectorize; + private readonly JumpTable _fallback; + + // Used to protect the initialization of the compiled delegate + private object _lock; + private bool _initializing; + private Task _task; + + // Will be replaced at runtime by the generated code. + // + // Internal for testing + internal Func _getDestination; + + public ILEmitTrieJumpTable( + int defaultDestination, + int exitDestination, + (string text, int destination)[] entries, + bool? vectorize, + JumpTable fallback) + { + _defaultDestination = defaultDestination; + _exitDestination = exitDestination; + _entries = entries; + _vectorize = vectorize; + _fallback = fallback; + + _getDestination = FallbackGetDestination; + } + + public override int GetDestination(string path, PathSegment segment) + { + return _getDestination(path, segment); + } + + private int FallbackGetDestination(string path, PathSegment segment) + { + if (path.Length == 0) + { + return _exitDestination; + } + + // We only hit this code path if the IL delegate is still initializing. + LazyInitializer.EnsureInitialized(ref _task, ref _initializing, ref _lock, InitializeILDelegateAsync); + + return _fallback.GetDestination(path, segment); + } + + // Internal for testing + internal async Task InitializeILDelegateAsync() + { + // Offload the creation of the IL delegate to the thread pool. + await Task.Run(() => + { + InitializeILDelegate(); + }); + } + + // Internal for testing + internal void InitializeILDelegate() + { + var generated = ILEmitTrieFactory.Create(_defaultDestination, _exitDestination, _entries, _vectorize); + _getDestination = (string path, PathSegment segment) => + { + if (segment.Length == 0) + { + return _exitDestination; + } + + var result = generated(path, segment.Start, segment.Length); + if (result == ILEmitTrieFactory.NotAscii) + { + result = _fallback.GetDestination(path, segment); + } + + return result; + }; + } + } +} +#endif diff --git a/src/Microsoft.AspNetCore.Routing/Matching/JumpTableBuilder.cs b/src/Microsoft.AspNetCore.Routing/Matching/JumpTableBuilder.cs index 2800b5eca8..dcd5e3abc3 100644 --- a/src/Microsoft.AspNetCore.Routing/Matching/JumpTableBuilder.cs +++ b/src/Microsoft.AspNetCore.Routing/Matching/JumpTableBuilder.cs @@ -38,25 +38,65 @@ namespace Microsoft.AspNetCore.Routing.Matching throw new InvalidOperationException(message); } - // The JumpTable implementation is chosen based on the number of entries. Right - // now this is simple and minimal. + // The JumpTable implementation is chosen based on the number of entries. + // + // Basically the concerns that we're juggling here are that different implementations + // make sense depending on the characteristics of the entries. + // + // On netcoreapp we support IL generation of optimized tries that is much faster + // than anything we can do with string.Compare or dictionaries. However the IL emit + // strategy requires us to produce a fallback jump table - see comments on the class. + + // We have an optimized fast path for zero entries since we don't have to + // do any string comparisons. if (_entries.Count == 0) { return new ZeroEntryJumpTable(DefaultDestination, ExitDestination); } + // The IL Emit jump table is not faster for a single entry if (_entries.Count == 1) { var entry = _entries[0]; return new SingleEntryJumpTable(DefaultDestination, ExitDestination, entry.text, entry.destination); } - if (_entries.Count < 10) + // We choose a hard upper bound of 100 as the limit for when we switch to a dictionary + // over a trie. The reason is that while the dictionary has a bigger constant factor, + // it is O(1) vs a trie which is O(M * log(N)). Our perf testing shows that the trie + // is better for ~90 entries based on all of Azure's route table. Anything above 100 edges + // we'd consider to be a very very large node, and so while we don't think anyone will + // have a node this large in practice, we want to make sure the performance is reasonable + // for any size. + // + // Additionally if we're on 32bit, the scalability is worse, so switch to the dictionary at 50 + // entries. + var threshold = IntPtr.Size == 8 ? 100 : 50; + if (_entries.Count >= threshold) { - return new LinearSearchJumpTable(DefaultDestination, ExitDestination, _entries.ToArray()); + return new DictionaryJumpTable(DefaultDestination, ExitDestination, _entries.ToArray()); } - return new DictionaryJumpTable(DefaultDestination, ExitDestination, _entries.ToArray()); + // If we have more than a single string, the IL emit strategy is the fastest - but we need to decide + // what do for the fallback case. + JumpTable fallback; + + // Based on our testing a linear search is still faster than a dictionary at ten entries. + if (_entries.Count <= 10) + { + fallback = new LinearSearchJumpTable(DefaultDestination, ExitDestination, _entries.ToArray()); + } + else + { + fallback = new DictionaryJumpTable(DefaultDestination, ExitDestination, _entries.ToArray()); + } + +#if IL_EMIT + + return new ILEmitTrieJumpTable(DefaultDestination, ExitDestination, _entries.ToArray(), vectorize: null, fallback); +#else + return fallback; +#endif } } } diff --git a/src/Microsoft.AspNetCore.Routing/Microsoft.AspNetCore.Routing.csproj b/src/Microsoft.AspNetCore.Routing/Microsoft.AspNetCore.Routing.csproj index fa1ae2e1ac..495fe45caa 100644 --- a/src/Microsoft.AspNetCore.Routing/Microsoft.AspNetCore.Routing.csproj +++ b/src/Microsoft.AspNetCore.Routing/Microsoft.AspNetCore.Routing.csproj @@ -4,13 +4,25 @@ Commonly used types: Microsoft.AspNetCore.Routing.Route Microsoft.AspNetCore.Routing.RouteCollection - netstandard2.0 + netstandard2.0;netcoreapp2.2 $(NoWarn);CS1591 true aspnetcore;routing true + + + true + false + IL_EMIT;$(DefineConstants) + IL_EMIT_SAVE_ASSEMBLIES;$(DefineConstants) + + @@ -26,5 +38,7 @@ Microsoft.AspNetCore.Routing.RouteCollection + + diff --git a/test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieJumpTableTest.cs b/test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieJumpTableTest.cs new file mode 100644 index 0000000000..5106b40beb --- /dev/null +++ b/test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieJumpTableTest.cs @@ -0,0 +1,228 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#if IL_EMIT +using Moq; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.AspNetCore.Routing.Matching +{ + // We get a lot of good coverage of basics since this implementation is used + // as the default in many cases. The tests here are focused on details of the + // implementation (boundaries, casing, non-ASCII). + public abstract class ILEmitTreeJumpTableTestBase : MultipleEntryJumpTableTest + { + public abstract bool Vectorize { get; } + + internal override JumpTable CreateTable( + int defaultDestination, + int exitDestination, + params (string text, int destination)[] entries) + { + var fallback = new DictionaryJumpTable(defaultDestination, exitDestination, entries); + var table = new ILEmitTrieJumpTable(defaultDestination, exitDestination, entries, Vectorize, fallback); + table.InitializeILDelegate(); + return table; + } + + [Fact] // Not calling CreateTable here because we want to test the initialization + public async Task InitializeILDelegateAsync_ReplacesDelegate() + { + // Arrange + var table = new ILEmitTrieJumpTable(0, -1, new[] { ("hi", 1), }, Vectorize, Mock.Of()); + var original = table._getDestination; + + // Act + await table.InitializeILDelegateAsync(); + + // Assert + Assert.NotSame(original, table._getDestination); + } + + // Tests that we can detect non-ASCII characters and use the fallback jump table. + // Testing different indices since that affects which part of the code is running. + // \u007F = lowest non-ASCII character + // \uFFFF = highest non-ASCII character + [Theory] + + // non-ASCII character in first section non-vectorized comparisons + [InlineData("he\u007F", "he\u007Flo-world", 0, 3)] + [InlineData("he\uFFFF", "he\uFFFFlo-world", 0, 3)] + [InlineData("e\u007F", "he\u007Flo-world", 1, 2)] + [InlineData("e\uFFFF", "he\uFFFFlo-world", 1, 2)] + [InlineData("\u007F", "he\u007Flo-world", 2, 1)] + [InlineData("\uFFFF", "he\uFFFFlo-world", 2, 1)] + + // non-ASCII character in first section vectorized comparions + [InlineData("hel\u007F", "hel\u007Fo-world", 0, 4)] + [InlineData("hel\uFFFF", "hel\uFFFFo-world", 0, 4)] + [InlineData("el\u007Fo", "hel\u007Fo-world", 1, 4)] + [InlineData("el\uFFFFo", "hel\uFFFFo-world", 1, 4)] + [InlineData("l\u007Fo-", "hel\u007Fo-world", 2, 4)] + [InlineData("l\uFFFFo-", "hel\uFFFFo-world", 2, 4)] + [InlineData("\u007Fo-w", "hel\u007Fo-world", 3, 4)] + [InlineData("\uFFFFo-w", "hel\uFFFFo-world", 3, 4)] + + // non-ASCII character in second section non-vectorized comparisons + [InlineData("hello-\u007F", "hello-\u007Forld", 0, 7)] + [InlineData("hello-\uFFFF", "hello-\uFFFForld", 0, 7)] + [InlineData("ello-\u007F", "hello-\u007Forld", 1, 6)] + [InlineData("ello-\uFFFF", "hello-\uFFFForld", 1, 6)] + [InlineData("llo-\u007F", "hello-\u007Forld", 2, 5)] + [InlineData("llo-\uFFFF", "hello-\uFFFFForld", 2, 5)] + + // non-ASCII character in first section vectorized comparions + [InlineData("hello-w\u007F", "hello-w\u007Forld", 0, 8)] + [InlineData("hello-w\uFFFF", "hello-w\uFFFForld", 0, 8)] + [InlineData("ello-w\u007Fo", "hello-w\u007Forld", 1, 8)] + [InlineData("ello-w\uFFFFo", "hello-w\uFFFForld", 1, 8)] + [InlineData("llo-w\u007For", "hello-w\u007Forld", 2, 8)] + [InlineData("llo-w\uFFFFor", "hello-w\uFFFForld", 2, 8)] + [InlineData("lo-w\u007Forl", "hello-w\u007Forld", 3, 8)] + [InlineData("lo-w\uFFFForl", "hello-w\uFFFForld", 3, 8)] + public void GetDestination_Found_IncludesNonAsciiCharacters(string entry, string path, int start, int length) + { + // Makes it easy to spot invalid tests + Assert.Equal(entry.Length, length); + Assert.Equal(entry, path.Substring(start, length), ignoreCase: true); + + // Arrange + var table = CreateTable(0, -1, new[] { (entry, 1), }); + + var segment = new PathSegment(start, length); + + // Act + var result = table.GetDestination(path, segment); + + // Assert + Assert.Equal(1, result); + } + + // Tests for difference in casing with ASCII casing rules. Verifies our case + // manipulation algorthm is correct. + // + // We convert from upper case to lower + // 'A' and 'a' are 32 bits apart at the low end + // 'Z' and 'z' are 32 bits apart at the high end + [Theory] + + // character in first section non-vectorized comparisons + [InlineData("heA", "healo-world", 0, 3)] + [InlineData("heZ", "hezlo-world", 0, 3)] + [InlineData("eA", "healo-world", 1, 2)] + [InlineData("eZ", "hezlo-world", 1, 2)] + [InlineData("A", "healo-world", 2, 1)] + [InlineData("Z", "hezlo-world", 2, 1)] + + // character in first section vectorized comparions + [InlineData("helA", "helao-world", 0, 4)] + [InlineData("helZ", "helzo-world", 0, 4)] + [InlineData("elAo", "helao-world", 1, 4)] + [InlineData("elZo", "helzo-world", 1, 4)] + [InlineData("lAo-", "helao-world", 2, 4)] + [InlineData("lZo-", "helzo-world", 2, 4)] + [InlineData("Ao-w", "helao-world", 3, 4)] + [InlineData("Zo-w", "helzo-world", 3, 4)] + + // character in second section non-vectorized comparisons + [InlineData("hello-A", "hello-aorld", 0, 7)] + [InlineData("hello-Z", "hello-zorld", 0, 7)] + [InlineData("ello-A", "hello-aorld", 1, 6)] + [InlineData("ello-Z", "hello-zorld", 1, 6)] + [InlineData("llo-A", "hello-aorld", 2, 5)] + [InlineData("llo-Z", "hello-zForld", 2, 5)] + + // character in first section vectorized comparions + [InlineData("hello-wA", "hello-waorld", 0, 8)] + [InlineData("hello-wZ", "hello-wzorld", 0, 8)] + [InlineData("ello-wAo", "hello-waorld", 1, 8)] + [InlineData("ello-wZo", "hello-wzorld", 1, 8)] + [InlineData("llo-wAor", "hello-waorld", 2, 8)] + [InlineData("llo-wZor", "hello-wzorld", 2, 8)] + [InlineData("lo-wAorl", "hello-waorld", 3, 8)] + [InlineData("lo-wZorl", "hello-wzorld", 3, 8)] + public void GetDestination_Found_IncludesCharactersWithCasingDifference(string entry, string path, int start, int length) + { + // Makes it easy to spot invalid tests + Assert.Equal(entry.Length, length); + Assert.Equal(entry, path.Substring(start, length), ignoreCase: true); + + // Arrange + var table = CreateTable(0, -1, new[] { (entry, 1), }); + + var segment = new PathSegment(start, length); + + // Act + var result = table.GetDestination(path, segment); + + // Assert + Assert.Equal(1, result); + } + + // Tests for difference in casing with ASCII casing rules. Verifies our case + // manipulation algorthm is correct. + // + // We convert from upper case to lower + // '@' and '`' are 32 bits apart at the low end + // '[' and '}' are 32 bits apart at the high end + // + // How to understand these tests: + // "an @ should not be converted to a ` since it is out of range" + [Theory] + + // character in first section non-vectorized comparisons + [InlineData("he@", "he`lo-world", 0, 3)] + [InlineData("he[", "he{lo-world", 0, 3)] + [InlineData("e@", "he`lo-world", 1, 2)] + [InlineData("e[", "he{lo-world", 1, 2)] + [InlineData("@", "he`lo-world", 2, 1)] + [InlineData("[", "he{lo-world", 2, 1)] + + // character in first section vectorized comparions + [InlineData("hel@", "hel`o-world", 0, 4)] + [InlineData("hel[", "hel{o-world", 0, 4)] + [InlineData("el@o", "hel`o-world", 1, 4)] + [InlineData("el[o", "hel{o-world", 1, 4)] + [InlineData("l@o-", "hel`o-world", 2, 4)] + [InlineData("l[o-", "hel{o-world", 2, 4)] + [InlineData("@o-w", "hel`o-world", 3, 4)] + [InlineData("[o-w", "hel{o-world", 3, 4)] + + // character in second section non-vectorized comparisons + [InlineData("hello-@", "hello-`orld", 0, 7)] + [InlineData("hello-[", "hello-{orld", 0, 7)] + [InlineData("ello-@", "hello-`orld", 1, 6)] + [InlineData("ello-[", "hello-{orld", 1, 6)] + [InlineData("llo-@", "hello-`orld", 2, 5)] + [InlineData("llo-[", "hello-{Forld", 2, 5)] + + // character in first section vectorized comparions + [InlineData("hello-w@", "hello-w`orld", 0, 8)] + [InlineData("hello-w[", "hello-w{orld", 0, 8)] + [InlineData("ello-w@o", "hello-w`orld", 1, 8)] + [InlineData("ello-w[o", "hello-w{orld", 1, 8)] + [InlineData("llo-w@or", "hello-w`orld", 2, 8)] + [InlineData("llo-w[or", "hello-w{orld", 2, 8)] + [InlineData("lo-w@orl", "hello-w`orld", 3, 8)] + [InlineData("lo-w[orl", "hello-w{orld", 3, 8)] + public void GetDestination_NotFound_IncludesCharactersWithCasingDifference(string entry, string path, int start, int length) + { + // Makes it easy to spot invalid tests + Assert.Equal(entry.Length, length); + Assert.NotEqual(entry, path.Substring(start, length)); + + // Arrange + var table = CreateTable(0, -1, new[] { (entry, 1), }); + + var segment = new PathSegment(start, length); + + // Act + var result = table.GetDestination(path, segment); + + // Assert + Assert.Equal(0, result); + } + } +} +#endif diff --git a/test/Microsoft.AspNetCore.Routing.Tests/Matching/NonVectorizedILEmitTrieJumpTableTest.cs b/test/Microsoft.AspNetCore.Routing.Tests/Matching/NonVectorizedILEmitTrieJumpTableTest.cs new file mode 100644 index 0000000000..3e5b59baf6 --- /dev/null +++ b/test/Microsoft.AspNetCore.Routing.Tests/Matching/NonVectorizedILEmitTrieJumpTableTest.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#if IL_EMIT +namespace Microsoft.AspNetCore.Routing.Matching +{ + public class NonVectorizedILEmitTrieJumpTableTest : ILEmitTreeJumpTableTestBase + { + public override bool Vectorize => false; + } +} +#endif diff --git a/test/Microsoft.AspNetCore.Routing.Tests/Matching/VectorizedILEmitTrieJumpTableTest.cs b/test/Microsoft.AspNetCore.Routing.Tests/Matching/VectorizedILEmitTrieJumpTableTest.cs new file mode 100644 index 0000000000..fc5e7c555a --- /dev/null +++ b/test/Microsoft.AspNetCore.Routing.Tests/Matching/VectorizedILEmitTrieJumpTableTest.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#if IL_EMIT +namespace Microsoft.AspNetCore.Routing.Matching +{ + public class VectorizedILEmitTrieJumpTableTest : ILEmitTreeJumpTableTestBase + { + // We can still run the vectorized implementation on 32 bit, we just + // don't expect it to be performant - it will still be correct. + public override bool Vectorize => true; + } +} +#endif diff --git a/test/Microsoft.AspNetCore.Routing.Tests/Microsoft.AspNetCore.Routing.Tests.csproj b/test/Microsoft.AspNetCore.Routing.Tests/Microsoft.AspNetCore.Routing.Tests.csproj index 0a862d57d1..57c4d063fa 100644 --- a/test/Microsoft.AspNetCore.Routing.Tests/Microsoft.AspNetCore.Routing.Tests.csproj +++ b/test/Microsoft.AspNetCore.Routing.Tests/Microsoft.AspNetCore.Routing.Tests.csproj @@ -3,10 +3,19 @@ $(StandardTestTfms) Microsoft.AspNetCore.Routing + true + + + + + true + IL_EMIT;$(DefineConstants) - true @@ -19,6 +28,7 @@ + From 8d053853bba7681e4ad90999ab979675bb115e59 Mon Sep 17 00:00:00 2001 From: Ryan Nowak Date: Mon, 27 Aug 2018 14:52:16 -0700 Subject: [PATCH 2/2] Address PR feedback, I hit merge too soon. --- .../Matching/JumpTableSingleEntryBenchmark.cs | 37 +++++++------ .../Matching/ILEmitTrieFactory.cs | 46 ++++++++++------ .../Matching/ILEmitTrieJumpTable.cs | 2 + .../Matching/ILEmitTrieFactoryTest.cs | 53 +++++++++++++++++++ 4 files changed, 102 insertions(+), 36 deletions(-) create mode 100644 test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieFactoryTest.cs diff --git a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs index e1e3a745e9..690c1788b8 100644 --- a/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs +++ b/benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/JumpTableSingleEntryBenchmark.cs @@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Routing.Matching { _implementation = new SingleEntryJumpTable(0, -1, "hello-world", 1); _prototype = new SingleEntryAsciiVectorizedJumpTable(0, -2, "hello-world", 1); - _trie = new ILEmitTrieJumpTable(0, -1, new [] { ("hello-world", 1), }, vectorize: false, _implementation); + _trie = new ILEmitTrieJumpTable(0, -1, new[] { ("hello-world", 1), }, vectorize: false, _implementation); _vectorTrie = new ILEmitTrieJumpTable(0, -1, new[] { ("hello-world", 1), }, vectorize: true, _implementation); _strings = new string[] @@ -116,7 +116,7 @@ namespace Microsoft.AspNetCore.Routing.Matching var segments = _segments; var destination = 0; - for (int i = 0; i < strings.Length; i++) + for (var i = 0; i < strings.Length; i++) { destination = _trie.GetDestination(strings[i], segments[i]); } @@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Routing.Matching var segments = _segments; var destination = 0; - for (int i = 0; i < strings.Length; i++) + for (var i = 0; i < strings.Length; i++) { destination = _vectorTrie.GetDestination(strings[i], segments[i]); } @@ -165,17 +165,16 @@ namespace Microsoft.AspNetCore.Routing.Matching _text = text; _destination = destination; - int length = text.Length; - ReadOnlySpan span = text.ToLowerInvariant().AsSpan(); - ref byte p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + var length = text.Length; + var span = text.ToLowerInvariant().AsSpan(); + ref var p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); _values = new ulong[length / 4]; - for (int i = 0; i < length / 4; i++) + for (var i = 0; i < length / 4; i++) { _values[i] = Unsafe.ReadUnaligned(ref p); p = Unsafe.Add(ref p, 64); } - switch (length % 4) { case 1: @@ -224,11 +223,11 @@ namespace Microsoft.AspNetCore.Routing.Matching public override int GetDestination(string path, PathSegment segment) { - int length = segment.Length; - ReadOnlySpan span = path.AsSpan(segment.Start, length); - ref byte p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + var length = segment.Length; + var span = path.AsSpan(segment.Start, length); + ref var p = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); - int i = 0; + var i = 0; while (length > 3) { var value = Unsafe.ReadUnaligned(ref p); @@ -238,10 +237,10 @@ namespace Microsoft.AspNetCore.Routing.Matching return _defaultDestination; } - ulong ulongLowerIndicator = value + (0x0080008000800080UL - 0x0041004100410041UL); - ulong ulongUpperIndicator = value + (0x0080008000800080UL - 0x005B005B005B005BUL); - ulong ulongCombinedIndicator = (ulongLowerIndicator ^ ulongUpperIndicator) & 0x0080008000800080UL; - ulong mask = (ulongCombinedIndicator) >> 2; + var ulongLowerIndicator = value + (0x0080008000800080UL - 0x0041004100410041UL); + var ulongUpperIndicator = value + (0x0080008000800080UL - 0x005B005B005B005BUL); + var ulongCombinedIndicator = (ulongLowerIndicator ^ ulongUpperIndicator) & 0x0080008000800080UL; + var mask = (ulongCombinedIndicator) >> 2; value ^= mask; @@ -259,7 +258,7 @@ namespace Microsoft.AspNetCore.Routing.Matching { case 1: { - char c = Unsafe.ReadUnaligned(ref p); + var c = Unsafe.ReadUnaligned(ref p); if (c != _residue0Lower && c != _residue0Upper) { return _defaultDestination; @@ -270,7 +269,7 @@ namespace Microsoft.AspNetCore.Routing.Matching case 2: { - char c = Unsafe.ReadUnaligned(ref p); + var c = Unsafe.ReadUnaligned(ref p); if (c != _residue0Lower && c != _residue0Upper) { return _defaultDestination; @@ -288,7 +287,7 @@ namespace Microsoft.AspNetCore.Routing.Matching case 3: { - char c = Unsafe.ReadUnaligned(ref p); + var c = Unsafe.ReadUnaligned(ref p); if (c != _residue0Lower && c != _residue0Upper) { return _defaultDestination; diff --git a/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs index f3400238c7..3b3528932f 100644 --- a/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs +++ b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieFactory.cs @@ -16,8 +16,11 @@ namespace Microsoft.AspNetCore.Routing.Matching { // The algorthm we use only works for ASCII text. If we find non-ASCII text in the input // we need to reject it and let is be processed with a fallback technique. - public const int NotAscii = Int32.MinValue; + public const int NotAscii = int.MinValue; + // Creates a Func of (string path, int start, int length) => destination + // Not using PathSegment here because we don't want to mess with visibility checks and + // generating IL without it is easier. public static Func Create( int defaultDestination, int exitDestination, @@ -38,6 +41,21 @@ namespace Microsoft.AspNetCore.Routing.Matching return (Func)method.CreateDelegate(typeof(Func)); } + // Internal for testing + internal static bool ShouldVectorize((string text, int destination)[] entries) + { + // There's no value in vectorizing the computation if we're on 32bit or + // if no string is long enough. We do the vectorized comparison with uint64 ulongs + // which isn't beneficial if they don't map to the native size of the CPU. The + // vectorized algorithm introduces additional overhead for casing. + + // Vectorize by default on 64bit (allow override for testing) + return (IntPtr.Size == 8) && + + // Don't vectorize if all of the strings are small (prevents allocating unused locals) + entries.Any(e => e.text.Length >= 4); + } + private static void GenerateMethodBody( ILGenerator il, int defaultDestination, @@ -45,16 +63,8 @@ namespace Microsoft.AspNetCore.Routing.Matching (string text, int destination)[] entries, bool? vectorize) { - // There's no value in vectorizing the computation if we're on 32bit or - // if no string is long enough. We do the vectorized comparison with uint64 ulongs - // which isn't beneficial if they don't map to the native size of the CPU. The - // vectorized algorithm introduces additional overhead for casing. - // - // Vectorize by default on 64bit (allow override for testing) - vectorize = vectorize ?? (IntPtr.Size == 8); - // Don't vectorize if all of the strings are small (prevents allocating unused locals) - vectorize &= entries.Any(e => e.text.Length >= 4); + vectorize = vectorize ?? ShouldVectorize(entries); // See comments on Locals for details var locals = new Locals(il, vectorize.Value); @@ -428,32 +438,32 @@ namespace Microsoft.AspNetCore.Routing.Matching /// /// Holds current character when processing a character at a time. /// - public LocalBuilder UInt16Value { get; set; } + public LocalBuilder UInt16Value { get; } /// /// Holds current character when processing 4 characters at a time. /// - public LocalBuilder UInt64Value { get; set; } + public LocalBuilder UInt64Value { get; } /// /// Used to covert casing. See comments where it's used. /// - public LocalBuilder UInt64LowerIndicator { get; set; } - + public LocalBuilder UInt64LowerIndicator { get; } + /// /// Used to covert casing. See comments where it's used. /// - public LocalBuilder UInt64UpperIndicator { get; set; } + public LocalBuilder UInt64UpperIndicator { get; } /// /// Holds a 'ref byte' reference to the current character (in bytes). /// - public LocalBuilder P { get; set; } + public LocalBuilder P { get; } /// /// Holds the relevant portion of the path as a Span[byte]. /// - public LocalBuilder Span { get; set; } + public LocalBuilder Span { get; } } private class Labels @@ -471,6 +481,8 @@ namespace Microsoft.AspNetCore.Routing.Matching private class Methods { + // Caching because the methods won't change, if we're being called once we're likely to + // be called again. public static readonly Methods Instance = new Methods(); private Methods() diff --git a/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs index 2bd5951aca..3bd29c9695 100644 --- a/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs +++ b/src/Microsoft.AspNetCore.Routing/Matching/ILEmitTrieJumpTable.cs @@ -54,6 +54,8 @@ namespace Microsoft.AspNetCore.Routing.Matching return _getDestination(path, segment); } + // Used when we haven't yet initialized the IL trie. We defer compilation of the IL for startup + // performance. private int FallbackGetDestination(string path, PathSegment segment) { if (path.Length == 0) diff --git a/test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieFactoryTest.cs b/test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieFactoryTest.cs new file mode 100644 index 0000000000..738646dc11 --- /dev/null +++ b/test/Microsoft.AspNetCore.Routing.Tests/Matching/ILEmitTrieFactoryTest.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#if IL_EMIT +using System; +using Xunit; + +namespace Microsoft.AspNetCore.Routing.Matching +{ + public class ILEmitTrieFactoryTest + { + // We never vectorize on 32bit, so that's part of the test. + [Fact] + public void ShouldVectorize_ReturnsTrue_ForLargeEnoughStrings() + { + // Arrange + var is64Bit = IntPtr.Size == 8; + var expected = is64Bit; + + var entries = new[] + { + ("foo", 0), + ("badr", 0), + ("", 0), + }; + + // Act + var actual = ILEmitTrieFactory.ShouldVectorize(entries); + + // Assert + Assert.Equal(expected, actual); + } + + [Fact] + public void ShouldVectorize_ReturnsFalseForSmallStrings() + { + // Arrange + var entries = new[] + { + ("foo", 0), + ("sma", 0), + ("", 0), + }; + + // Act + var actual = ILEmitTrieFactory.ShouldVectorize(entries); + + // Assert + Assert.False(actual); + } + } +} +#endif