Tokenize Less

benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/MatcherFindCandidateSetAzureBenchmark.cs

@@ -9,6 +9,10 @@ namespace Microsoft.AspNetCore.Routing.Matching
     // Generated from https://github.com/Azure/azure-rest-api-specs
     public partial class MatcherFindCandidateSetAzureBenchmark : MatcherBenchmarkBase
     {
+        // SegmentCount should be max-segments + 1, but we don't have a good way to compute
+        // it here, so using 16 as a safe guess.
+        private const int SegmentCount = 16;
+
         private const int SampleCount = 100;
 
         private BarebonesMatcher _baseline;
@@ -58,7 +62,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                 var httpContext = Requests[sample];
 
                 var path = httpContext.Request.Path.Value;
-                Span<PathSegment> segments = stackalloc PathSegment[FastPathTokenizer.DefaultSegmentCount];
+                Span<PathSegment> segments = stackalloc PathSegment[SegmentCount];
                 var count = FastPathTokenizer.Tokenize(path, segments);
 
                 var candidates = _dfa.FindCandidateSet(httpContext, path, segments.Slice(0, count));

benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/MatcherFindCandidateSetGithubBenchmark.cs

@@ -11,6 +11,10 @@ namespace Microsoft.AspNetCore.Routing.Matching
     // Use https://editor2.swagger.io/ to convert from yaml to json-
     public partial class MatcherFindCandidateSetGithubBenchmark : MatcherBenchmarkBase
     {
+        // SegmentCount should be max-segments + 1, but we don't have a good way to compute
+        // it here, so using 16 as a safe guess.
+        private const int SegmentCount = 16;
+
         private BarebonesMatcher _baseline;
         private DfaMatcher _dfa;
 
@@ -50,7 +54,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                 var httpContext = Requests[i];
 
                 var path = httpContext.Request.Path.Value;
-                Span<PathSegment> segments = stackalloc PathSegment[FastPathTokenizer.DefaultSegmentCount];
+                Span<PathSegment> segments = stackalloc PathSegment[SegmentCount];
                 var count = FastPathTokenizer.Tokenize(path, segments);
 
                 var candidates = _dfa.FindCandidateSet(httpContext, path, segments.Slice(0, count));

benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/MatcherFindCandidateSetSingleEntryBenchmark.cs

@@ -7,8 +7,11 @@ using Microsoft.AspNetCore.Http;
 
 namespace Microsoft.AspNetCore.Routing.Matching
 {
-    public class MatcheFindCandidateSetSingleEntryBenchmark : MatcherBenchmarkBase
+    public class MatcherFindCandidateSetSingleEntryBenchmark : MatcherBenchmarkBase
     {
+        // SegmentCount should be max-segments + 1
+        private const int SegmentCount = 2;
+
         private TrivialMatcher _baseline;
         private DfaMatcher _dfa;
 
@@ -22,7 +25,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
             Requests[0] = new DefaultHttpContext();
             Requests[0].RequestServices = CreateServices();
             Requests[0].Request.Path = "/plaintext";
-            
+
             _baseline = (TrivialMatcher)SetupMatcher(new TrivialMatcherBuilder());
             _dfa = (DfaMatcher)SetupMatcher(CreateDfaMatcherBuilder());
         }
@@ -51,7 +54,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
         {
             var httpContext = Requests[0];
             var path = httpContext.Request.Path.Value;
-            Span<PathSegment> segments = stackalloc PathSegment[FastPathTokenizer.DefaultSegmentCount];
+            Span<PathSegment> segments = stackalloc PathSegment[SegmentCount];
             var count = FastPathTokenizer.Tokenize(path, segments);
 
             var candidates = _dfa.FindCandidateSet(httpContext, path, segments.Slice(0, count));

benchmarks/Microsoft.AspNetCore.Routing.Performance/Matching/MatcherFindCandidateSetSmallEntryCountBenchmark.cs

@@ -9,6 +9,9 @@ namespace Microsoft.AspNetCore.Routing.Matching
 {
     public class MatcherFindCandidateSetSmallEntryCountBenchmark : MatcherBenchmarkBase
     {
+        // SegmentCount should be max-segments + 1
+        private const int SegmentCount = 6;
+
         private TrivialMatcher _baseline;
         private DfaMatcher _dfa;
 
@@ -87,7 +90,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
             var httpContext = Requests[0];
             var path = httpContext.Request.Path.Value;
 
-            Span<PathSegment> segments = stackalloc PathSegment[FastPathTokenizer.DefaultSegmentCount];
+            Span<PathSegment> segments = stackalloc PathSegment[SegmentCount];
             var count = FastPathTokenizer.Tokenize(path, segments);
 
             var candidates = _dfa.FindCandidateSet(httpContext, path, segments.Slice(0, count));

src/Microsoft.AspNetCore.Routing/Matching/DfaMatcher.cs

@@ -13,11 +13,13 @@ namespace Microsoft.AspNetCore.Routing.Matching
     {
         private readonly EndpointSelector _selector;
         private readonly DfaState[] _states;
-
+        private readonly int _maxSegmentCount;
-        public DfaMatcher(EndpointSelector selector, DfaState[] states)
+        
+        public DfaMatcher(EndpointSelector selector, DfaState[] states, int maxSegmentCount)
         {
             _selector = selector;
             _states = states;
+            _maxSegmentCount = maxSegmentCount;
         }
 
         public sealed override Task MatchAsync(HttpContext httpContext, IEndpointFeature feature)
@@ -38,7 +40,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
             var path = httpContext.Request.Path.Value;
 
             // First tokenize the path into series of segments.
-            Span<PathSegment> buffer = stackalloc PathSegment[FastPathTokenizer.DefaultSegmentCount];
+            Span<PathSegment> buffer = stackalloc PathSegment[_maxSegmentCount];
             var count = FastPathTokenizer.Tokenize(path, buffer);
             var segments = buffer.Slice(0, count);
 

src/Microsoft.AspNetCore.Routing/Matching/DfaMatcherBuilder.cs

@@ -50,7 +50,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
             // stage.
             var work = new List<(MatcherEndpoint endpoint, List<DfaNode> parents)>();
 
-            var root = new DfaNode() { Depth = 0, Label = "/" };
+            var root = new DfaNode() { PathDepth = 0, Label = "/" };
 
             // To prepare for this we need to compute the max depth, as well as
             // a seed list of items to process (entry, root).
@@ -101,7 +101,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                             {
                                 next = new DfaNode()
                                 {
-                                    Depth = parent.Depth + 1,
+                                    PathDepth = parent.PathDepth + 1,
                                     Label = parent.Label + literal + "/",
                                 };
                                 parent.Literals.Add(literal, next);
@@ -129,7 +129,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                             {
                                 parent.CatchAll = new DfaNode()
                                 {
-                                    Depth = parent.Depth + 1,
+                                    PathDepth = parent.PathDepth + 1,
                                     Label = parent.Label + "{*...}/",
                                 };
 
@@ -146,7 +146,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                             {
                                 parent.Parameters = new DfaNode()
                                 {
-                                    Depth = parent.Depth + 1,
+                                    PathDepth = parent.PathDepth + 1,
                                     Label = parent.Label + "{...}/",
                                 };
                             }
@@ -165,7 +165,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                             {
                                 parent.Parameters = new DfaNode()
                                 {
-                                    Depth = parent.Depth + 1,
+                                    PathDepth = parent.PathDepth + 1,
                                     Label = parent.Label + "{...}/",
                                 };
                             }
@@ -217,6 +217,13 @@ namespace Microsoft.AspNetCore.Routing.Matching
         {
             var root = BuildDfaTree();
 
+            var maxSegmentCount = 0;
+            root.Visit((node) => maxSegmentCount = Math.Max(maxSegmentCount, node.PathDepth));
+
+            // The max segment count is the maximum path-node-depth +1. We need
+            // the +1 to capture any additional content after the 'last' segment.
+            maxSegmentCount++;
+
             var states = new List<DfaState>();
             var tableBuilders = new List<(JumpTableBuilder pathBuilder, PolicyJumpTableBuilder policyBuilder)>();
             AddNode(root, states, tableBuilders);
@@ -251,7 +258,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
                     tableBuilders[i].policyBuilder?.Build());
             }
 
-            return new DfaMatcher(_selector, states.ToArray());
+            return new DfaMatcher(_selector, states.ToArray(), maxSegmentCount);
         }
 
         private int AddNode(

src/Microsoft.AspNetCore.Routing/Matching/DfaNode.cs

@@ -22,7 +22,9 @@ namespace Microsoft.AspNetCore.Routing.Matching
 
         // The depth of the node. The depth indicates the number of segments
         // that must be processed to arrive at this node.
-        public int Depth { get; set; }
+        //
+        // This value is not computed for Policy nodes and will be set to -1.
+        public int PathDepth { get; set; } = -1;
 
         // Just for diagnostics and debugging
         public string Label { get; set; }
@@ -71,7 +73,7 @@ namespace Microsoft.AspNetCore.Routing.Matching
             var builder = new StringBuilder();
             builder.Append(Label);
             builder.Append(" d:");
-            builder.Append(Depth);
+            builder.Append(PathDepth);
             builder.Append(" m:");
             builder.Append(Matches.Count);
             builder.Append(" c: ");

src/Microsoft.AspNetCore.Routing/Matching/FastPathTokenizer.cs

@@ -9,16 +9,6 @@ namespace Microsoft.AspNetCore.Routing.Matching
     // to PathTokenizer.
     internal static class FastPathTokenizer
     {
-        // The default limit for the number of segments we tokenize.
-        //
-        // Historically the limit on the number of segments routing supports is 28.
-        // RoutePrecedence computes precedence based on a decimal, which supports 28
-        // or 29 digits.
-        //
-        // So setting this limit to 32 should work pretty well. We also expect the tokenizer
-        // to be used with stackalloc, so we want a small number.
-        public const int DefaultSegmentCount = 32;
-
         // This section tokenizes the path by marking the sequence of slashes, and their
         // and the length of the text between them.
         //