Add end-to-end support for extensible directives

This change adds a way to actually configure the RazorEngine to use
extensible directives (previously buried behind legacy API). As part of
this feature adds the RazorParserOptions class to encapsulate anything
else that becomes a parser options (ahem taghelpers).

Now we have a pattern for this when we get there.

Options are propagated as part of the RazorSyntaxTree for
testability/sanity and this was actually responsible for the bulk of the
changes.

Also added some extension methods for adding directives to the
IRazorEngineBuilder and an end to end integration test.

src/Microsoft.AspNetCore.Razor.Evolution/DefaultRazorDirectiveFeature.cs

@@ -0,0 +1,26 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.AspNetCore.Razor.Evolution
+{
+    internal class DefaultRazorDirectiveFeature : IRazorDirectiveFeature, IRazorConfigureParserFeature
+    {
+        public ICollection<DirectiveDescriptor> Directives { get; } = new List<DirectiveDescriptor>();
+
+        public RazorEngine Engine { get; set; }
+
+        public int Order => 100;
+
+        void IRazorConfigureParserFeature.Configure(RazorParserOptions options)
+        {
+            options.Directives.Clear();
+
+            foreach (var directive in Directives)
+            {
+                options.Directives.Add(directive);
+            }
+        }
+    }
+}

src/Microsoft.AspNetCore.Razor.Evolution/DefaultRazorParsingPhase.cs

@@ -1,15 +1,28 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
+using System.Linq;
 
 namespace Microsoft.AspNetCore.Razor.Evolution
 {
     internal class DefaultRazorParsingPhase : RazorEnginePhaseBase, IRazorParsingPhase
     {
+        private IRazorConfigureParserFeature[] _parserOptionsCallbacks;
+
+        protected override void OnIntialized()
+        {
+            _parserOptionsCallbacks = Engine.Features.OfType<IRazorConfigureParserFeature>().ToArray();
+        }
+
         protected override void ExecuteCore(RazorCodeDocument codeDocument)
         {
-            var syntaxTree = RazorSyntaxTree.Parse(codeDocument.Source);
+            var options = RazorParserOptions.CreateDefaultOptions();
+            for (var i = 0; i < _parserOptionsCallbacks.Length; i++)
+            {
+                _parserOptionsCallbacks[i].Configure(options);
+            }
+
+            var syntaxTree = RazorSyntaxTree.Parse(codeDocument.Source, options);
             codeDocument.SetSyntaxTree(syntaxTree);
         }
     }

src/Microsoft.AspNetCore.Razor.Evolution/DefaultRazorSyntaxTree.cs

@@ -8,14 +8,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 {
     internal class DefaultRazorSyntaxTree : RazorSyntaxTree
     {
-        public DefaultRazorSyntaxTree(Block root, IReadOnlyList<RazorError> diagnostics)
+        public DefaultRazorSyntaxTree(Block root, IReadOnlyList<RazorError> diagnostics, RazorParserOptions options)
         {
             Root = root;
             Diagnostics = diagnostics;
+            Options = options;
         }
 
         internal override IReadOnlyList<RazorError> Diagnostics { get; }
 
+        public override RazorParserOptions Options { get; }
+
         internal override Block Root { get; }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/HtmlNodeOptimizationPass.cs

@@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             var whitespaceRewriter = new WhiteSpaceRewriter();
             rewritten = whitespaceRewriter.Rewrite(rewritten);
 
-            var rewrittenSyntaxTree = RazorSyntaxTree.Create(rewritten, syntaxTree.Diagnostics);
+            var rewrittenSyntaxTree = RazorSyntaxTree.Create(rewritten, syntaxTree.Diagnostics, syntaxTree.Options);
             return rewrittenSyntaxTree;
         }
     }

src/Microsoft.AspNetCore.Razor.Evolution/IRazorConfigureParserFeature.cs

@@ -0,0 +1,12 @@
+// Copyright(c) .NET Foundation.All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.AspNetCore.Razor.Evolution
+{
+    internal interface IRazorConfigureParserFeature : IRazorEngineFeature
+    {
+        int Order { get; }
+
+        void Configure(RazorParserOptions options);
+    }
+}

src/Microsoft.AspNetCore.Razor.Evolution/IRazorDirectiveFeature.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.AspNetCore.Razor.Evolution
+{
+    public interface IRazorDirectiveFeature : IRazorEngineFeature
+    {
+        ICollection<DirectiveDescriptor> Directives { get; }
+    }
+}

src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ParserContext.cs

@@ -38,14 +38,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         {
             get { return Source.Peek() == -1; }
         }
-
-        public RazorSyntaxTree BuildRazorSyntaxTree()
-        {
-            var syntaxTree = Builder.Build();
-            var razorSyntaxTree = RazorSyntaxTree.Create(syntaxTree, ErrorSink.Errors);
-
-            return razorSyntaxTree;
-        }
     }
 
     // Debug Helpers

src/Microsoft.AspNetCore.Razor.Evolution/Legacy/RazorParser.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
 
 namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
@@ -8,10 +9,21 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
     internal class RazorParser
     {
         public RazorParser()
+            : this(RazorParserOptions.CreateDefaultOptions())
         {
         }
 
-        public bool DesignTimeMode { get; set; }
+        public RazorParser(RazorParserOptions options)
+        {
+            if (options == null)
+            {
+                throw new ArgumentNullException(nameof(options));
+            }
+
+            Options = options;
+        }
+
+        public RazorParserOptions Options { get; }
 
         public virtual RazorSyntaxTree Parse(TextReader input) => Parse(input.ReadToEnd());
 
@@ -23,22 +35,19 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
         private RazorSyntaxTree ParseCore(ITextDocument input)
         {
-            var context = new ParserContext(input, DesignTimeMode);
+            var context = new ParserContext(input, Options.DesignTimeMode);
 
-            var codeParser = new CSharpCodeParser(context);
+            var codeParser = new CSharpCodeParser(Options.Directives, context);
             var markupParser = new HtmlMarkupParser(context);
 
             codeParser.HtmlParser = markupParser;
             markupParser.CodeParser = codeParser;
 
-            // Execute the parse
             markupParser.ParseDocument();
-
-            // Get the result
-            var razorSyntaxTree = context.BuildRazorSyntaxTree();
-
-            // Return the new result
-            return razorSyntaxTree;
+            
+            var root = context.Builder.Build();
+            var diagnostics = context.ErrorSink.Errors;
+            return RazorSyntaxTree.Create(root, diagnostics, Options);
         }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/RazorEngineBuilderExtensions.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+
+namespace Microsoft.AspNetCore.Razor.Evolution
+{
+    public static class RazorEngineBuilderExtensions
+    {
+        public static IRazorEngineBuilder AddDirective(this IRazorEngineBuilder builder, DirectiveDescriptor directive)
+        {
+            if (builder == null)
+            {
+                throw new ArgumentNullException(nameof(directive));
+            }
+
+            if (directive == null)
+            {
+                throw new ArgumentNullException(nameof(directive));
+            }
+
+            var directiveFeature = GetDirectiveFeature(builder);
+            directiveFeature.Directives.Add(directive);
+
+            return builder;
+        }
+
+        private static IRazorDirectiveFeature GetDirectiveFeature(IRazorEngineBuilder builder)
+        {
+            var directiveFeature = builder.Features.OfType<IRazorDirectiveFeature>().FirstOrDefault();
+            if (directiveFeature == null)
+            {
+                directiveFeature = new DefaultRazorDirectiveFeature();
+                builder.Features.Add(directiveFeature);
+            }
+
+            return directiveFeature;
+        }
+    }
+}

src/Microsoft.AspNetCore.Razor.Evolution/RazorParserOptions.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.AspNetCore.Razor.Evolution
+{
+    public sealed class RazorParserOptions
+    {
+        public static RazorParserOptions CreateDefaultOptions()
+        {
+            return new RazorParserOptions();
+        }
+
+        private RazorParserOptions()
+        {
+            Directives = new List<DirectiveDescriptor>();
+        }
+
+        public bool DesignTimeMode { get; set; }
+
+        public ICollection<DirectiveDescriptor> Directives { get; }
+    }
+}

src/Microsoft.AspNetCore.Razor.Evolution/RazorSyntaxTree.cs

@@ -9,7 +9,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 {
     public abstract class RazorSyntaxTree
     {
-        internal static RazorSyntaxTree Create(Block root, IEnumerable<RazorError> diagnostics)
+        internal static RazorSyntaxTree Create(
+            Block root,
+            IEnumerable<RazorError> diagnostics,
+            RazorParserOptions options)
         {
             if (root == null)
             {
@@ -21,7 +24,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 throw new ArgumentNullException(nameof(diagnostics));
             }
 
-            return new DefaultRazorSyntaxTree(root, new List<RazorError>(diagnostics));
+            if (options == null)
+            {
+                throw new ArgumentNullException(nameof(options));
+            }
+
+            return new DefaultRazorSyntaxTree(root, new List<RazorError>(diagnostics), options);
         }
 
         public static RazorSyntaxTree Parse(RazorSourceDocument source)
@@ -31,7 +39,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 throw new ArgumentNullException(nameof(source));
             }
 
-            var parser = new RazorParser();
+            return Parse(source, options: null);
+        }
+
+        public static RazorSyntaxTree Parse(RazorSourceDocument source, RazorParserOptions options)
+        {
+            if (source == null)
+            {
+                throw new ArgumentNullException(nameof(source));
+            }
+            
+            var parser = new RazorParser(options ?? RazorParserOptions.CreateDefaultOptions());
             var sourceContent = new char[source.Length];
             source.CopyTo(0, sourceContent, 0, source.Length);
 
@@ -40,6 +58,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 
         internal abstract IReadOnlyList<RazorError> Diagnostics { get; }
 
+        public abstract RazorParserOptions Options { get; }
+
         internal abstract Block Root { get; }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/TagHelperBinderSyntaxTreePass.cs

@@ -31,7 +31,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 if (errorSink.Errors.Count > 0)
                 {
                     var combinedErrors = CombineErrors(syntaxTree.Diagnostics, errorSink.Errors);
-                    var erroredTree = RazorSyntaxTree.Create(syntaxTree.Root, combinedErrors);
+                    var erroredTree = RazorSyntaxTree.Create(syntaxTree.Root, combinedErrors, syntaxTree.Options);
 
                     return erroredTree;
                 }
@@ -49,7 +49,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 diagnostics = CombineErrors(diagnostics, errorSink.Errors);
             }
 
-            var newSyntaxTree = RazorSyntaxTree.Create(rewrittenRoot, diagnostics);
+            var newSyntaxTree = RazorSyntaxTree.Create(rewrittenRoot, diagnostics, syntaxTree.Options);
             return newSyntaxTree;
         }
 

test/Microsoft.AspNetCore.Razor.Evolution.Test/DefaultRazorParsingPhaseTest.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Razor.Evolution
@@ -22,5 +23,39 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             // Assert
             Assert.NotNull(codeDocument.GetSyntaxTree());
         }
+
+        [Fact]
+        public void Execute_UsesConfigureParserFeatures()
+        {
+            // Arrange
+            var phase = new DefaultRazorParsingPhase();
+            var engine = RazorEngine.CreateEmpty((b) =>
+            {
+                b.Phases.Add(phase);
+                b.Features.Add(new MyConfigureParserOptions());
+            });
+
+            var codeDocument = TestRazorCodeDocument.CreateEmpty();
+
+            // Act
+            phase.Execute(codeDocument);
+
+            // Assert
+            var syntaxTree = codeDocument.GetSyntaxTree();
+            var directive = Assert.Single(syntaxTree.Options.Directives);
+            Assert.Equal("test_directive", directive.Name);
+        }
+
+        private class MyConfigureParserOptions : IRazorConfigureParserFeature
+        {
+            public RazorEngine Engine { get; set; }
+
+            public int Order { get; }
+
+            public void Configure(RazorParserOptions options)
+            {
+                options.Directives.Add(DirectiveDescriptorBuilder.Create("test_directive").Build());
+            }
+        }
     }
 }

test/Microsoft.AspNetCore.Razor.Evolution.Test/IntegrationTest.cs

@@ -1,6 +1,8 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
+using Microsoft.AspNetCore.Razor.Evolution.Legacy;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Razor.Evolution
@@ -20,6 +22,38 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 
             // Assert
             Assert.NotNull(document.GetSyntaxTree());
+            Assert.NotNull(document.GetIRDocument());
+        }
+
+        [Fact]
+        public void Process_CustomDirective()
+        {
+            // Arrange
+            var engine = RazorEngine.Create(b =>
+            {
+                b.AddDirective(DirectiveDescriptorBuilder.Create("test_directive").Build());
+            });
+
+            var document = RazorCodeDocument.Create(TestRazorSourceDocument.Create("@test_directive"));
+
+            // Act
+            engine.Process(document);
+
+            // Assert
+            var syntaxTree = document.GetSyntaxTree();
+
+            // This is fragile for now, but we don't want to invest in the legacy API until we're ready
+            // to replace it properly.
+            var directiveBlock = (Block)syntaxTree.Root.Children[1];
+            var directiveSpan = (Span)directiveBlock.Children[1];
+            Assert.Equal("test_directive", directiveSpan.Content);
+
+            var irDocument = document.GetIRDocument();
+            var irNamespace = irDocument.Children[0];
+            var irClass = irNamespace.Children[0];
+            var irMethod = irClass.Children[0];
+            var irDirective = (DirectiveIRNode)irMethod.Children[1];
+            Assert.Equal("test_directive", irDirective.Name);
         }
     }
 }

test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/ParserTestBase.cs

@@ -29,10 +29,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         {
             using (var reader = new SeekableTextReader(document))
             {
-                var parser = new RazorParser()
-                {
-                    DesignTimeMode = designTime,
-                };
+                var options = RazorParserOptions.CreateDefaultOptions();
+                options.DesignTimeMode = designTime;
+
+                var parser = new RazorParser(options);
 
                 return parser.Parse((ITextDocument)reader);
             }
@@ -52,9 +52,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
                 parser.ParseBlock();
 
-                var razorSyntaxTree = context.BuildRazorSyntaxTree();
+                var root = context.Builder.Build();
+                var diagnostics = context.ErrorSink.Errors;
+                var options = RazorParserOptions.CreateDefaultOptions();
+                options.DesignTimeMode = designTime;
 
-                return razorSyntaxTree;
+                return RazorSyntaxTree.Create(root, diagnostics, options);
             }
         }
 
@@ -80,9 +83,19 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
                 parser.ParseBlock();
 
-                var razorSyntaxTree = context.BuildRazorSyntaxTree();
+                var root = context.Builder.Build();
+                var diagnostics = context.ErrorSink.Errors;
 
-                return razorSyntaxTree;
+                var options = RazorParserOptions.CreateDefaultOptions();
+                options.DesignTimeMode = designTime;
+
+                options.Directives.Clear();
+                foreach (var directive in descriptors)
+                {
+                    options.Directives.Add(directive);
+                }
+
+                return RazorSyntaxTree.Create(root, diagnostics, options);
             }
         }
 

test/Microsoft.AspNetCore.Razor.Evolution.Test/RazorEngineBuilderExtensionsTest.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Linq;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Razor.Evolution
+{
+    public class RazorEngineBuilderExtensionsTest
+    {
+        [Fact]
+        public void AddDirective_ExistingFeature_UsesFeature()
+        {
+            // Arrange
+            var expected = new DefaultRazorDirectiveFeature();
+            var engine = RazorEngine.CreateEmpty(b =>
+            {
+                b.Features.Add(expected);
+
+                // Act
+                b.AddDirective(DirectiveDescriptorBuilder.Create("test_directive").Build());
+            });
+
+            // Assert
+            var actual = Assert.Single(engine.Features.OfType<IRazorDirectiveFeature>());
+            Assert.Same(expected, actual);
+
+            var directive = Assert.Single(actual.Directives);
+            Assert.Equal("test_directive", directive.Name);
+        }
+
+        public void AddDirective_NoFeature_CreatesFeature()
+        {
+            // Arrange
+            var engine = RazorEngine.CreateEmpty(b =>
+            {
+                // Act
+                b.AddDirective(DirectiveDescriptorBuilder.Create("test_directive").Build());
+            });
+
+            // Assert
+            var actual = Assert.Single(engine.Features.OfType<IRazorDirectiveFeature>());
+            Assert.IsType<DefaultRazorDirectiveFeature>(actual);
+
+            var directive = Assert.Single(actual.Directives);
+            Assert.Equal("test_directive", directive.Name);
+        }
+    }
+}

test/Microsoft.AspNetCore.Razor.Evolution.Test/TagHelperBinderSyntaxTreePassTest.cs

@@ -145,7 +145,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             var codeDocument = RazorCodeDocument.Create(sourceDocument);
             var originalTree = RazorSyntaxTree.Parse(sourceDocument);
             var initialError = new RazorError("Initial test error", SourceLocation.Zero, length: 1);
-            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { initialError });
+            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { initialError }, originalTree.Options);
 
             // Act
             var outputTree = pass.Execute(codeDocument, erroredOriginalTree);
@@ -194,7 +194,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 LegacyResources.FormatTagHelpersParseTreeRewriter_FoundMalformedTagHelper("form"),
                 new SourceLocation(Environment.NewLine.Length * 2 + 30, 2, 1),
                 length: 4);
-            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { initialError });
+            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { initialError }, originalTree.Options);
 
             // Act
             var outputTree = pass.Execute(codeDocument, erroredOriginalTree);