Add parsing of includes and imports

Also adds the source document to the RazorSyntaxTree and does some cleanup
related to this. This lets us verify which tree goes to which document and
that seems important.

Added basic tests to verify that parsing happens, though it's not being
used for anything right now.

src/Microsoft.AspNetCore.Razor.Evolution/DefaultDirectiveSyntaxTreePass.cs

@@ -21,7 +21,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             if (errorSink.Errors.Count > 0)
             {
                 var combinedErrors = syntaxTree.Diagnostics.Concat(errorSink.Errors).ToList();
-                syntaxTree = RazorSyntaxTree.Create(syntaxTree.Root, combinedErrors, syntaxTree.Options);
+                syntaxTree = RazorSyntaxTree.Create(syntaxTree.Root, syntaxTree.Source, combinedErrors, syntaxTree.Options);
             }
 
             return syntaxTree;

src/Microsoft.AspNetCore.Razor.Evolution/DefaultRazorParsingPhase.cs

@@ -24,6 +24,20 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 
             var syntaxTree = RazorSyntaxTree.Parse(codeDocument.Source, options);
             codeDocument.SetSyntaxTree(syntaxTree);
+
+            var importSyntaxTrees = new RazorSyntaxTree[codeDocument.Imports.Count];
+            for (var i = 0; i < codeDocument.Imports.Count; i++)
+            {
+                importSyntaxTrees[i] = RazorSyntaxTree.Parse(codeDocument.Imports[i], options);
+            }
+            codeDocument.SetImportSyntaxTrees(importSyntaxTrees);
+
+            var includeSyntaxTrees = new RazorSyntaxTree[codeDocument.Includes.Count];
+            for (var i = 0; i < codeDocument.Includes.Count; i++)
+            {
+                includeSyntaxTrees[i] = RazorSyntaxTree.Parse(codeDocument.Includes[i], options);
+            }
+            codeDocument.SetIncludeSyntaxTrees(includeSyntaxTrees);
         }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/DefaultRazorSyntaxTree.cs

@@ -8,9 +8,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 {
     internal class DefaultRazorSyntaxTree : RazorSyntaxTree
     {
-        public DefaultRazorSyntaxTree(Block root, IReadOnlyList<RazorError> diagnostics, RazorParserOptions options)
+        public DefaultRazorSyntaxTree(
+            Block root,
+            RazorSourceDocument source,
+            IReadOnlyList<RazorError> diagnostics,
+            RazorParserOptions options)
         {
             Root = root;
+            Source = source;
             Diagnostics = diagnostics;
             Options = options;
         }
@@ -20,5 +25,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
         public override RazorParserOptions Options { get; }
 
         internal override Block Root { get; }
+
+        public override RazorSourceDocument Source { get; }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/HtmlNodeOptimizationPass.cs

@@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             var whitespaceRewriter = new WhiteSpaceRewriter();
             rewritten = whitespaceRewriter.Rewrite(rewritten);
 
-            var rewrittenSyntaxTree = RazorSyntaxTree.Create(rewritten, syntaxTree.Diagnostics, syntaxTree.Options);
+            var rewrittenSyntaxTree = RazorSyntaxTree.Create(rewritten, syntaxTree.Source, syntaxTree.Diagnostics, syntaxTree.Options);
             return rewrittenSyntaxTree;
         }
     }

src/Microsoft.AspNetCore.Razor.Evolution/Legacy/RazorParser.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.IO;
 
 namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 {
@@ -25,17 +24,19 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
         public RazorParserOptions Options { get; }
 
-        public virtual RazorSyntaxTree Parse(TextReader input) => Parse(input.ReadToEnd());
+        public virtual RazorSyntaxTree Parse(RazorSourceDocument source)
-
-        public virtual RazorSyntaxTree Parse(string input) => Parse(((ITextDocument)new SeekableTextReader(input)));
-
-        public virtual RazorSyntaxTree Parse(char[] input) => Parse(((ITextDocument)new SeekableTextReader(input)));
-
-        public virtual RazorSyntaxTree Parse(ITextDocument input) => ParseCore(input);
-
-        private RazorSyntaxTree ParseCore(ITextDocument input)
         {
-            var context = new ParserContext(input, Options.DesignTimeMode);
+            if (source == null)
+            {
+                throw new ArgumentNullException(nameof(source));
+            }
+
+            var chars = new char[source.Length];
+            source.CopyTo(0, chars, 0, source.Length);
+
+            var reader = new SeekableTextReader(chars);
+
+            var context = new ParserContext(reader, Options.DesignTimeMode);
 
             var codeParser = new CSharpCodeParser(Options.Directives, context);
             var markupParser = new HtmlMarkupParser(context);
@@ -47,7 +48,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
             
             var root = context.Builder.Build();
             var diagnostics = context.ErrorSink.Errors;
-            return RazorSyntaxTree.Create(root, diagnostics, Options);
+            return RazorSyntaxTree.Create(root, source, diagnostics, Options);
         }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/RazorCodeDocumentExtensions.cs

@@ -3,6 +3,7 @@
 
 using System;
 using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
+using System.Collections.Generic;
 
 namespace Microsoft.AspNetCore.Razor.Evolution
 {
@@ -28,6 +29,46 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             document.Items[typeof(RazorSyntaxTree)] = syntaxTree;
         }
 
+        public static IReadOnlyList<RazorSyntaxTree> GetImportSyntaxTrees(this RazorCodeDocument document)
+        {
+            if (document == null)
+            {
+                throw new ArgumentNullException(nameof(document));
+            }
+
+            return (document.Items[typeof(ImportSyntaxTreesHolder)] as ImportSyntaxTreesHolder)?.SyntaxTrees;
+        }
+
+        public static void SetImportSyntaxTrees(this RazorCodeDocument document, IReadOnlyList<RazorSyntaxTree> syntaxTrees)
+        {
+            if (document == null)
+            {
+                throw new ArgumentNullException(nameof(document));
+            }
+
+            document.Items[typeof(ImportSyntaxTreesHolder)] = new ImportSyntaxTreesHolder(syntaxTrees);
+        }
+
+        public static IReadOnlyList<RazorSyntaxTree> GetIncludeSyntaxTrees(this RazorCodeDocument document)
+        {
+            if (document == null)
+            {
+                throw new ArgumentNullException(nameof(document));
+            }
+
+            return (document.Items[typeof(IncludeSyntaxTreesHolder)] as IncludeSyntaxTreesHolder)?.SyntaxTrees;
+        }
+
+        public static void SetIncludeSyntaxTrees(this RazorCodeDocument document, IReadOnlyList<RazorSyntaxTree> syntaxTrees)
+        {
+            if (document == null)
+            {
+                throw new ArgumentNullException(nameof(document));
+            }
+
+            document.Items[typeof(IncludeSyntaxTreesHolder)] = new IncludeSyntaxTreesHolder(syntaxTrees);
+        }
+
         public static DocumentIRNode GetIRDocument(this RazorCodeDocument document)
         {
             if (document == null)
@@ -67,5 +108,25 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 
             document.Items[typeof(RazorCSharpDocument)] = csharp;
         }
+
+        private class ImportSyntaxTreesHolder
+        {
+            public ImportSyntaxTreesHolder(IReadOnlyList<RazorSyntaxTree> syntaxTrees)
+            {
+                SyntaxTrees = syntaxTrees;
+            }
+
+            public IReadOnlyList<RazorSyntaxTree> SyntaxTrees { get; }
+        }
+
+        private class IncludeSyntaxTreesHolder
+        {
+            public IncludeSyntaxTreesHolder(IReadOnlyList<RazorSyntaxTree> syntaxTrees)
+            {
+                SyntaxTrees = syntaxTrees;
+            }
+
+            public IReadOnlyList<RazorSyntaxTree> SyntaxTrees { get; }
+        }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/RazorSyntaxTree.cs

@@ -11,6 +11,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
     {
         internal static RazorSyntaxTree Create(
             Block root,
+            RazorSourceDocument source,
             IEnumerable<RazorError> diagnostics,
             RazorParserOptions options)
         {
@@ -19,6 +20,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 throw new ArgumentNullException(nameof(root));
             }
 
+            if (source == null)
+            {
+                throw new ArgumentNullException(nameof(source));
+            }
+
             if (diagnostics == null)
             {
                 throw new ArgumentNullException(nameof(diagnostics));
@@ -29,7 +35,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 throw new ArgumentNullException(nameof(options));
             }
 
-            return new DefaultRazorSyntaxTree(root, new List<RazorError>(diagnostics), options);
+            return new DefaultRazorSyntaxTree(root, source, new List<RazorError>(diagnostics), options);
         }
 
         public static RazorSyntaxTree Parse(RazorSourceDocument source)
@@ -50,10 +56,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             }
             
             var parser = new RazorParser(options ?? RazorParserOptions.CreateDefaultOptions());
-            var sourceContent = new char[source.Length];
+            return parser.Parse(source);
-            source.CopyTo(0, sourceContent, 0, source.Length);
-
-            return parser.Parse(sourceContent);
         }
 
         internal abstract IReadOnlyList<RazorError> Diagnostics { get; }
@@ -61,5 +64,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
         public abstract RazorParserOptions Options { get; }
 
         internal abstract Block Root { get; }
+
+        public abstract RazorSourceDocument Source { get; }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/TagHelperBinderSyntaxTreePass.cs

@@ -31,7 +31,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 if (errorSink.Errors.Count > 0)
                 {
                     var combinedErrors = CombineErrors(syntaxTree.Diagnostics, errorSink.Errors);
-                    var erroredTree = RazorSyntaxTree.Create(syntaxTree.Root, combinedErrors, syntaxTree.Options);
+                    var erroredTree = RazorSyntaxTree.Create(syntaxTree.Root, syntaxTree.Source, combinedErrors, syntaxTree.Options);
 
                     return erroredTree;
                 }
@@ -49,7 +49,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 diagnostics = CombineErrors(diagnostics, errorSink.Errors);
             }
 
-            var newSyntaxTree = RazorSyntaxTree.Create(rewrittenRoot, diagnostics, syntaxTree.Options);
+            var newSyntaxTree = RazorSyntaxTree.Create(rewrittenRoot, syntaxTree.Source, diagnostics, syntaxTree.Options);
             return newSyntaxTree;
         }
 

test/Microsoft.AspNetCore.Razor.Evolution.Test/DefaultDirectiveSyntaxTreePassTest.cs

@@ -106,7 +106,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             var sourceDocument = TestRazorSourceDocument.Create(content);
             var codeDocument = RazorCodeDocument.Create(sourceDocument);
             var originalTree = RazorSyntaxTree.Parse(sourceDocument);
-            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { expectedErrors[0] }, originalTree.Options);
+            var erroredOriginalTree = RazorSyntaxTree.Create(
+                originalTree.Root, 
+                originalTree.Source, 
+                new[] { expectedErrors[0] }, 
+                originalTree.Options);
 
             // Act
             var outputTree = pass.Execute(codeDocument, erroredOriginalTree);

test/Microsoft.AspNetCore.Razor.Evolution.Test/DefaultRazorParsingPhaseTest.cs

@@ -46,6 +46,46 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             Assert.Equal("test_directive", directive.Name);
         }
 
+        [Fact]
+        public void Execute_ParsesIncludesAndImports()
+        {
+            // Arrange
+            var phase = new DefaultRazorParsingPhase();
+            var engine = RazorEngine.CreateEmpty((b) =>
+            {
+                b.Phases.Add(phase);
+                b.Features.Add(new MyConfigureParserOptions());
+            });
+
+            var imports = new[]
+            {
+                TestRazorSourceDocument.Create(),
+                TestRazorSourceDocument.Create(),
+            };
+
+            var includes = new[]
+            {
+                TestRazorSourceDocument.Create(),
+                TestRazorSourceDocument.Create(),
+            };
+
+            var codeDocument = TestRazorCodeDocument.Create(TestRazorSourceDocument.Create(), imports, includes);
+
+            // Act
+            phase.Execute(codeDocument);
+
+            // Assert
+            Assert.Collection(
+                codeDocument.GetImportSyntaxTrees(),
+                t => { Assert.Same(t.Source, imports[0]); Assert.Equal("test_directive", Assert.Single(t.Options.Directives).Name); },
+                t => { Assert.Same(t.Source, imports[1]); Assert.Equal("test_directive", Assert.Single(t.Options.Directives).Name); });
+
+            Assert.Collection(
+                codeDocument.GetIncludeSyntaxTrees(),
+                t => { Assert.Same(t.Source, includes[0]); Assert.Equal("test_directive", Assert.Single(t.Options.Directives).Name); },
+                t => { Assert.Same(t.Source, includes[1]); Assert.Equal("test_directive", Assert.Single(t.Options.Directives).Name); });
+        }
+
         private class MyConfigureParserOptions : IRazorConfigureParserFeature
         {
             public RazorEngine Engine { get; set; }

test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/ParserTestBase.cs

@@ -41,7 +41,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
                 var parser = new RazorParser(options);
 
-                var tree = parser.Parse((ITextDocument)reader);
+                var tree = parser.Parse(TestRazorSourceDocument.Create(document));
                 var defaultDirectivePass = new DefaultDirectiveSyntaxTreePass();
                 tree = defaultDirectivePass.Execute(codeDocument: null, syntaxTree: tree);
 
@@ -51,6 +51,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
         internal virtual RazorSyntaxTree ParseHtmlBlock(string document, bool designTime = false)
         {
+            var source = TestRazorSourceDocument.Create(document);
+
             using (var reader = new SeekableTextReader(document))
             {
                 var context = new ParserContext(reader, designTime);
@@ -68,7 +70,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
                 var options = RazorParserOptions.CreateDefaultOptions();
                 options.DesignTimeMode = designTime;
 
-                return RazorSyntaxTree.Create(root, diagnostics, options);
+                return RazorSyntaxTree.Create(root, source, diagnostics, options);
             }
         }
 
@@ -82,6 +84,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
             IEnumerable<DirectiveDescriptor> descriptors,
             bool designTime)
         {
+            var source = TestRazorSourceDocument.Create(document);
+
             using (var reader = new SeekableTextReader(document))
             {
                 var context = new ParserContext(reader, designTime);
@@ -106,7 +110,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
                     options.Directives.Add(directive);
                 }
 
-                return RazorSyntaxTree.Create(root, diagnostics, options);
+                return RazorSyntaxTree.Create(root, source, diagnostics, options);
             }
         }
 

test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RazorParserTest.cs

@@ -13,9 +13,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         {
             var parser = new RazorParser();
             var sourceDocument = TestRazorSourceDocument.CreateResource("TestFiles/Source/BasicMarkup.cshtml");
-            var sourceContent = new char[sourceDocument.Length];
+            var output = parser.Parse(sourceDocument);
-            sourceDocument.CopyTo(0, sourceContent, 0, sourceDocument.Length);
-            var output = parser.Parse(sourceContent);
 
             Assert.NotNull(output);
         }
@@ -29,7 +27,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
             var parser = new RazorParser();
 
             // Act/Assert
-            ParserTestBase.EvaluateResults(parser.Parse(new StringReader("foo @bar baz")),
+            ParserTestBase.EvaluateResults(parser.Parse(TestRazorSourceDocument.Create("foo @bar baz")),
                 new MarkupBlock(
                     factory.Markup("foo "),
                     new ExpressionBlock(
@@ -49,7 +47,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
             var parser = new RazorParser();
 
             // Act
-            var results = parser.Parse(new StringReader("foo @bar baz"));
+            var results = parser.Parse(TestRazorSourceDocument.Create("foo @bar baz"));
 
             // Assert
             ParserTestBase.EvaluateResults(results,

test/Microsoft.AspNetCore.Razor.Evolution.Test/RazorCodeDocumentExtensionsTest.cs

@@ -39,6 +39,38 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             Assert.Same(expected, codeDocument.Items[typeof(RazorSyntaxTree)]);
         }
 
+        [Fact]
+        public void GetAndSetImportSyntaxTrees_ReturnsSyntaxTrees()
+        {
+            // Arrange
+            var codeDocument = TestRazorCodeDocument.CreateEmpty();
+
+            var expected = new[] { RazorSyntaxTree.Parse(codeDocument.Source), };
+            codeDocument.SetImportSyntaxTrees(expected);
+
+            // Act
+            var actual = codeDocument.GetImportSyntaxTrees();
+
+            // Assert
+            Assert.Same(expected, actual);
+        }
+
+        [Fact]
+        public void GetAndSetIncludeSyntaxTrees_ReturnsSyntaxTrees()
+        {
+            // Arrange
+            var codeDocument = TestRazorCodeDocument.CreateEmpty();
+
+            var expected = new[] { RazorSyntaxTree.Parse(codeDocument.Source), };
+            codeDocument.SetIncludeSyntaxTrees(expected);
+
+            // Act
+            var actual = codeDocument.GetIncludeSyntaxTrees();
+
+            // Assert
+            Assert.Same(expected, actual);
+        }
+
         [Fact]
         public void GetIRDocument_ReturnsIRDocument()
         {

test/Microsoft.AspNetCore.Razor.Evolution.Test/TagHelperBinderSyntaxTreePassTest.cs

@@ -145,7 +145,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             var codeDocument = RazorCodeDocument.Create(sourceDocument);
             var originalTree = RazorSyntaxTree.Parse(sourceDocument);
             var initialError = new RazorError("Initial test error", SourceLocation.Zero, length: 1);
-            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { initialError }, originalTree.Options);
+            var erroredOriginalTree = RazorSyntaxTree.Create(
+                originalTree.Root,
+                originalTree.Source,
+                new[] { initialError },
+                originalTree.Options);
 
             // Act
             var outputTree = pass.Execute(codeDocument, erroredOriginalTree);
@@ -194,7 +198,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
                 LegacyResources.FormatTagHelpersParseTreeRewriter_FoundMalformedTagHelper("form"),
                 new SourceLocation(Environment.NewLine.Length * 2 + 30, 2, 1),
                 length: 4);
-            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, new[] { initialError }, originalTree.Options);
+            var erroredOriginalTree = RazorSyntaxTree.Create(originalTree.Root, originalTree.Source, new[] { initialError }, originalTree.Options);
 
             // Act
             var outputTree = pass.Execute(codeDocument, erroredOriginalTree);