diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs index d00d3570f1..ca206343a5 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs @@ -10,7 +10,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy { internal class CSharpCodeParser : TokenizerBackedParser { - internal static readonly int UsingKeywordLength = 5; // using private static readonly Func IsValidStatementSpacingSymbol = IsSpacingToken(includeNewLines: true, includeComments: true); @@ -88,7 +87,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy } public CSharpCodeParser(IEnumerable directiveDescriptors, ParserContext context) - : base(CSharpLanguageCharacteristics.Instance, context) + : base(context.StopParsingAfterFirstDirective ? FirstDirectiveCSharpLanguageCharacteristics.Instance : CSharpLanguageCharacteristics.Instance, context) { Keywords = new HashSet(); SetUpKeywords(); @@ -106,7 +105,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy protected void MapDirectives(Action handler, params string[] directives) { - foreach (string directive in directives) + foreach (var directive in directives) { _directiveParsers.Add(directive, handler); Keywords.Add(directive); @@ -143,7 +142,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy private void MapKeywords(Action handler, bool topLevel, params CSharpKeyword[] keywords) { - foreach (CSharpKeyword keyword in keywords) + foreach (var keyword in keywords) { _keywordParsers.Add(keyword, handler); if (topLevel) @@ -587,17 +586,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy private void CaptureWhitespaceAtEndOfCodeOnlyLine() { - IEnumerable ws = ReadWhile(sym => sym.Type == CSharpSymbolType.WhiteSpace); + var whitespace = ReadWhile(sym => sym.Type == CSharpSymbolType.WhiteSpace); if (At(CSharpSymbolType.NewLine)) { - Accept(ws); + Accept(whitespace); AcceptAndMoveNext(); PutCurrentBack(); } else { PutCurrentBack(); - PutBack(ws); + PutBack(whitespace); } } @@ -807,11 +806,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy private void WhileClause() { Span.EditHandler.AcceptedCharacters = AcceptedCharacters.Any; - IEnumerable ws = SkipToNextImportantToken(); + var whitespace = SkipToNextImportantToken(); if (At(CSharpKeyword.While)) { - Accept(ws); + Accept(whitespace); Assert(CSharpKeyword.While); AcceptAndMoveNext(); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); @@ -823,7 +822,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy else { PutCurrentBack(); - PutBack(ws); + PutBack(whitespace); } } @@ -1042,12 +1041,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy private void AfterIfClause() { // Grab whitespace and razor comments - IEnumerable ws = SkipToNextImportantToken(); + var whitespace = SkipToNextImportantToken(); // Check for an else part if (At(CSharpKeyword.Else)) { - Accept(ws); + Accept(whitespace); Assert(CSharpKeyword.Else); ElseClause(); } @@ -1055,7 +1054,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy { // No else, return whitespace PutCurrentBack(); - PutBack(ws); + PutBack(whitespace); Span.EditHandler.AcceptedCharacters = AcceptedCharacters.Any; } } @@ -1355,13 +1354,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy while (!EndOfFile) { var bookmark = CurrentStart.AbsoluteIndex; - IEnumerable read = ReadWhile(sym => sym.Type != CSharpSymbolType.Semicolon && - sym.Type != CSharpSymbolType.RazorCommentTransition && - sym.Type != CSharpSymbolType.Transition && - sym.Type != CSharpSymbolType.LeftBrace && - sym.Type != CSharpSymbolType.LeftParenthesis && - sym.Type != CSharpSymbolType.LeftBracket && - sym.Type != CSharpSymbolType.RightBrace); + var read = ReadWhile(sym => + sym.Type != CSharpSymbolType.Semicolon && + sym.Type != CSharpSymbolType.RazorCommentTransition && + sym.Type != CSharpSymbolType.Transition && + sym.Type != CSharpSymbolType.LeftBrace && + sym.Type != CSharpSymbolType.LeftParenthesis && + sym.Type != CSharpSymbolType.LeftBracket && + sym.Type != CSharpSymbolType.RightBrace); + if (At(CSharpSymbolType.LeftBrace) || At(CSharpSymbolType.LeftParenthesis) || At(CSharpSymbolType.LeftBracket)) @@ -1443,8 +1444,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy private void HandleKeyword(bool topLevel, Action fallback) { Debug.Assert(CurrentSymbol.Type == CSharpSymbolType.Keyword && CurrentSymbol.Keyword != null); - Action handler; - if (_keywordParsers.TryGetValue(CurrentSymbol.Keyword.Value, out handler)) + if (_keywordParsers.TryGetValue(CurrentSymbol.Keyword.Value, out var handler)) { handler(topLevel); } @@ -1458,16 +1458,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy { while (!EndOfFile) { - IEnumerable ws = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); + var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); if (At(CSharpSymbolType.RazorCommentTransition)) { - Accept(ws); + Accept(whitespace); Span.EditHandler.AcceptedCharacters = AcceptedCharacters.Any; RazorComment(); } else { - return ws; + return whitespace; } } return Enumerable.Empty(); @@ -1814,7 +1814,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy } // Pull out the type name - string baseType = string.Concat(Span.Symbols.Select(s => s.Content)); + var baseType = string.Concat(Span.Symbols.Select(s => s.Content)); // Set up chunk generation Span.ChunkGenerator = createChunkGenerator(baseType.Trim()); diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs index 897cceb2d9..7a251cc7e9 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs @@ -63,7 +63,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy { CSharpSymbolType.Transition, "@" }, }; - private CSharpLanguageCharacteristics() + protected CSharpLanguageCharacteristics() { } @@ -172,4 +172,4 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy return keyword.ToString().ToLowerInvariant(); } } -} +} \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs new file mode 100644 index 0000000000..d9507e8522 --- /dev/null +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics; + +namespace Microsoft.AspNetCore.Razor.Language.Legacy +{ + internal class DirectiveCSharpTokenizer : CSharpTokenizer + { + private bool _visitedFirstTokenStart = false; + private bool _visitedFirstTokenLineEnd = false; + + public DirectiveCSharpTokenizer(ITextDocument source) : base(source) + { + } + + protected override StateResult Dispatch() + { + var result = base.Dispatch(); + if (result.Result != null && !_visitedFirstTokenStart && IsValidTokenType(result.Result.Type)) + { + _visitedFirstTokenStart = true; + } + else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpSymbolType.NewLine) + { + _visitedFirstTokenLineEnd = true; + } + + return result; + } + + public override CSharpSymbol NextSymbol() + { + // Post-Condition: Buffer should be empty at the start of Next() + Debug.Assert(Buffer.Length == 0); + StartSymbol(); + + if (EndOfFile || (_visitedFirstTokenStart && _visitedFirstTokenLineEnd)) + { + return null; + } + + var symbol = Turn(); + + // Post-Condition: Buffer should be empty at the end of Next() + Debug.Assert(Buffer.Length == 0); + + return symbol; + } + + private bool IsValidTokenType(CSharpSymbolType type) + { + return type != CSharpSymbolType.WhiteSpace && + type != CSharpSymbolType.NewLine && + type != CSharpSymbolType.Comment && + type != CSharpSymbolType.RazorComment && + type != CSharpSymbolType.RazorCommentStar && + type != CSharpSymbolType.RazorCommentTransition && + type != CSharpSymbolType.Transition; + } + } +} diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs new file mode 100644 index 0000000000..5e52dd6ade --- /dev/null +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs @@ -0,0 +1,56 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics; + +namespace Microsoft.AspNetCore.Razor.Language.Legacy +{ + internal class DirectiveHtmlTokenizer : HtmlTokenizer + { + private bool _visitedFirstTokenStart = false; + + public DirectiveHtmlTokenizer(ITextDocument source) : base(source) + { + } + + protected override StateResult Dispatch() + { + var result = base.Dispatch(); + if (result.Result != null && IsValidTokenType(result.Result.Type)) + { + _visitedFirstTokenStart = true; + } + + return result; + } + + public override HtmlSymbol NextSymbol() + { + // Post-Condition: Buffer should be empty at the start of Next() + Debug.Assert(Buffer.Length == 0); + StartSymbol(); + + if (EndOfFile || _visitedFirstTokenStart) + { + return null; + } + + var symbol = Turn(); + + // Post-Condition: Buffer should be empty at the end of Next() + Debug.Assert(Buffer.Length == 0); + + return symbol; + } + + private bool IsValidTokenType(HtmlSymbolType type) + { + return type != HtmlSymbolType.WhiteSpace && + type != HtmlSymbolType.NewLine && + type != HtmlSymbolType.RazorComment && + type != HtmlSymbolType.RazorCommentStar && + type != HtmlSymbolType.RazorCommentTransition && + type != HtmlSymbolType.Transition; + } + } +} diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/FirstDirectiveCSharpLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/FirstDirectiveCSharpLanguageCharacteristics.cs new file mode 100644 index 0000000000..c3761ce5c5 --- /dev/null +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/FirstDirectiveCSharpLanguageCharacteristics.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.AspNetCore.Razor.Language.Legacy +{ + internal class FirstDirectiveCSharpLanguageCharacteristics : CSharpLanguageCharacteristics + { + private static readonly FirstDirectiveCSharpLanguageCharacteristics _instance = new FirstDirectiveCSharpLanguageCharacteristics(); + + private FirstDirectiveCSharpLanguageCharacteristics() + { + } + + public new static FirstDirectiveCSharpLanguageCharacteristics Instance => _instance; + + public override CSharpTokenizer CreateTokenizer(ITextDocument source) => new DirectiveCSharpTokenizer(source); + } +} diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/FirstDirectiveHtmlLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/FirstDirectiveHtmlLanguageCharacteristics.cs new file mode 100644 index 0000000000..2e2c7bea6b --- /dev/null +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/FirstDirectiveHtmlLanguageCharacteristics.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.AspNetCore.Razor.Language.Legacy +{ + internal class FirstDirectiveHtmlLanguageCharacteristics : HtmlLanguageCharacteristics + { + private static readonly FirstDirectiveHtmlLanguageCharacteristics _instance = new FirstDirectiveHtmlLanguageCharacteristics(); + + private FirstDirectiveHtmlLanguageCharacteristics() + { + } + + public new static FirstDirectiveHtmlLanguageCharacteristics Instance => _instance; + + public override HtmlTokenizer CreateTokenizer(ITextDocument source) => new DirectiveHtmlTokenizer(source); + } +} diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs index b035cf1da8..407130510e 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs @@ -10,7 +10,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy { private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics(); - private HtmlLanguageCharacteristics() + protected HtmlLanguageCharacteristics() { } @@ -89,7 +89,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy Debug.Fail("FlipBracket must be called with a bracket character"); #else - Debug.Assert(false, "FlipBracket must be called with a bracket character"); + Debug.Assert(false, "FlipBracket must be called with a bracket character"); #endif return HtmlSymbolType.Unknown; } @@ -130,4 +130,4 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy return new HtmlSymbol(content, type, errors); } } -} +} \ No newline at end of file diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs index 939e5867c9..8b35853989 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs @@ -38,7 +38,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy }; public HtmlMarkupParser(ParserContext context) - : base(HtmlLanguageCharacteristics.Instance, context) + : base(context.StopParsingAfterFirstDirective ? FirstDirectiveHtmlLanguageCharacteristics.Instance : HtmlLanguageCharacteristics.Instance, context) { } diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs index 41c3666559..02b5864458 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.IO; namespace Microsoft.AspNetCore.Razor.Language.Legacy { diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ParserContext.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/ParserContext.cs index d065d566e7..a7e78ca8c9 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ParserContext.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/ParserContext.cs @@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy { internal partial class ParserContext { - public ParserContext(ITextDocument source, bool designTime) + public ParserContext(ITextDocument source, RazorParserOptions options) { if (source == null) { @@ -17,7 +17,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy } Source = source; - DesignTimeMode = designTime; + DesignTimeMode = options.DesignTimeMode; + StopParsingAfterFirstDirective = options.StopParsingAfterFirstDirective; Builder = new SyntaxTreeBuilder(); ErrorSink = new ErrorSink(); } @@ -30,6 +31,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy public bool DesignTimeMode { get; } + public bool StopParsingAfterFirstDirective { get; } + public bool WhiteSpaceIsSignificantToAncestorBlock { get; set; } public bool NullGenerateWhitespaceAndNewLine { get; set; } diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/RazorParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/RazorParser.cs index 684351841e..bf9026cd19 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/RazorParser.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/RazorParser.cs @@ -37,7 +37,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy var reader = new SeekableTextReader(chars, source.FileName); - var context = new ParserContext(reader, Options.DesignTimeMode); + var context = new ParserContext(reader, Options); var codeParser = new CSharpCodeParser(Options.Directives, context); var markupParser = new HtmlMarkupParser(context); diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs index 28ebd29e06..b85f0cd10b 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs @@ -1,9 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.Diagnostics; - namespace Microsoft.AspNetCore.Razor.Language.Legacy { internal class TokenizerView diff --git a/src/Microsoft.AspNetCore.Razor.Language/RazorParserOptions.cs b/src/Microsoft.AspNetCore.Razor.Language/RazorParserOptions.cs index b49def8560..d4e43f25da 100644 --- a/src/Microsoft.AspNetCore.Razor.Language/RazorParserOptions.cs +++ b/src/Microsoft.AspNetCore.Razor.Language/RazorParserOptions.cs @@ -26,6 +26,8 @@ namespace Microsoft.AspNetCore.Razor.Language public bool IsIndentingWithTabs { get; set; } + public bool StopParsingAfterFirstDirective { get; set; } + public ICollection Directives { get; } public HashSet NamespaceImports { get; } diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs new file mode 100644 index 0000000000..dcbbd2b1ef --- /dev/null +++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace Microsoft.AspNetCore.Razor.Language.Legacy +{ + public class DirectiveCSharpTokenizerTest : CSharpTokenizerTestBase + { + [Fact] + public void Next_ReturnsNull_AfterTokenizingFirstDirective() + { + TestTokenizer( + "\r\n @something \r\n @this is ignored", + new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), + new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), + new CSharpSymbol("@", CSharpSymbolType.Transition), + new CSharpSymbol("something", CSharpSymbolType.Identifier), + new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), + new CSharpSymbol("\r\n", CSharpSymbolType.NewLine)); + } + + [Fact] + public void Next_IncludesComments_ReturnsNull_AfterTokenizingFirstDirective() + { + TestTokenizer( + "@*included*@\r\n @something \"value\"\r\n @this is ignored", + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol("included", CSharpSymbolType.RazorComment), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), + new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), + new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), + new CSharpSymbol("@", CSharpSymbolType.Transition), + new CSharpSymbol("something", CSharpSymbolType.Identifier), + new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), + new CSharpSymbol("\"value\"", CSharpSymbolType.StringLiteral), + new CSharpSymbol("\r\n", CSharpSymbolType.NewLine)); + } + + internal override object CreateTokenizer(ITextDocument source) + { + return new DirectiveCSharpTokenizer(source); + } + } +} diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs new file mode 100644 index 0000000000..b75874b681 --- /dev/null +++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace Microsoft.AspNetCore.Razor.Language.Legacy +{ + public class DirectiveHtmlTokenizerTest : HtmlTokenizerTestBase + { + [Fact] + public void Next_ReturnsNull_WhenHtmlIsSeen() + { + TestTokenizer( + "\r\n
Ignored
", + new HtmlSymbol("\r\n", HtmlSymbolType.NewLine), + new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace), + new HtmlSymbol("<", HtmlSymbolType.OpenAngle)); + } + + [Fact] + public void Next_IncludesRazorComments_ReturnsNull_WhenHtmlIsSeen() + { + TestTokenizer( + "\r\n @*included*@
Ignored
", + new HtmlSymbol("\r\n", HtmlSymbolType.NewLine), + new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace), + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol("included", HtmlSymbolType.RazorComment), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), + new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace), + new HtmlSymbol("<", HtmlSymbolType.OpenAngle)); + } + + internal override object CreateTokenizer(ITextDocument source) + { + return new DirectiveHtmlTokenizer(source); + } + } +} diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ParserTestBase.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ParserTestBase.cs index fd8013b7ea..e73b1664b4 100644 --- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ParserTestBase.cs +++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ParserTestBase.cs @@ -37,7 +37,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy var source = TestRazorSourceDocument.Create(document); var reader = new SeekableTextReader(document, filePath: null); - var context = new ParserContext(reader, designTime); + var options = RazorParserOptions.CreateDefaultOptions(); + options.DesignTimeMode = designTime; + var context = new ParserContext(reader, options); var codeParser = new CSharpCodeParser(context); var markupParser = new HtmlMarkupParser(context); @@ -49,8 +51,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy var root = context.Builder.Build(); var diagnostics = context.ErrorSink.Errors?.Select(error => RazorDiagnostic.Create(error)); - var options = RazorParserOptions.CreateDefaultOptions(); - options.DesignTimeMode = designTime; var tree = RazorSyntaxTree.Create(root, source, diagnostics, options); var defaultDirectivePass = new DefaultDirectiveSyntaxTreePass(); @@ -65,7 +65,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy using (var reader = new SeekableTextReader(document, filePath: null)) { - var context = new ParserContext(reader, designTime); + var options = RazorParserOptions.CreateDefaultOptions(); + options.DesignTimeMode = designTime; + var context = new ParserContext(reader, options); var parser = new HtmlMarkupParser(context); parser.CodeParser = new CSharpCodeParser(context) @@ -77,8 +79,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy var root = context.Builder.Build(); var diagnostics = context.ErrorSink.Errors?.Select(error => RazorDiagnostic.Create(error)); - var options = RazorParserOptions.CreateDefaultOptions(); - options.DesignTimeMode = designTime; return RazorSyntaxTree.Create(root, source, diagnostics, options); } @@ -98,7 +98,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy using (var reader = new SeekableTextReader(document, filePath: null)) { - var context = new ParserContext(reader, designTime); + var options = RazorParserOptions.CreateDefaultOptions(); + options.DesignTimeMode = designTime; + var context = new ParserContext(reader, options); var parser = new CSharpCodeParser(descriptors, context); parser.HtmlParser = new HtmlMarkupParser(context) @@ -111,9 +113,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy var root = context.Builder.Build(); var diagnostics = context.ErrorSink.Errors?.Select(error => RazorDiagnostic.Create(error)); - var options = RazorParserOptions.CreateDefaultOptions(); - options.DesignTimeMode = designTime; - options.Directives.Clear(); foreach (var directive in descriptors) { diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/RazorSyntaxTreeTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/RazorSyntaxTreeTest.cs index 02421a67e8..f34b761db9 100644 --- a/test/Microsoft.AspNetCore.Razor.Language.Test/RazorSyntaxTreeTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Language.Test/RazorSyntaxTreeTest.cs @@ -56,5 +56,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Test } } } + + [Fact] + public void Parse_UseDirectiveTokenizer_ParsesUntilFirstDirective() + { + // Arrange + var source = TestRazorSourceDocument.Create("\r\n \r\n @*SomeComment*@ \r\n @tagHelperPrefix \"SomePrefix\"\r\n\r\n@if (true) {\r\n @if(false) {
@something.
} \r\n}"); + var options = RazorParserOptions.CreateDefaultOptions(); + options.StopParsingAfterFirstDirective = true; + + // Act + var syntaxTree = RazorSyntaxTree.Parse(source, options); + + // Assert + Assert.NotNull(syntaxTree); + Assert.Equal(6, syntaxTree.Root.Children.Count); + var block = Assert.IsType(syntaxTree.Root.Children[4]); + Assert.Equal(BlockKind.Directive, block.Type); + Assert.Empty(syntaxTree.Diagnostics); + } } }