[Fixes #1195] Add a way to parse just until the first directive

This commit is contained in:
Ajay Bhargav Baaskaran 2017-04-11 23:14:18 -07:00
parent c0e3519bc3
commit cd486226d6
17 changed files with 310 additions and 49 deletions

View File

@ -10,7 +10,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CSharpCodeParser : TokenizerBackedParser<CSharpTokenizer, CSharpSymbol, CSharpSymbolType>
{
internal static readonly int UsingKeywordLength = 5; // using
private static readonly Func<CSharpSymbol, bool> IsValidStatementSpacingSymbol =
IsSpacingToken(includeNewLines: true, includeComments: true);
@ -88,7 +87,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
public CSharpCodeParser(IEnumerable<DirectiveDescriptor> directiveDescriptors, ParserContext context)
: base(CSharpLanguageCharacteristics.Instance, context)
: base(context.StopParsingAfterFirstDirective ? FirstDirectiveCSharpLanguageCharacteristics.Instance : CSharpLanguageCharacteristics.Instance, context)
{
Keywords = new HashSet<string>();
SetUpKeywords();
@ -106,7 +105,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected void MapDirectives(Action handler, params string[] directives)
{
foreach (string directive in directives)
foreach (var directive in directives)
{
_directiveParsers.Add(directive, handler);
Keywords.Add(directive);
@ -143,7 +142,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void MapKeywords(Action<bool> handler, bool topLevel, params CSharpKeyword[] keywords)
{
foreach (CSharpKeyword keyword in keywords)
foreach (var keyword in keywords)
{
_keywordParsers.Add(keyword, handler);
if (topLevel)
@ -587,17 +586,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaptureWhitespaceAtEndOfCodeOnlyLine()
{
IEnumerable<CSharpSymbol> ws = ReadWhile(sym => sym.Type == CSharpSymbolType.WhiteSpace);
var whitespace = ReadWhile(sym => sym.Type == CSharpSymbolType.WhiteSpace);
if (At(CSharpSymbolType.NewLine))
{
Accept(ws);
Accept(whitespace);
AcceptAndMoveNext();
PutCurrentBack();
}
else
{
PutCurrentBack();
PutBack(ws);
PutBack(whitespace);
}
}
@ -807,11 +806,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void WhileClause()
{
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.Any;
IEnumerable<CSharpSymbol> ws = SkipToNextImportantToken();
var whitespace = SkipToNextImportantToken();
if (At(CSharpKeyword.While))
{
Accept(ws);
Accept(whitespace);
Assert(CSharpKeyword.While);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -823,7 +822,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
PutCurrentBack();
PutBack(ws);
PutBack(whitespace);
}
}
@ -1042,12 +1041,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void AfterIfClause()
{
// Grab whitespace and razor comments
IEnumerable<CSharpSymbol> ws = SkipToNextImportantToken();
var whitespace = SkipToNextImportantToken();
// Check for an else part
if (At(CSharpKeyword.Else))
{
Accept(ws);
Accept(whitespace);
Assert(CSharpKeyword.Else);
ElseClause();
}
@ -1055,7 +1054,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// No else, return whitespace
PutCurrentBack();
PutBack(ws);
PutBack(whitespace);
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.Any;
}
}
@ -1355,13 +1354,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
while (!EndOfFile)
{
var bookmark = CurrentStart.AbsoluteIndex;
IEnumerable<CSharpSymbol> read = ReadWhile(sym => sym.Type != CSharpSymbolType.Semicolon &&
sym.Type != CSharpSymbolType.RazorCommentTransition &&
sym.Type != CSharpSymbolType.Transition &&
sym.Type != CSharpSymbolType.LeftBrace &&
sym.Type != CSharpSymbolType.LeftParenthesis &&
sym.Type != CSharpSymbolType.LeftBracket &&
sym.Type != CSharpSymbolType.RightBrace);
var read = ReadWhile(sym =>
sym.Type != CSharpSymbolType.Semicolon &&
sym.Type != CSharpSymbolType.RazorCommentTransition &&
sym.Type != CSharpSymbolType.Transition &&
sym.Type != CSharpSymbolType.LeftBrace &&
sym.Type != CSharpSymbolType.LeftParenthesis &&
sym.Type != CSharpSymbolType.LeftBracket &&
sym.Type != CSharpSymbolType.RightBrace);
if (At(CSharpSymbolType.LeftBrace) ||
At(CSharpSymbolType.LeftParenthesis) ||
At(CSharpSymbolType.LeftBracket))
@ -1443,8 +1444,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void HandleKeyword(bool topLevel, Action fallback)
{
Debug.Assert(CurrentSymbol.Type == CSharpSymbolType.Keyword && CurrentSymbol.Keyword != null);
Action<bool> handler;
if (_keywordParsers.TryGetValue(CurrentSymbol.Keyword.Value, out handler))
if (_keywordParsers.TryGetValue(CurrentSymbol.Keyword.Value, out var handler))
{
handler(topLevel);
}
@ -1458,16 +1458,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
while (!EndOfFile)
{
IEnumerable<CSharpSymbol> ws = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (At(CSharpSymbolType.RazorCommentTransition))
{
Accept(ws);
Accept(whitespace);
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.Any;
RazorComment();
}
else
{
return ws;
return whitespace;
}
}
return Enumerable.Empty<CSharpSymbol>();
@ -1814,7 +1814,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Pull out the type name
string baseType = string.Concat(Span.Symbols.Select(s => s.Content));
var baseType = string.Concat(Span.Symbols.Select(s => s.Content));
// Set up chunk generation
Span.ChunkGenerator = createChunkGenerator(baseType.Trim());

View File

@ -63,7 +63,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ CSharpSymbolType.Transition, "@" },
};
private CSharpLanguageCharacteristics()
protected CSharpLanguageCharacteristics()
{
}
@ -172,4 +172,4 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return keyword.ToString().ToLowerInvariant();
}
}
}
}

View File

@ -0,0 +1,62 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class DirectiveCSharpTokenizer : CSharpTokenizer
{
private bool _visitedFirstTokenStart = false;
private bool _visitedFirstTokenLineEnd = false;
public DirectiveCSharpTokenizer(ITextDocument source) : base(source)
{
}
protected override StateResult Dispatch()
{
var result = base.Dispatch();
if (result.Result != null && !_visitedFirstTokenStart && IsValidTokenType(result.Result.Type))
{
_visitedFirstTokenStart = true;
}
else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpSymbolType.NewLine)
{
_visitedFirstTokenLineEnd = true;
}
return result;
}
public override CSharpSymbol NextSymbol()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
StartSymbol();
if (EndOfFile || (_visitedFirstTokenStart && _visitedFirstTokenLineEnd))
{
return null;
}
var symbol = Turn();
// Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0);
return symbol;
}
private bool IsValidTokenType(CSharpSymbolType type)
{
return type != CSharpSymbolType.WhiteSpace &&
type != CSharpSymbolType.NewLine &&
type != CSharpSymbolType.Comment &&
type != CSharpSymbolType.RazorComment &&
type != CSharpSymbolType.RazorCommentStar &&
type != CSharpSymbolType.RazorCommentTransition &&
type != CSharpSymbolType.Transition;
}
}
}

View File

@ -0,0 +1,56 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class DirectiveHtmlTokenizer : HtmlTokenizer
{
private bool _visitedFirstTokenStart = false;
public DirectiveHtmlTokenizer(ITextDocument source) : base(source)
{
}
protected override StateResult Dispatch()
{
var result = base.Dispatch();
if (result.Result != null && IsValidTokenType(result.Result.Type))
{
_visitedFirstTokenStart = true;
}
return result;
}
public override HtmlSymbol NextSymbol()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
StartSymbol();
if (EndOfFile || _visitedFirstTokenStart)
{
return null;
}
var symbol = Turn();
// Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0);
return symbol;
}
private bool IsValidTokenType(HtmlSymbolType type)
{
return type != HtmlSymbolType.WhiteSpace &&
type != HtmlSymbolType.NewLine &&
type != HtmlSymbolType.RazorComment &&
type != HtmlSymbolType.RazorCommentStar &&
type != HtmlSymbolType.RazorCommentTransition &&
type != HtmlSymbolType.Transition;
}
}
}

View File

@ -0,0 +1,18 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class FirstDirectiveCSharpLanguageCharacteristics : CSharpLanguageCharacteristics
{
private static readonly FirstDirectiveCSharpLanguageCharacteristics _instance = new FirstDirectiveCSharpLanguageCharacteristics();
private FirstDirectiveCSharpLanguageCharacteristics()
{
}
public new static FirstDirectiveCSharpLanguageCharacteristics Instance => _instance;
public override CSharpTokenizer CreateTokenizer(ITextDocument source) => new DirectiveCSharpTokenizer(source);
}
}

View File

@ -0,0 +1,18 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class FirstDirectiveHtmlLanguageCharacteristics : HtmlLanguageCharacteristics
{
private static readonly FirstDirectiveHtmlLanguageCharacteristics _instance = new FirstDirectiveHtmlLanguageCharacteristics();
private FirstDirectiveHtmlLanguageCharacteristics()
{
}
public new static FirstDirectiveHtmlLanguageCharacteristics Instance => _instance;
public override HtmlTokenizer CreateTokenizer(ITextDocument source) => new DirectiveHtmlTokenizer(source);
}
}

View File

@ -10,7 +10,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics();
private HtmlLanguageCharacteristics()
protected HtmlLanguageCharacteristics()
{
}
@ -89,7 +89,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Debug.Fail("FlipBracket must be called with a bracket character");
#else
Debug.Assert(false, "FlipBracket must be called with a bracket character");
Debug.Assert(false, "FlipBracket must be called with a bracket character");
#endif
return HtmlSymbolType.Unknown;
}
@ -130,4 +130,4 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlSymbol(content, type, errors);
}
}
}
}

View File

@ -38,7 +38,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
};
public HtmlMarkupParser(ParserContext context)
: base(HtmlLanguageCharacteristics.Instance, context)
: base(context.StopParsingAfterFirstDirective ? FirstDirectiveHtmlLanguageCharacteristics.Instance : HtmlLanguageCharacteristics.Instance, context)
{
}

View File

@ -3,7 +3,6 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{

View File

@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal partial class ParserContext
{
public ParserContext(ITextDocument source, bool designTime)
public ParserContext(ITextDocument source, RazorParserOptions options)
{
if (source == null)
{
@ -17,7 +17,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
Source = source;
DesignTimeMode = designTime;
DesignTimeMode = options.DesignTimeMode;
StopParsingAfterFirstDirective = options.StopParsingAfterFirstDirective;
Builder = new SyntaxTreeBuilder();
ErrorSink = new ErrorSink();
}
@ -30,6 +31,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public bool DesignTimeMode { get; }
public bool StopParsingAfterFirstDirective { get; }
public bool WhiteSpaceIsSignificantToAncestorBlock { get; set; }
public bool NullGenerateWhitespaceAndNewLine { get; set; }

View File

@ -37,7 +37,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var reader = new SeekableTextReader(chars, source.FileName);
var context = new ParserContext(reader, Options.DesignTimeMode);
var context = new ParserContext(reader, Options);
var codeParser = new CSharpCodeParser(Options.Directives, context);
var markupParser = new HtmlMarkupParser(context);

View File

@ -1,9 +1,6 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class TokenizerView<TTokenizer, TSymbol, TSymbolType>

View File

@ -26,6 +26,8 @@ namespace Microsoft.AspNetCore.Razor.Language
public bool IsIndentingWithTabs { get; set; }
public bool StopParsingAfterFirstDirective { get; set; }
public ICollection<DirectiveDescriptor> Directives { get; }
public HashSet<string> NamespaceImports { get; }

View File

@ -0,0 +1,47 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class DirectiveCSharpTokenizerTest : CSharpTokenizerTestBase
{
[Fact]
public void Next_ReturnsNull_AfterTokenizingFirstDirective()
{
TestTokenizer(
"\r\n @something \r\n @this is ignored",
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol("@", CSharpSymbolType.Transition),
new CSharpSymbol("something", CSharpSymbolType.Identifier),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine));
}
[Fact]
public void Next_IncludesComments_ReturnsNull_AfterTokenizingFirstDirective()
{
TestTokenizer(
"@*included*@\r\n @something \"value\"\r\n @this is ignored",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol("included", CSharpSymbolType.RazorComment),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol("@", CSharpSymbolType.Transition),
new CSharpSymbol("something", CSharpSymbolType.Identifier),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol("\"value\"", CSharpSymbolType.StringLiteral),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine));
}
internal override object CreateTokenizer(ITextDocument source)
{
return new DirectiveCSharpTokenizer(source);
}
}
}

View File

@ -0,0 +1,41 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class DirectiveHtmlTokenizerTest : HtmlTokenizerTestBase
{
[Fact]
public void Next_ReturnsNull_WhenHtmlIsSeen()
{
TestTokenizer(
"\r\n <div>Ignored</div>",
new HtmlSymbol("\r\n", HtmlSymbolType.NewLine),
new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace),
new HtmlSymbol("<", HtmlSymbolType.OpenAngle));
}
[Fact]
public void Next_IncludesRazorComments_ReturnsNull_WhenHtmlIsSeen()
{
TestTokenizer(
"\r\n @*included*@ <div>Ignored</div>",
new HtmlSymbol("\r\n", HtmlSymbolType.NewLine),
new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol("included", HtmlSymbolType.RazorComment),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace),
new HtmlSymbol("<", HtmlSymbolType.OpenAngle));
}
internal override object CreateTokenizer(ITextDocument source)
{
return new DirectiveHtmlTokenizer(source);
}
}
}

View File

@ -37,7 +37,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var source = TestRazorSourceDocument.Create(document);
var reader = new SeekableTextReader(document, filePath: null);
var context = new ParserContext(reader, designTime);
var options = RazorParserOptions.CreateDefaultOptions();
options.DesignTimeMode = designTime;
var context = new ParserContext(reader, options);
var codeParser = new CSharpCodeParser(context);
var markupParser = new HtmlMarkupParser(context);
@ -49,8 +51,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var root = context.Builder.Build();
var diagnostics = context.ErrorSink.Errors?.Select(error => RazorDiagnostic.Create(error));
var options = RazorParserOptions.CreateDefaultOptions();
options.DesignTimeMode = designTime;
var tree = RazorSyntaxTree.Create(root, source, diagnostics, options);
var defaultDirectivePass = new DefaultDirectiveSyntaxTreePass();
@ -65,7 +65,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
using (var reader = new SeekableTextReader(document, filePath: null))
{
var context = new ParserContext(reader, designTime);
var options = RazorParserOptions.CreateDefaultOptions();
options.DesignTimeMode = designTime;
var context = new ParserContext(reader, options);
var parser = new HtmlMarkupParser(context);
parser.CodeParser = new CSharpCodeParser(context)
@ -77,8 +79,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var root = context.Builder.Build();
var diagnostics = context.ErrorSink.Errors?.Select(error => RazorDiagnostic.Create(error));
var options = RazorParserOptions.CreateDefaultOptions();
options.DesignTimeMode = designTime;
return RazorSyntaxTree.Create(root, source, diagnostics, options);
}
@ -98,7 +98,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
using (var reader = new SeekableTextReader(document, filePath: null))
{
var context = new ParserContext(reader, designTime);
var options = RazorParserOptions.CreateDefaultOptions();
options.DesignTimeMode = designTime;
var context = new ParserContext(reader, options);
var parser = new CSharpCodeParser(descriptors, context);
parser.HtmlParser = new HtmlMarkupParser(context)
@ -111,9 +113,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var root = context.Builder.Build();
var diagnostics = context.ErrorSink.Errors?.Select(error => RazorDiagnostic.Create(error));
var options = RazorParserOptions.CreateDefaultOptions();
options.DesignTimeMode = designTime;
options.Directives.Clear();
foreach (var directive in descriptors)
{

View File

@ -56,5 +56,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Test
}
}
}
[Fact]
public void Parse_UseDirectiveTokenizer_ParsesUntilFirstDirective()
{
// Arrange
var source = TestRazorSourceDocument.Create("\r\n \r\n @*SomeComment*@ \r\n @tagHelperPrefix \"SomePrefix\"\r\n<html>\r\n@if (true) {\r\n @if(false) { <div>@something.</div> } \r\n}");
var options = RazorParserOptions.CreateDefaultOptions();
options.StopParsingAfterFirstDirective = true;
// Act
var syntaxTree = RazorSyntaxTree.Parse(source, options);
// Assert
Assert.NotNull(syntaxTree);
Assert.Equal(6, syntaxTree.Root.Children.Count);
var block = Assert.IsType<Block>(syntaxTree.Root.Children[4]);
Assert.Equal(BlockKind.Directive, block.Type);
Assert.Empty(syntaxTree.Diagnostics);
}
}
}