huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated the tokenizer and parser to operate on SyntaxTokens directly (#2540) 

* Updated the tokenizer and parser to operate on SyntaxTokens directly
- The tokenizer and parser now operates directly on SyntaxTokens (Green)
- The SyntaxToken(Red) is now created when the Span is built with the correct parent and position
- All other passes that run after the parsing is complete(TagHelperRewriter etc) will operate on the Red tokens.
- There is now only one type for all SyntaxTokens. They are differentiated by their SyntaxKind.
- Added equivalence checking for tokens
- Updated test code to react
- Regenerated baselines
This commit is contained in:
Ajay Bhargav Baaskaran 2018-08-16 16:11:01 -07:00 committed by GitHub
parent 0c25c2958c
commit db2a142132
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
996 changed files with 16796 additions and 17288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs
View File

@ -595,9 +595,9 @@ namespace Microsoft.AspNetCore.Razor.Language
{
if (span.Tokens.Count == 1)
{
var token = span.Tokens[0] as HtmlToken;
var token = span.Tokens[0];
if (token != null &&
token.Type == HtmlTokenType.Unknown &&
token.Kind == SyntaxKind.Unknown &&
token.Content.Length == 0)
{
// We don't want to create IR nodes for marker tokens.

3
src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs
View File

@ -4,12 +4,13 @@
using System;
using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language
{
internal class DirectiveTokenEditHandler : SpanEditHandler
{
public DirectiveTokenEditHandler(Func<string, IEnumerable<IToken>> tokenizer) : base(tokenizer)
public DirectiveTokenEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer) : base(tokenizer)
{
}

7
src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs
View File

@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -11,18 +12,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
private static readonly int TypeHashCode = typeof(AutoCompleteEditHandler).GetHashCode();
public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer)
public AutoCompleteEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer)
: base(tokenizer)
{
}
public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer, bool autoCompleteAtEndOfSpan)
public AutoCompleteEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer, bool autoCompleteAtEndOfSpan)
: this(tokenizer)
{
AutoCompleteAtEndOfSpan = autoCompleteAtEndOfSpan;
}
public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer, AcceptedCharactersInternal accepted)
public AutoCompleteEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer, AcceptedCharactersInternal accepted)
: base(tokenizer, accepted)
{
}

328
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
View File

@ -5,17 +5,18 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CSharpCodeParser : TokenizerBackedParser<CSharpTokenizer, CSharpToken, CSharpTokenType>
internal class CSharpCodeParser : TokenizerBackedParser<CSharpTokenizer>
{
private static HashSet<char> InvalidNonWhitespaceNameCharacters = new HashSet<char>(new[]
{
'@', '!', '<', '/', '?', '[', '>', ']', '=', '"', '\'', '*'
});
private static readonly Func<CSharpToken, bool> IsValidStatementSpacingToken =
private static readonly Func<SyntaxToken, bool> IsValidStatementSpacingToken =
IsSpacingToken(includeNewLines: true, includeComments: true);
internal static readonly DirectiveDescriptor AddTagHelperDirectiveDescriptor = DirectiveDescriptor.CreateDirective(
@ -102,7 +103,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public bool IsNested { get; set; }
protected override bool TokenTypeEquals(CSharpTokenType x, CSharpTokenType y) => x == y;
protected override bool TokenKindEquals(SyntaxKind x, SyntaxKind y) => x == y;
protected void MapDirectives(Action handler, params string[] directives)
{
@ -161,16 +162,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Conditional("DEBUG")]
internal void Assert(CSharpKeyword expectedKeyword)
{
Debug.Assert(CurrentToken.Type == CSharpTokenType.Keyword &&
CurrentToken.Keyword.HasValue &&
CurrentToken.Keyword.Value == expectedKeyword);
var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
Debug.Assert(CurrentToken.Kind == SyntaxKind.Keyword &&
result.HasValue &&
result.Value == expectedKeyword);
}
protected internal bool At(CSharpKeyword keyword)
{
return At(CSharpTokenType.Keyword) &&
CurrentToken.Keyword.HasValue &&
CurrentToken.Keyword.Value == keyword;
var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
return At(SyntaxKind.Keyword) &&
result.HasValue &&
result.Value == keyword;
}
protected internal bool AcceptIf(CSharpKeyword keyword)
@ -183,11 +186,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
protected static Func<CSharpToken, bool> IsSpacingToken(bool includeNewLines, bool includeComments)
protected static Func<SyntaxToken, bool> IsSpacingToken(bool includeNewLines, bool includeComments)
{
return token => token.Type == CSharpTokenType.WhiteSpace ||
(includeNewLines && token.Type == CSharpTokenType.NewLine) ||
(includeComments && token.Type == CSharpTokenType.Comment);
return token => token.Kind == SyntaxKind.Whitespace ||
(includeNewLines && token.Kind == SyntaxKind.NewLine) ||
(includeComments && token.Kind == SyntaxKind.CSharpComment);
}
public override void ParseBlock()
@ -209,24 +212,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
var current = CurrentToken;
if (At(CSharpTokenType.StringLiteral) &&
if (At(SyntaxKind.StringLiteral) &&
CurrentToken.Content.Length > 0 &&
CurrentToken.Content[0] == SyntaxConstants.TransitionCharacter)
{
var split = Language.SplitToken(CurrentToken, 1, CSharpTokenType.Transition);
var split = Language.SplitToken(CurrentToken, 1, SyntaxKind.Transition);
current = split.Item1;
// Back up to the end of the transition
Context.Source.Position -= split.Item2.Content.Length;
NextToken();
}
else if (At(CSharpTokenType.Transition))
else if (At(SyntaxKind.Transition))
{
NextToken();
}
// Accept "@" if we see it, but if we don't, that's OK. We assume we were started for a good reason
if (current.Type == CSharpTokenType.Transition)
if (current.Kind == SyntaxKind.Transition)
{
if (Span.Tokens.Count > 0)
{
@ -251,9 +254,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.ChunkGenerator = new StatementChunkGenerator();
}
private void AtTransition(CSharpToken current)
private void AtTransition(SyntaxToken current)
{
Debug.Assert(current.Type == CSharpTokenType.Transition);
Debug.Assert(current.Kind == SyntaxKind.Transition);
Accept(current);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
@ -273,14 +276,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// What type of block is this?
if (!EndOfFile)
{
if (CurrentToken.Type == CSharpTokenType.LeftParenthesis)
if (CurrentToken.Kind == SyntaxKind.LeftParenthesis)
{
Context.Builder.CurrentBlock.Type = BlockKindInternal.Expression;
Context.Builder.CurrentBlock.ChunkGenerator = new ExpressionChunkGenerator();
ExplicitExpression();
return;
}
else if (CurrentToken.Type == CSharpTokenType.Identifier)
else if (CurrentToken.Kind == SyntaxKind.Identifier)
{
if (TryGetDirectiveHandler(CurrentToken.Content, out var handler))
{
@ -306,7 +309,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
}
else if (CurrentToken.Type == CSharpTokenType.Keyword)
else if (CurrentToken.Kind == SyntaxKind.Keyword)
{
if (TryGetDirectiveHandler(CurrentToken.Content, out var handler))
{
@ -320,7 +323,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
}
else if (CurrentToken.Type == CSharpTokenType.LeftBrace)
else if (CurrentToken.Kind == SyntaxKind.LeftBrace)
{
VerbatimBlock();
return;
@ -339,7 +342,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
AcceptedCharacters = AcceptedCharactersInternal.NonWhiteSpace
};
if (At(CSharpTokenType.WhiteSpace) || At(CSharpTokenType.NewLine))
if (At(SyntaxKind.Whitespace) || At(SyntaxKind.NewLine))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedWhiteSpaceAtStartOfCodeBlock(
@ -369,7 +372,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void VerbatimBlock()
{
Assert(CSharpTokenType.LeftBrace);
Assert(SyntaxKind.LeftBrace);
var block = new Block(Resources.BlockName_Code, CurrentStart);
AcceptAndMoveNext();
@ -385,13 +388,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = new StatementChunkGenerator();
AddMarkerTokenIfNecessary();
if (!At(CSharpTokenType.RightBrace))
if (!At(SyntaxKind.RightBrace))
{
editHandler.AutoCompleteString = "}";
}
Output(SpanKindInternal.Code);
if (Optional(CSharpTokenType.RightBrace))
if (Optional(SyntaxKind.RightBrace))
{
// Set up the "}" span
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
@ -401,8 +404,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!IsNested)
{
EnsureCurrent();
if (At(CSharpTokenType.NewLine) ||
(At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.NewLine)))
if (At(SyntaxKind.NewLine) ||
(At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.NewLine)))
{
Context.NullGenerateWhitespaceAndNewLine = true;
}
@ -456,13 +459,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
if (!EndOfFile)
{
if (CurrentToken.Type == CSharpTokenType.LeftParenthesis ||
CurrentToken.Type == CSharpTokenType.LeftBracket)
if (CurrentToken.Kind == SyntaxKind.LeftParenthesis ||
CurrentToken.Kind == SyntaxKind.LeftBracket)
{
// If we end within "(", whitespace is fine
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
CSharpTokenType right;
SyntaxKind right;
bool success;
using (PushSpanConfig((span, prev) =>
@ -471,13 +474,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
}))
{
right = Language.FlipBracket(CurrentToken.Type);
right = Language.FlipBracket(CurrentToken.Kind);
success = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
}
if (!success)
{
AcceptUntil(CSharpTokenType.LessThan);
AcceptUntil(SyntaxKind.LessThan);
}
if (At(right))
{
@ -488,22 +491,22 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
return MethodCallOrArrayIndex(acceptedCharacters);
}
if (At(CSharpTokenType.QuestionMark))
if (At(SyntaxKind.QuestionMark))
{
var next = Lookahead(count: 1);
if (next != null)
{
if (next.Type == CSharpTokenType.Dot)
if (next.Kind == SyntaxKind.Dot)
{
// Accept null conditional dot operator (?.).
AcceptAndMoveNext();
AcceptAndMoveNext();
// If the next piece after the ?. is a keyword or identifier then we want to continue.
return At(CSharpTokenType.Identifier) || At(CSharpTokenType.Keyword);
return At(SyntaxKind.Identifier) || At(SyntaxKind.Keyword);
}
else if (next.Type == CSharpTokenType.LeftBracket)
else if (next.Kind == SyntaxKind.LeftBracket)
{
// We're at the ? for a null conditional bracket operator (?[).
AcceptAndMoveNext();
@ -513,12 +516,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
}
else if (At(CSharpTokenType.Dot))
else if (At(SyntaxKind.Dot))
{
var dot = CurrentToken;
if (NextToken())
{
if (At(CSharpTokenType.Identifier) || At(CSharpTokenType.Keyword))
if (At(SyntaxKind.Identifier) || At(SyntaxKind.Keyword))
{
// Accept the dot and return to the start
Accept(dot);
@ -540,7 +543,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(dot);
}
}
else if (!At(CSharpTokenType.WhiteSpace) && !At(CSharpTokenType.NewLine))
else if (!At(SyntaxKind.Whitespace) && !At(SyntaxKind.NewLine))
{
PutCurrentBack();
}
@ -587,8 +590,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaptureWhitespaceAtEndOfCodeOnlyLine()
{
var whitespace = ReadWhile(token => token.Type == CSharpTokenType.WhiteSpace);
if (At(CSharpTokenType.NewLine))
var whitespace = ReadWhile(token => token.Kind == SyntaxKind.Whitespace);
if (At(SyntaxKind.NewLine))
{
Accept(whitespace);
AcceptAndMoveNext();
@ -610,7 +613,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ExplicitExpression()
{
var block = new Block(Resources.BlockName_ExplicitExpression, CurrentStart);
Assert(CSharpTokenType.LeftParenthesis);
Assert(SyntaxKind.LeftParenthesis);
AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
@ -621,13 +624,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
BalancingModes.BacktrackOnFailure |
BalancingModes.NoErrorOnFailure |
BalancingModes.AllowCommentsAndTemplates,
CSharpTokenType.LeftParenthesis,
CSharpTokenType.RightParenthesis,
SyntaxKind.LeftParenthesis,
SyntaxKind.RightParenthesis,
block.Start);
if (!success)
{
AcceptUntil(CSharpTokenType.LessThan);
AcceptUntil(SyntaxKind.LessThan);
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF(
new SourceSpan(block.Start, contentLength: 1 /* ( */), block.Name, ")", "("));
@ -636,13 +639,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// If necessary, put an empty-content marker token here
if (Span.Tokens.Count == 0)
{
Accept(new CSharpToken(string.Empty, CSharpTokenType.Unknown));
Accept(SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty));
}
// Output the content span and then capture the ")"
Output(SpanKindInternal.Code);
}
Optional(CSharpTokenType.RightParenthesis);
Optional(SyntaxKind.RightParenthesis);
if (!EndOfFile)
{
PutCurrentBack();
@ -702,7 +705,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// No embedded transitions in C#, so ignore that param
return allowTemplatesAndComments
&& ((Language.IsTransition(CurrentToken)
&& NextIs(CSharpTokenType.LessThan, CSharpTokenType.Colon, CSharpTokenType.DoubleColon))
&& NextIs(SyntaxKind.LessThan, SyntaxKind.Colon, SyntaxKind.DoubleColon))
|| Language.IsCommentStart(CurrentToken));
}
@ -783,12 +786,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaseStatement(bool topLevel)
{
Assert(CSharpTokenType.Keyword);
Debug.Assert(CurrentToken.Keyword != null &&
(CurrentToken.Keyword.Value == CSharpKeyword.Case ||
CurrentToken.Keyword.Value == CSharpKeyword.Default));
AcceptUntil(CSharpTokenType.Colon);
Optional(CSharpTokenType.Colon);
Assert(SyntaxKind.Keyword);
var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
Debug.Assert(result.HasValue &&
(result.Value == CSharpKeyword.Case ||
result.Value == CSharpKeyword.Default));
AcceptUntil(SyntaxKind.Colon);
Optional(SyntaxKind.Colon);
}
private void DoStatement(bool topLevel)
@ -813,7 +817,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Assert(CSharpKeyword.While);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (AcceptCondition() && Optional(CSharpTokenType.Semicolon))
if (AcceptCondition() && Optional(SyntaxKind.Semicolon))
{
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
}
@ -832,12 +836,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
if (At(CSharpTokenType.LeftParenthesis))
if (At(SyntaxKind.LeftParenthesis))
{
// using ( ==> Using Statement
UsingStatement(block);
}
else if (At(CSharpTokenType.Identifier) || At(CSharpKeyword.Static))
else if (At(SyntaxKind.Identifier) || At(CSharpKeyword.Static))
{
// using Identifier ==> Using Declaration
if (!topLevel)
@ -865,16 +869,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Context.Builder.CurrentBlock.Type = BlockKindInternal.Directive;
var start = CurrentStart;
if (At(CSharpTokenType.Identifier))
if (At(SyntaxKind.Identifier))
{
// non-static using
NamespaceOrTypeName();
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (At(CSharpTokenType.Assign))
if (At(SyntaxKind.Assign))
{
// Alias
Accept(whitespace);
Assert(CSharpTokenType.Assign);
Assert(SyntaxKind.Assign);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -904,7 +908,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Optional ";"
if (EnsureCurrent())
{
Optional(CSharpTokenType.Semicolon);
Optional(SyntaxKind.Semicolon);
}
}
@ -919,16 +923,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var expectingDot = false;
var tokens = ReadWhile(token =>
{
var type = token.Type;
if ((expectingDot && type == CSharpTokenType.Dot) ||
(!expectingDot && type == CSharpTokenType.Identifier))
var type = token.Kind;
if ((expectingDot && type == SyntaxKind.Dot) ||
(!expectingDot && type == SyntaxKind.Identifier))
{
expectingDot = !expectingDot;
return true;
}
if (type != CSharpTokenType.WhiteSpace &&
type != CSharpTokenType.NewLine)
if (type != SyntaxKind.Whitespace &&
type != SyntaxKind.NewLine)
{
expectingDot = false;
currentIdentifierLength += token.Content.Length;
@ -966,69 +970,69 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool NamespaceOrTypeName()
{
if (Optional(CSharpTokenType.LeftParenthesis))
if (Optional(SyntaxKind.LeftParenthesis))
{
while (!Optional(CSharpTokenType.RightParenthesis) && !EndOfFile)
while (!Optional(SyntaxKind.RightParenthesis) && !EndOfFile)
{
Optional(CSharpTokenType.WhiteSpace);
Optional(SyntaxKind.Whitespace);
if (!NamespaceOrTypeName())
{
return false;
}
Optional(CSharpTokenType.WhiteSpace);
Optional(CSharpTokenType.Identifier);
Optional(CSharpTokenType.WhiteSpace);
Optional(CSharpTokenType.Comma);
Optional(SyntaxKind.Whitespace);
Optional(SyntaxKind.Identifier);
Optional(SyntaxKind.Whitespace);
Optional(SyntaxKind.Comma);
}
if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.QuestionMark))
if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.QuestionMark))
{
// Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext();
}
Optional(CSharpTokenType.QuestionMark); // Nullable
Optional(SyntaxKind.QuestionMark); // Nullable
return true;
}
else if (Optional(CSharpTokenType.Identifier) || Optional(CSharpTokenType.Keyword))
else if (Optional(SyntaxKind.Identifier) || Optional(SyntaxKind.Keyword))
{
if (Optional(CSharpTokenType.DoubleColon))
if (Optional(SyntaxKind.DoubleColon))
{
if (!Optional(CSharpTokenType.Identifier))
if (!Optional(SyntaxKind.Identifier))
{
Optional(CSharpTokenType.Keyword);
Optional(SyntaxKind.Keyword);
}
}
if (At(CSharpTokenType.LessThan))
if (At(SyntaxKind.LessThan))
{
TypeArgumentList();
}
if (Optional(CSharpTokenType.Dot))
if (Optional(SyntaxKind.Dot))
{
NamespaceOrTypeName();
}
if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.QuestionMark))
if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.QuestionMark))
{
// Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext();
}
Optional(CSharpTokenType.QuestionMark); // Nullable
Optional(SyntaxKind.QuestionMark); // Nullable
if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.LeftBracket))
if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.LeftBracket))
{
// Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext();
}
while (At(CSharpTokenType.LeftBracket))
while (At(SyntaxKind.LeftBracket))
{
Balance(BalancingModes.None);
Optional(CSharpTokenType.RightBracket);
Optional(SyntaxKind.RightBracket);
}
return true;
}
@ -1040,14 +1044,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TypeArgumentList()
{
Assert(CSharpTokenType.LessThan);
Assert(SyntaxKind.LessThan);
Balance(BalancingModes.None);
Optional(CSharpTokenType.GreaterThan);
Optional(SyntaxKind.GreaterThan);
}
private void UsingStatement(Block block)
{
Assert(CSharpTokenType.LeftParenthesis);
Assert(SyntaxKind.LeftParenthesis);
// Parse condition
if (AcceptCondition())
@ -1159,12 +1163,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!EndOfFile)
{
// Check for "{" to make sure we're at a block
if (!At(CSharpTokenType.LeftBrace))
if (!At(SyntaxKind.LeftBrace))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_SingleLineControlFlowStatementsNotAllowed(
new SourceSpan(CurrentStart, CurrentToken.Content.Length),
Language.GetSample(CSharpTokenType.LeftBrace),
Language.GetSample(SyntaxKind.LeftBrace),
CurrentToken.Content));
}
@ -1175,7 +1179,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void UnconditionalBlock()
{
Assert(CSharpTokenType.Keyword);
Assert(SyntaxKind.Keyword);
var block = new Block(CurrentToken, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -1219,7 +1223,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ConditionalBlock(bool topLevel)
{
Assert(CSharpTokenType.Keyword);
Assert(SyntaxKind.Keyword);
var block = new Block(CurrentToken, CurrentStart);
ConditionalBlock(block);
if (topLevel)
@ -1243,16 +1247,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool AcceptCondition()
{
if (At(CSharpTokenType.LeftParenthesis))
if (At(SyntaxKind.LeftParenthesis))
{
var complete = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
if (!complete)
{
AcceptUntil(CSharpTokenType.NewLine);
AcceptUntil(SyntaxKind.NewLine);
}
else
{
Optional(CSharpTokenType.RightParenthesis);
Optional(SyntaxKind.RightParenthesis);
}
return complete;
}
@ -1280,16 +1284,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
var type = CurrentToken.Type;
var type = CurrentToken.Kind;
var loc = CurrentStart;
// Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text
var isSingleLineMarkup = type == CSharpTokenType.Transition &&
(NextIs(CSharpTokenType.Colon, CSharpTokenType.DoubleColon));
var isSingleLineMarkup = type == SyntaxKind.Transition &&
(NextIs(SyntaxKind.Colon, SyntaxKind.DoubleColon));
var isMarkup = isSingleLineMarkup ||
type == CSharpTokenType.LessThan ||
(type == CSharpTokenType.Transition && NextIs(CSharpTokenType.LessThan));
type == SyntaxKind.LessThan ||
(type == SyntaxKind.Transition && NextIs(SyntaxKind.LessThan));
if (Context.DesignTimeMode || !isMarkup)
{
@ -1321,7 +1325,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (isMarkup)
{
if (type == CSharpTokenType.Transition && !isSingleLineMarkup)
if (type == SyntaxKind.Transition && !isSingleLineMarkup)
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_AtInCodeMustBeFollowedByColonParenOrIdentifierStart(
@ -1331,7 +1335,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Markup block
Output(SpanKindInternal.Code);
if (Context.DesignTimeMode && CurrentToken != null &&
(CurrentToken.Type == CSharpTokenType.LessThan || CurrentToken.Type == CSharpTokenType.Transition))
(CurrentToken.Kind == SyntaxKind.LessThan || CurrentToken.Kind == SyntaxKind.Transition))
{
PutCurrentBack();
}
@ -1344,33 +1348,33 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
private void HandleStatement(Block block, CSharpTokenType type)
private void HandleStatement(Block block, SyntaxKind type)
{
switch (type)
{
case CSharpTokenType.RazorCommentTransition:
case SyntaxKind.RazorCommentTransition:
Output(SpanKindInternal.Code);
RazorComment();
Statement(block);
break;
case CSharpTokenType.LeftBrace:
case SyntaxKind.LeftBrace:
// Verbatim Block
block = block ?? new Block(Resources.BlockName_Code, CurrentStart);
AcceptAndMoveNext();
CodeBlock(block);
break;
case CSharpTokenType.Keyword:
case SyntaxKind.Keyword:
// Keyword block
HandleKeyword(false, StandardStatement);
break;
case CSharpTokenType.Transition:
case SyntaxKind.Transition:
// Embedded Expression block
EmbeddedExpression();
break;
case CSharpTokenType.RightBrace:
case SyntaxKind.RightBrace:
// Possible end of Code Block, just run the continuation
break;
case CSharpTokenType.Comment:
case SyntaxKind.CSharpComment:
AcceptAndMoveNext();
break;
default:
@ -1383,11 +1387,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void EmbeddedExpression()
{
// First, verify the type of the block
Assert(CSharpTokenType.Transition);
Assert(SyntaxKind.Transition);
var transition = CurrentToken;
NextToken();
if (At(CSharpTokenType.Transition))
if (At(SyntaxKind.Transition))
{
// Escaped "@"
Output(SpanKindInternal.Code);
@ -1397,14 +1401,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Code);
Assert(CSharpTokenType.Transition);
Assert(SyntaxKind.Transition);
AcceptAndMoveNext();
StandardStatement();
}
else
{
// Throw errors as necessary, but continue parsing
if (At(CSharpTokenType.LeftBrace))
if (At(SyntaxKind.LeftBrace))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedNestedCodeBlock(
@ -1428,48 +1432,48 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
var bookmark = CurrentStart.AbsoluteIndex;
var read = ReadWhile(token =>
token.Type != CSharpTokenType.Semicolon &&
token.Type != CSharpTokenType.RazorCommentTransition &&
token.Type != CSharpTokenType.Transition &&
token.Type != CSharpTokenType.LeftBrace &&
token.Type != CSharpTokenType.LeftParenthesis &&
token.Type != CSharpTokenType.LeftBracket &&
token.Type != CSharpTokenType.RightBrace);
token.Kind != SyntaxKind.Semicolon &&
token.Kind != SyntaxKind.RazorCommentTransition &&
token.Kind != SyntaxKind.Transition &&
token.Kind != SyntaxKind.LeftBrace &&
token.Kind != SyntaxKind.LeftParenthesis &&
token.Kind != SyntaxKind.LeftBracket &&
token.Kind != SyntaxKind.RightBrace);
if (At(CSharpTokenType.LeftBrace) ||
At(CSharpTokenType.LeftParenthesis) ||
At(CSharpTokenType.LeftBracket))
if (At(SyntaxKind.LeftBrace) ||
At(SyntaxKind.LeftParenthesis) ||
At(SyntaxKind.LeftBracket))
{
Accept(read);
if (Balance(BalancingModes.AllowCommentsAndTemplates | BalancingModes.BacktrackOnFailure))
{
Optional(CSharpTokenType.RightBrace);
Optional(SyntaxKind.RightBrace);
}
else
{
// Recovery
AcceptUntil(CSharpTokenType.LessThan, CSharpTokenType.RightBrace);
AcceptUntil(SyntaxKind.LessThan, SyntaxKind.RightBrace);
return;
}
}
else if (At(CSharpTokenType.Transition) && (NextIs(CSharpTokenType.LessThan, CSharpTokenType.Colon)))
else if (At(SyntaxKind.Transition) && (NextIs(SyntaxKind.LessThan, SyntaxKind.Colon)))
{
Accept(read);
Output(SpanKindInternal.Code);
Template();
}
else if (At(CSharpTokenType.RazorCommentTransition))
else if (At(SyntaxKind.RazorCommentTransition))
{
Accept(read);
RazorComment();
}
else if (At(CSharpTokenType.Semicolon))
else if (At(SyntaxKind.Semicolon))
{
Accept(read);
AcceptAndMoveNext();
return;
}
else if (At(CSharpTokenType.RightBrace))
else if (At(SyntaxKind.RightBrace))
{
Accept(read);
return;
@ -1478,7 +1482,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.Source.Position = bookmark;
NextToken();
AcceptUntil(CSharpTokenType.LessThan, CSharpTokenType.LeftBrace, CSharpTokenType.RightBrace);
AcceptUntil(SyntaxKind.LessThan, SyntaxKind.LeftBrace, SyntaxKind.RightBrace);
return;
}
}
@ -1492,7 +1496,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CodeBlock(bool acceptTerminatingBrace, Block block)
{
EnsureCurrent();
while (!EndOfFile && !At(CSharpTokenType.RightBrace))
while (!EndOfFile && !At(SyntaxKind.RightBrace))
{
// Parse a statement, then return here
Statement();
@ -1507,7 +1511,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else if (acceptTerminatingBrace)
{
Assert(CSharpTokenType.RightBrace);
Assert(SyntaxKind.RightBrace);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
AcceptAndMoveNext();
}
@ -1515,8 +1519,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void HandleKeyword(bool topLevel, Action fallback)
{
Debug.Assert(CurrentToken.Type == CSharpTokenType.Keyword && CurrentToken.Keyword != null);
if (_keywordParsers.TryGetValue(CurrentToken.Keyword.Value, out var handler))
var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
Debug.Assert(CurrentToken.Kind == SyntaxKind.Keyword && result.HasValue);
if (_keywordParsers.TryGetValue(result.Value, out var handler))
{
handler(topLevel);
}
@ -1526,12 +1531,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
private IEnumerable<CSharpToken> SkipToNextImportantToken()
private IEnumerable<SyntaxToken> SkipToNextImportantToken()
{
while (!EndOfFile)
{
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (At(CSharpTokenType.RazorCommentTransition))
if (At(SyntaxKind.RazorCommentTransition))
{
Accept(whitespace);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
@ -1542,7 +1547,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return whitespace;
}
}
return Enumerable.Empty<CSharpToken>();
return Enumerable.Empty<SyntaxToken>();
}
// Common code for Parsers, but FxCop REALLY doesn't like it in the base class.. moving it here for now.
@ -1644,8 +1649,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
for (var i = 0; i < descriptor.Tokens.Count; i++)
{
if (!At(CSharpTokenType.WhiteSpace) &&
!At(CSharpTokenType.NewLine) &&
if (!At(SyntaxKind.Whitespace) &&
!At(SyntaxKind.NewLine) &&
!EndOfFile)
{
Context.ErrorSink.OnError(
@ -1664,7 +1669,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Code, AcceptedCharactersInternal.WhiteSpace);
if (EndOfFile || At(CSharpTokenType.NewLine))
if (EndOfFile || At(SyntaxKind.NewLine))
{
// Add a marker token to provide CSharp intellisense when we start typing the directive token.
AddMarkerTokenIfNecessary();
@ -1679,7 +1684,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup, AcceptedCharactersInternal.WhiteSpace);
}
if (tokenDescriptor.Optional && (EndOfFile || At(CSharpTokenType.NewLine)))
if (tokenDescriptor.Optional && (EndOfFile || At(SyntaxKind.NewLine)))
{
break;
}
@ -1718,7 +1723,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break;
case DirectiveTokenKind.Member:
if (At(CSharpTokenType.Identifier))
if (At(SyntaxKind.Identifier))
{
AcceptAndMoveNext();
}
@ -1732,7 +1737,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break;
case DirectiveTokenKind.String:
if (At(CSharpTokenType.StringLiteral) && CurrentToken.Errors.Count == 0)
if (At(SyntaxKind.StringLiteral) && !CurrentToken.ContainsDiagnostics)
{
AcceptAndMoveNext();
}
@ -1759,13 +1764,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
case DirectiveKind.SingleLine:
Output(SpanKindInternal.None, AcceptedCharactersInternal.WhiteSpace);
Optional(CSharpTokenType.Semicolon);
Optional(SyntaxKind.Semicolon);
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.WhiteSpace);
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
if (At(CSharpTokenType.NewLine))
if (At(SyntaxKind.NewLine))
{
AcceptAndMoveNext();
}
@ -1816,7 +1821,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
ParseDirectiveBlock(descriptor, parseChildren: (startingBraceLocation) =>
{
NextToken();
Balance(BalancingModes.NoErrorOnFailure, CSharpTokenType.LeftBrace, CSharpTokenType.RightBrace, startingBraceLocation);
Balance(BalancingModes.NoErrorOnFailure, SyntaxKind.LeftBrace, SyntaxKind.RightBrace, startingBraceLocation);
Span.ChunkGenerator = new StatementChunkGenerator();
var existingEditHandler = Span.EditHandler;
Span.EditHandler = new CodeBlockEditHandler(Language.TokenizeString);
@ -1868,7 +1873,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnexpectedEOFAfterDirective(
new SourceSpan(CurrentStart, contentLength: 1 /* { */), descriptor.Directive, "{"));
}
else if (!At(CSharpTokenType.LeftBrace))
else if (!At(SyntaxKind.LeftBrace))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral(
@ -1886,7 +1891,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
parseChildren(startingBraceLocation);
Span.ChunkGenerator = SpanChunkGenerator.Null;
if (!Optional(CSharpTokenType.RightBrace))
if (!Optional(SyntaxKind.RightBrace))
{
editHandler.AutoCompleteString = "}";
Context.ErrorSink.OnError(
@ -1982,7 +1987,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Ex: @addTagHelper "*, Microsoft.AspNetCore.CoolLibrary"
// ^ ^
// Start End
if (Span.Tokens.Count == 1 && (Span.Tokens[0] as CSharpToken)?.Type == CSharpTokenType.StringLiteral)
if (Span.Tokens.Count == 1 && (Span.Tokens[0] as SyntaxToken)?.Kind == SyntaxKind.StringLiteral)
{
offset += Span.Tokens[0].Content.IndexOf(directiveText, StringComparison.Ordinal);
@ -2081,7 +2086,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Conditional("DEBUG")]
protected void AssertDirective(string directive)
{
Debug.Assert(CurrentToken.Type == CSharpTokenType.Identifier || CurrentToken.Type == CSharpTokenType.Keyword);
Debug.Assert(CurrentToken.Kind == SyntaxKind.Identifier || CurrentToken.Kind == SyntaxKind.Keyword);
Debug.Assert(string.Equals(CurrentToken.Content, directive, StringComparison.Ordinal));
}
@ -2108,18 +2113,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
var foundWhitespace = At(CSharpTokenType.WhiteSpace);
var foundWhitespace = At(SyntaxKind.Whitespace);
// If we found whitespace then any content placed within the whitespace MAY cause a destructive change
// to the document. We can't accept it.
var acceptedCharacters = foundWhitespace ? AcceptedCharactersInternal.None : AcceptedCharactersInternal.AnyExceptNewline;
Output(SpanKindInternal.MetaCode, acceptedCharacters);
AcceptWhile(CSharpTokenType.WhiteSpace);
AcceptWhile(SyntaxKind.Whitespace);
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Markup, acceptedCharacters);
if (EndOfFile || At(CSharpTokenType.NewLine))
if (EndOfFile || At(SyntaxKind.NewLine))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveMustHaveValue(
@ -2134,7 +2139,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code
// etc.
AcceptUntil(CSharpTokenType.NewLine);
AcceptUntil(SyntaxKind.NewLine);
// Pull out the value and remove whitespaces and optional quotes
var rawValue = string.Concat(Span.Tokens.Select(s => s.Content)).Trim();
@ -2170,7 +2175,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Start = start;
}
public Block(CSharpToken token, SourceLocation start)
public Block(SyntaxToken token, SourceLocation start)
: this(GetName(token), start)
{
}
@ -2178,11 +2183,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public string Name { get; set; }
public SourceLocation Start { get; set; }
private static string GetName(CSharpToken token)
private static string GetName(SyntaxToken token)
{
if (token.Type == CSharpTokenType.Keyword)
var result = CSharpTokenizer.GetTokenKeyword(token);
if (result.HasValue && token.Kind == SyntaxKind.Keyword)
{
return CSharpLanguageCharacteristics.GetKeyword(token.Keyword.Value);
return CSharpLanguageCharacteristics.GetKeyword(result.Value);
}
return token.Content;
}

191
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs
View File

@ -3,64 +3,65 @@
using System.Collections.Generic;
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer, CSharpToken, CSharpTokenType>
internal class CSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer>
{
private static readonly CSharpLanguageCharacteristics _instance = new CSharpLanguageCharacteristics();
private static Dictionary<CSharpTokenType, string> _tokenSamples = new Dictionary<CSharpTokenType, string>()
private static Dictionary<SyntaxKind, string> _tokenSamples = new Dictionary<SyntaxKind, string>()
{
{ CSharpTokenType.Arrow, "->" },
{ CSharpTokenType.Minus, "-" },
{ CSharpTokenType.Decrement, "--" },
{ CSharpTokenType.MinusAssign, "-=" },
{ CSharpTokenType.NotEqual, "!=" },
{ CSharpTokenType.Not, "!" },
{ CSharpTokenType.Modulo, "%" },
{ CSharpTokenType.ModuloAssign, "%=" },
{ CSharpTokenType.AndAssign, "&=" },
{ CSharpTokenType.And, "&" },
{ CSharpTokenType.DoubleAnd, "&&" },
{ CSharpTokenType.LeftParenthesis, "(" },
{ CSharpTokenType.RightParenthesis, ")" },
{ CSharpTokenType.Star, "*" },
{ CSharpTokenType.MultiplyAssign, "*=" },
{ CSharpTokenType.Comma, "," },
{ CSharpTokenType.Dot, "." },
{ CSharpTokenType.Slash, "/" },
{ CSharpTokenType.DivideAssign, "/=" },
{ CSharpTokenType.DoubleColon, "::" },
{ CSharpTokenType.Colon, ":" },
{ CSharpTokenType.Semicolon, ";" },
{ CSharpTokenType.QuestionMark, "?" },
{ CSharpTokenType.NullCoalesce, "??" },
{ CSharpTokenType.RightBracket, "]" },
{ CSharpTokenType.LeftBracket, "[" },
{ CSharpTokenType.XorAssign, "^=" },
{ CSharpTokenType.Xor, "^" },
{ CSharpTokenType.LeftBrace, "{" },
{ CSharpTokenType.OrAssign, "|=" },
{ CSharpTokenType.DoubleOr, "||" },
{ CSharpTokenType.Or, "|" },
{ CSharpTokenType.RightBrace, "}" },
{ CSharpTokenType.Tilde, "~" },
{ CSharpTokenType.Plus, "+" },
{ CSharpTokenType.PlusAssign, "+=" },
{ CSharpTokenType.Increment, "++" },
{ CSharpTokenType.LessThan, "<" },
{ CSharpTokenType.LessThanEqual, "<=" },
{ CSharpTokenType.LeftShift, "<<" },
{ CSharpTokenType.LeftShiftAssign, "<<=" },
{ CSharpTokenType.Assign, "=" },
{ CSharpTokenType.Equals, "==" },
{ CSharpTokenType.GreaterThan, ">" },
{ CSharpTokenType.GreaterThanEqual, ">=" },
{ CSharpTokenType.RightShift, ">>" },
{ CSharpTokenType.RightShiftAssign, ">>=" },
{ CSharpTokenType.Hash, "#" },
{ CSharpTokenType.Transition, "@" },
{ SyntaxKind.Arrow, "->" },
{ SyntaxKind.Minus, "-" },
{ SyntaxKind.Decrement, "--" },
{ SyntaxKind.MinusAssign, "-=" },
{ SyntaxKind.NotEqual, "!=" },
{ SyntaxKind.Not, "!" },
{ SyntaxKind.Modulo, "%" },
{ SyntaxKind.ModuloAssign, "%=" },
{ SyntaxKind.AndAssign, "&=" },
{ SyntaxKind.And, "&" },
{ SyntaxKind.DoubleAnd, "&&" },
{ SyntaxKind.LeftParenthesis, "(" },
{ SyntaxKind.RightParenthesis, ")" },
{ SyntaxKind.Star, "*" },
{ SyntaxKind.MultiplyAssign, "*=" },
{ SyntaxKind.Comma, "," },
{ SyntaxKind.Dot, "." },
{ SyntaxKind.Slash, "/" },
{ SyntaxKind.DivideAssign, "/=" },
{ SyntaxKind.DoubleColon, "::" },
{ SyntaxKind.Colon, ":" },
{ SyntaxKind.Semicolon, ";" },
{ SyntaxKind.QuestionMark, "?" },
{ SyntaxKind.NullCoalesce, "??" },
{ SyntaxKind.RightBracket, "]" },
{ SyntaxKind.LeftBracket, "[" },
{ SyntaxKind.XorAssign, "^=" },
{ SyntaxKind.Xor, "^" },
{ SyntaxKind.LeftBrace, "{" },
{ SyntaxKind.OrAssign, "|=" },
{ SyntaxKind.DoubleOr, "||" },
{ SyntaxKind.Or, "|" },
{ SyntaxKind.RightBrace, "}" },
{ SyntaxKind.Tilde, "~" },
{ SyntaxKind.Plus, "+" },
{ SyntaxKind.PlusAssign, "+=" },
{ SyntaxKind.Increment, "++" },
{ SyntaxKind.LessThan, "<" },
{ SyntaxKind.LessThanEqual, "<=" },
{ SyntaxKind.LeftShift, "<<" },
{ SyntaxKind.LeftShiftAssign, "<<=" },
{ SyntaxKind.Assign, "=" },
{ SyntaxKind.Equals, "==" },
{ SyntaxKind.GreaterThan, ">" },
{ SyntaxKind.GreaterThanEqual, ">=" },
{ SyntaxKind.RightShift, ">>" },
{ SyntaxKind.RightShiftAssign, ">>=" },
{ SyntaxKind.Hash, "#" },
{ SyntaxKind.Transition, "@" },
};
protected CSharpLanguageCharacteristics()
@ -74,35 +75,35 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source);
}
protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList<RazorDiagnostic> errors)
protected override SyntaxToken CreateToken(string content, SyntaxKind kind, IReadOnlyList<RazorDiagnostic> errors)
{
return new CSharpToken(content, type, errors);
return SyntaxFactory.Token(kind, content, errors);
}
public override string GetSample(CSharpTokenType type)
public override string GetSample(SyntaxKind kind)
{
string sample;
if (!_tokenSamples.TryGetValue(type, out sample))
if (!_tokenSamples.TryGetValue(kind, out sample))
{
switch (type)
switch (kind)
{
case CSharpTokenType.Identifier:
case SyntaxKind.Identifier:
return Resources.CSharpToken_Identifier;
case CSharpTokenType.Keyword:
case SyntaxKind.Keyword:
return Resources.CSharpToken_Keyword;
case CSharpTokenType.IntegerLiteral:
case SyntaxKind.IntegerLiteral:
return Resources.CSharpToken_IntegerLiteral;
case CSharpTokenType.NewLine:
case SyntaxKind.NewLine:
return Resources.CSharpToken_Newline;
case CSharpTokenType.WhiteSpace:
case SyntaxKind.Whitespace:
return Resources.CSharpToken_Whitespace;
case CSharpTokenType.Comment:
case SyntaxKind.CSharpComment:
return Resources.CSharpToken_Comment;
case CSharpTokenType.RealLiteral:
case SyntaxKind.RealLiteral:
return Resources.CSharpToken_RealLiteral;
case CSharpTokenType.CharacterLiteral:
case SyntaxKind.CharacterLiteral:
return Resources.CSharpToken_CharacterLiteral;
case CSharpTokenType.StringLiteral:
case SyntaxKind.StringLiteral:
return Resources.CSharpToken_StringLiteral;
default:
return Resources.Token_Unknown;
@ -111,59 +112,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return sample;
}
public override CSharpToken CreateMarkerToken()
public override SyntaxToken CreateMarkerToken()
{
return new CSharpToken(string.Empty, CSharpTokenType.Unknown);
return SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
}
public override CSharpTokenType GetKnownTokenType(KnownTokenType type)
public override SyntaxKind GetKnownTokenType(KnownTokenType type)
{
switch (type)
{
case KnownTokenType.Identifier:
return CSharpTokenType.Identifier;
return SyntaxKind.Identifier;
case KnownTokenType.Keyword:
return CSharpTokenType.Keyword;
return SyntaxKind.Keyword;
case KnownTokenType.NewLine:
return CSharpTokenType.NewLine;
return SyntaxKind.NewLine;
case KnownTokenType.WhiteSpace:
return CSharpTokenType.WhiteSpace;
return SyntaxKind.Whitespace;
case KnownTokenType.Transition:
return CSharpTokenType.Transition;
return SyntaxKind.Transition;
case KnownTokenType.CommentStart:
return CSharpTokenType.RazorCommentTransition;
return SyntaxKind.RazorCommentTransition;
case KnownTokenType.CommentStar:
return CSharpTokenType.RazorCommentStar;
return SyntaxKind.RazorCommentStar;
case KnownTokenType.CommentBody:
return CSharpTokenType.RazorComment;
return SyntaxKind.RazorComment;
default:
return CSharpTokenType.Unknown;
return SyntaxKind.Unknown;
}
}
public override CSharpTokenType FlipBracket(CSharpTokenType bracket)
public override SyntaxKind FlipBracket(SyntaxKind bracket)
{
switch (bracket)
{
case CSharpTokenType.LeftBrace:
return CSharpTokenType.RightBrace;
case CSharpTokenType.LeftBracket:
return CSharpTokenType.RightBracket;
case CSharpTokenType.LeftParenthesis:
return CSharpTokenType.RightParenthesis;
case CSharpTokenType.LessThan:
return CSharpTokenType.GreaterThan;
case CSharpTokenType.RightBrace:
return CSharpTokenType.LeftBrace;
case CSharpTokenType.RightBracket:
return CSharpTokenType.LeftBracket;
case CSharpTokenType.RightParenthesis:
return CSharpTokenType.LeftParenthesis;
case CSharpTokenType.GreaterThan:
return CSharpTokenType.LessThan;
case SyntaxKind.LeftBrace:
return SyntaxKind.RightBrace;
case SyntaxKind.LeftBracket:
return SyntaxKind.RightBracket;
case SyntaxKind.LeftParenthesis:
return SyntaxKind.RightParenthesis;
case SyntaxKind.LessThan:
return SyntaxKind.GreaterThan;
case SyntaxKind.RightBrace:
return SyntaxKind.LeftBrace;
case SyntaxKind.RightBracket:
return SyntaxKind.LeftBracket;
case SyntaxKind.RightParenthesis:
return SyntaxKind.LeftParenthesis;
case SyntaxKind.GreaterThan:
return SyntaxKind.LessThan;
default:
Debug.Fail("FlipBracket must be called with a bracket character");
return CSharpTokenType.Unknown;
return SyntaxKind.Unknown;
}
}

60
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpToken.cs
View File

@ -1,60 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CSharpToken : TokenBase<CSharpTokenType>
{
public CSharpToken(
string content,
CSharpTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpToken(
string content,
CSharpTokenType type,
IReadOnlyList<RazorDiagnostic> errors)
: base(content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpKeyword? Keyword { get; set; }
protected override SyntaxToken GetSyntaxToken()
{
switch (Type)
{
default:
return SyntaxFactory.UnknownToken(Content, Errors.ToArray());
}
}
public override bool Equals(object obj)
{
var other = obj as CSharpToken;
return base.Equals(other) &&
other.Keyword == Keyword;
}
public override int GetHashCode()
{
// Hash code should include only immutable properties.
return base.GetHashCode();
}
}
}

75
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenType.cs
View File

@ -1,75 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal enum CSharpTokenType
{
Unknown,
Identifier,
Keyword,
IntegerLiteral,
NewLine,
WhiteSpace,
Comment,
RealLiteral,
CharacterLiteral,
StringLiteral,
// Operators
Arrow,
Minus,
Decrement,
MinusAssign,
NotEqual,
Not,
Modulo,
ModuloAssign,
AndAssign,
And,
DoubleAnd,
LeftParenthesis,
RightParenthesis,
Star,
MultiplyAssign,
Comma,
Dot,
Slash,
DivideAssign,
DoubleColon,
Colon,
Semicolon,
QuestionMark,
NullCoalesce,
RightBracket,
LeftBracket,
XorAssign,
Xor,
LeftBrace,
OrAssign,
DoubleOr,
Or,
RightBrace,
Tilde,
Plus,
PlusAssign,
Increment,
LessThan,
LessThanEqual,
LeftShift,
LeftShiftAssign,
Assign,
Equals,
GreaterThan,
GreaterThanEqual,
RightShift,
RightShiftAssign,
Hash,
Transition,
// Razor specific
RazorCommentTransition,
RazorCommentStar,
RazorComment
}
}

260
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs
View File

@ -5,12 +5,13 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CSharpTokenizer : Tokenizer<CSharpToken, CSharpTokenType>
internal class CSharpTokenizer : Tokenizer
{
private Dictionary<char, Func<CSharpTokenType>> _operatorHandlers;
private Dictionary<char, Func<SyntaxKind>> _operatorHandlers;
private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal)
{
@ -100,31 +101,31 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
base.CurrentState = StartState;
_operatorHandlers = new Dictionary<char, Func<CSharpTokenType>>()
_operatorHandlers = new Dictionary<char, Func<SyntaxKind>>()
{
{ '-', MinusOperator },
{ '<', LessThanOperator },
{ '>', GreaterThanOperator },
{ '&', CreateTwoCharOperatorHandler(CSharpTokenType.And, '=', CSharpTokenType.AndAssign, '&', CSharpTokenType.DoubleAnd) },
{ '|', CreateTwoCharOperatorHandler(CSharpTokenType.Or, '=', CSharpTokenType.OrAssign, '|', CSharpTokenType.DoubleOr) },
{ '+', CreateTwoCharOperatorHandler(CSharpTokenType.Plus, '=', CSharpTokenType.PlusAssign, '+', CSharpTokenType.Increment) },
{ '=', CreateTwoCharOperatorHandler(CSharpTokenType.Assign, '=', CSharpTokenType.Equals, '>', CSharpTokenType.GreaterThanEqual) },
{ '!', CreateTwoCharOperatorHandler(CSharpTokenType.Not, '=', CSharpTokenType.NotEqual) },
{ '%', CreateTwoCharOperatorHandler(CSharpTokenType.Modulo, '=', CSharpTokenType.ModuloAssign) },
{ '*', CreateTwoCharOperatorHandler(CSharpTokenType.Star, '=', CSharpTokenType.MultiplyAssign) },
{ ':', CreateTwoCharOperatorHandler(CSharpTokenType.Colon, ':', CSharpTokenType.DoubleColon) },
{ '?', CreateTwoCharOperatorHandler(CSharpTokenType.QuestionMark, '?', CSharpTokenType.NullCoalesce) },
{ '^', CreateTwoCharOperatorHandler(CSharpTokenType.Xor, '=', CSharpTokenType.XorAssign) },
{ '(', () => CSharpTokenType.LeftParenthesis },
{ ')', () => CSharpTokenType.RightParenthesis },
{ '{', () => CSharpTokenType.LeftBrace },
{ '}', () => CSharpTokenType.RightBrace },
{ '[', () => CSharpTokenType.LeftBracket },
{ ']', () => CSharpTokenType.RightBracket },
{ ',', () => CSharpTokenType.Comma },
{ ';', () => CSharpTokenType.Semicolon },
{ '~', () => CSharpTokenType.Tilde },
{ '#', () => CSharpTokenType.Hash }
{ '&', CreateTwoCharOperatorHandler(SyntaxKind.And, '=', SyntaxKind.AndAssign, '&', SyntaxKind.DoubleAnd) },
{ '|', CreateTwoCharOperatorHandler(SyntaxKind.Or, '=', SyntaxKind.OrAssign, '|', SyntaxKind.DoubleOr) },
{ '+', CreateTwoCharOperatorHandler(SyntaxKind.Plus, '=', SyntaxKind.PlusAssign, '+', SyntaxKind.Increment) },
{ '=', CreateTwoCharOperatorHandler(SyntaxKind.Assign, '=', SyntaxKind.Equals, '>', SyntaxKind.GreaterThanEqual) },
{ '!', CreateTwoCharOperatorHandler(SyntaxKind.Not, '=', SyntaxKind.NotEqual) },
{ '%', CreateTwoCharOperatorHandler(SyntaxKind.Modulo, '=', SyntaxKind.ModuloAssign) },
{ '*', CreateTwoCharOperatorHandler(SyntaxKind.Star, '=', SyntaxKind.MultiplyAssign) },
{ ':', CreateTwoCharOperatorHandler(SyntaxKind.Colon, ':', SyntaxKind.DoubleColon) },
{ '?', CreateTwoCharOperatorHandler(SyntaxKind.QuestionMark, '?', SyntaxKind.NullCoalesce) },
{ '^', CreateTwoCharOperatorHandler(SyntaxKind.Xor, '=', SyntaxKind.XorAssign) },
{ '(', () => SyntaxKind.LeftParenthesis },
{ ')', () => SyntaxKind.RightParenthesis },
{ '{', () => SyntaxKind.LeftBrace },
{ '}', () => SyntaxKind.RightBrace },
{ '[', () => SyntaxKind.LeftBracket },
{ ']', () => SyntaxKind.RightBracket },
{ ',', () => SyntaxKind.Comma },
{ ';', () => SyntaxKind.Semicolon },
{ '~', () => SyntaxKind.Tilde },
{ '#', () => SyntaxKind.Hash }
};
}
@ -132,11 +133,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState;
public override CSharpTokenType RazorCommentType => CSharpTokenType.RazorComment;
public override SyntaxKind RazorCommentKind => SyntaxKind.RazorComment;
public override CSharpTokenType RazorCommentTransitionType => CSharpTokenType.RazorCommentTransition;
public override SyntaxKind RazorCommentTransitionKind => SyntaxKind.RazorCommentTransition;
public override CSharpTokenType RazorCommentStarType => CSharpTokenType.RazorCommentStar;
public override SyntaxKind RazorCommentStarKind => SyntaxKind.RazorCommentStar;
protected override StateResult Dispatch()
{
@ -169,7 +170,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Optimize memory allocation by returning constants for the most frequent cases
protected override string GetTokenContent(CSharpTokenType type)
protected override string GetTokenContent(SyntaxKind type)
{
var tokenLength = Buffer.Length;
@ -177,7 +178,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
switch (type)
{
case CSharpTokenType.IntegerLiteral:
case SyntaxKind.IntegerLiteral:
switch (Buffer[0])
{
case '0':
@ -202,13 +203,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "9";
}
break;
case CSharpTokenType.NewLine:
case SyntaxKind.NewLine:
if (Buffer[0] == '\n')
{
return "\n";
}
break;
case CSharpTokenType.WhiteSpace:
case SyntaxKind.Whitespace:
if (Buffer[0] == ' ')
{
return " ";
@ -218,57 +219,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t";
}
break;
case CSharpTokenType.Minus:
case SyntaxKind.Minus:
return "-";
case CSharpTokenType.Not:
case SyntaxKind.Not:
return "!";
case CSharpTokenType.Modulo:
case SyntaxKind.Modulo:
return "%";
case CSharpTokenType.And:
case SyntaxKind.And:
return "&";
case CSharpTokenType.LeftParenthesis:
case SyntaxKind.LeftParenthesis:
return "(";
case CSharpTokenType.RightParenthesis:
case SyntaxKind.RightParenthesis:
return ")";
case CSharpTokenType.Star:
case SyntaxKind.Star:
return "*";
case CSharpTokenType.Comma:
case SyntaxKind.Comma:
return ",";
case CSharpTokenType.Dot:
case SyntaxKind.Dot:
return ".";
case CSharpTokenType.Slash:
case SyntaxKind.Slash:
return "/";
case CSharpTokenType.Colon:
case SyntaxKind.Colon:
return ":";
case CSharpTokenType.Semicolon:
case SyntaxKind.Semicolon:
return ";";
case CSharpTokenType.QuestionMark:
case SyntaxKind.QuestionMark:
return "?";
case CSharpTokenType.RightBracket:
case SyntaxKind.RightBracket:
return "]";
case CSharpTokenType.LeftBracket:
case SyntaxKind.LeftBracket:
return "[";
case CSharpTokenType.Xor:
case SyntaxKind.Xor:
return "^";
case CSharpTokenType.LeftBrace:
case SyntaxKind.LeftBrace:
return "{";
case CSharpTokenType.Or:
case SyntaxKind.Or:
return "|";
case CSharpTokenType.RightBrace:
case SyntaxKind.RightBrace:
return "}";
case CSharpTokenType.Tilde:
case SyntaxKind.Tilde:
return "~";
case CSharpTokenType.Plus:
case SyntaxKind.Plus:
return "+";
case CSharpTokenType.LessThan:
case SyntaxKind.LessThan:
return "<";
case CSharpTokenType.Assign:
case SyntaxKind.Assign:
return "=";
case CSharpTokenType.GreaterThan:
case SyntaxKind.GreaterThan:
return ">";
case CSharpTokenType.Hash:
case SyntaxKind.Hash:
return "#";
case CSharpTokenType.Transition:
case SyntaxKind.Transition:
return "@";
}
@ -277,53 +278,53 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
switch (type)
{
case CSharpTokenType.NewLine:
case SyntaxKind.NewLine:
return "\r\n";
case CSharpTokenType.Arrow:
case SyntaxKind.Arrow:
return "->";
case CSharpTokenType.Decrement:
case SyntaxKind.Decrement:
return "--";
case CSharpTokenType.MinusAssign:
case SyntaxKind.MinusAssign:
return "-=";
case CSharpTokenType.NotEqual:
case SyntaxKind.NotEqual:
return "!=";
case CSharpTokenType.ModuloAssign:
case SyntaxKind.ModuloAssign:
return "%=";
case CSharpTokenType.AndAssign:
case SyntaxKind.AndAssign:
return "&=";
case CSharpTokenType.DoubleAnd:
case SyntaxKind.DoubleAnd:
return "&&";
case CSharpTokenType.MultiplyAssign:
case SyntaxKind.MultiplyAssign:
return "*=";
case CSharpTokenType.DivideAssign:
case SyntaxKind.DivideAssign:
return "/=";
case CSharpTokenType.DoubleColon:
case SyntaxKind.DoubleColon:
return "::";
case CSharpTokenType.NullCoalesce:
case SyntaxKind.NullCoalesce:
return "??";
case CSharpTokenType.XorAssign:
case SyntaxKind.XorAssign:
return "^=";
case CSharpTokenType.OrAssign:
case SyntaxKind.OrAssign:
return "|=";
case CSharpTokenType.DoubleOr:
case SyntaxKind.DoubleOr:
return "||";
case CSharpTokenType.PlusAssign:
case SyntaxKind.PlusAssign:
return "+=";
case CSharpTokenType.Increment:
case SyntaxKind.Increment:
return "++";
case CSharpTokenType.LessThanEqual:
case SyntaxKind.LessThanEqual:
return "<=";
case CSharpTokenType.LeftShift:
case SyntaxKind.LeftShift:
return "<<";
case CSharpTokenType.Equals:
case SyntaxKind.Equals:
return "==";
case CSharpTokenType.GreaterThanEqual:
case SyntaxKind.GreaterThanEqual:
if (Buffer[0] == '=')
{
return "=>";
}
return ">=";
case CSharpTokenType.RightShift:
case SyntaxKind.RightShift:
return ">>";
@ -333,9 +334,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
switch (type)
{
case CSharpTokenType.LeftShiftAssign:
case SyntaxKind.LeftShiftAssign:
return "<<=";
case CSharpTokenType.RightShiftAssign:
case SyntaxKind.RightShiftAssign:
return ">>=";
}
}
@ -343,9 +344,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return base.GetTokenContent(type);
}
protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList<RazorDiagnostic> errors)
protected override SyntaxToken CreateToken(string content, SyntaxKind kind, IReadOnlyList<RazorDiagnostic> errors)
{
return new CSharpToken(content, type, errors);
return SyntaxFactory.Token(kind, content, errors);
}
private StateResult Data()
@ -359,13 +360,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent();
}
return Stay(EndToken(CSharpTokenType.NewLine));
return Stay(EndToken(SyntaxKind.NewLine));
}
else if (ParserHelpers.IsWhitespace(CurrentCharacter))
{
// CSharp Spec §2.3.3
TakeUntil(c => !ParserHelpers.IsWhitespace(c));
return Stay(EndToken(CSharpTokenType.WhiteSpace));
return Stay(EndToken(SyntaxKind.Whitespace));
}
else if (IsIdentifierStart(CurrentCharacter))
{
@ -390,7 +391,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return RealLiteral();
}
return Stay(Single(CSharpTokenType.Dot));
return Stay(Single(SyntaxKind.Dot));
case '/':
TakeCurrent();
if (CurrentCharacter == '/')
@ -406,11 +407,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else if (CurrentCharacter == '=')
{
TakeCurrent();
return Stay(EndToken(CSharpTokenType.DivideAssign));
return Stay(EndToken(SyntaxKind.DivideAssign));
}
else
{
return Stay(EndToken(CSharpTokenType.Slash));
return Stay(EndToken(SyntaxKind.Slash));
}
default:
return Stay(EndToken(Operator()));
@ -429,78 +430,78 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
CSharpTokenizerState.AfterRazorCommentTransition,
EndToken(CSharpTokenType.RazorCommentTransition));
EndToken(SyntaxKind.RazorCommentTransition));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return Transition(
CSharpTokenizerState.EscapedRazorCommentTransition,
EndToken(CSharpTokenType.Transition));
EndToken(SyntaxKind.Transition));
}
return Stay(EndToken(CSharpTokenType.Transition));
return Stay(EndToken(SyntaxKind.Transition));
}
private StateResult EscapedRazorCommentTransition()
{
TakeCurrent();
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Transition));
return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.Transition));
}
private CSharpTokenType Operator()
private SyntaxKind Operator()
{
var first = CurrentCharacter;
TakeCurrent();
Func<CSharpTokenType> handler;
Func<SyntaxKind> handler;
if (_operatorHandlers.TryGetValue(first, out handler))
{
return handler();
}
return CSharpTokenType.Unknown;
return SyntaxKind.Unknown;
}
private CSharpTokenType LessThanOperator()
private SyntaxKind LessThanOperator()
{
if (CurrentCharacter == '=')
{
TakeCurrent();
return CSharpTokenType.LessThanEqual;
return SyntaxKind.LessThanEqual;
}
return CSharpTokenType.LessThan;
return SyntaxKind.LessThan;
}
private CSharpTokenType GreaterThanOperator()
private SyntaxKind GreaterThanOperator()
{
if (CurrentCharacter == '=')
{
TakeCurrent();
return CSharpTokenType.GreaterThanEqual;
return SyntaxKind.GreaterThanEqual;
}
return CSharpTokenType.GreaterThan;
return SyntaxKind.GreaterThan;
}
private CSharpTokenType MinusOperator()
private SyntaxKind MinusOperator()
{
if (CurrentCharacter == '>')
{
TakeCurrent();
return CSharpTokenType.Arrow;
return SyntaxKind.Arrow;
}
else if (CurrentCharacter == '-')
{
TakeCurrent();
return CSharpTokenType.Decrement;
return SyntaxKind.Decrement;
}
else if (CurrentCharacter == '=')
{
TakeCurrent();
return CSharpTokenType.MinusAssign;
return SyntaxKind.MinusAssign;
}
return CSharpTokenType.Minus;
return SyntaxKind.Minus;
}
private Func<CSharpTokenType> CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char second, CSharpTokenType typeIfBoth)
private Func<SyntaxKind> CreateTwoCharOperatorHandler(SyntaxKind typeIfOnlyFirst, char second, SyntaxKind typeIfBoth)
{
return () =>
{
@ -513,7 +514,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
};
}
private Func<CSharpTokenType> CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char option1, CSharpTokenType typeIfOption1, char option2, CSharpTokenType typeIfOption2)
private Func<SyntaxKind> CreateTwoCharOperatorHandler(SyntaxKind typeIfOnlyFirst, char option1, SyntaxKind typeIfOption1, char option2, SyntaxKind typeIfOption2)
{
return () =>
{
@ -550,14 +551,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
}
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.StringLiteral));
return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.StringLiteral));
}
private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', CSharpTokenType.CharacterLiteral);
private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', SyntaxKind.CharacterLiteral);
private StateResult QuotedStringLiteral() => QuotedLiteral('\"', CSharpTokenType.StringLiteral);
private StateResult QuotedStringLiteral() => QuotedLiteral('\"', SyntaxKind.StringLiteral);
private StateResult QuotedLiteral(char quote, CSharpTokenType literalType)
private StateResult QuotedLiteral(char quote, SyntaxKind literalType)
{
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\')
@ -594,7 +595,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment));
return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment));
}
if (CurrentCharacter == '*')
{
@ -602,7 +603,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (CurrentCharacter == '/')
{
TakeCurrent();
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment));
return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment));
}
}
return Stay();
@ -612,7 +613,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult SingleLineComment()
{
TakeUntil(c => ParserHelpers.IsNewLine(c));
return Stay(EndToken(CSharpTokenType.Comment));
return Stay(EndToken(SyntaxKind.CSharpComment));
}
// CSharp Spec §2.4.4
@ -632,7 +633,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeUntil(c => !IsHexDigit(c));
TakeIntegerSuffix();
return Stay(EndToken(CSharpTokenType.IntegerLiteral));
return Stay(EndToken(SyntaxKind.IntegerLiteral));
}
private StateResult DecimalLiteral()
@ -650,7 +651,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
TakeIntegerSuffix();
return Stay(EndToken(CSharpTokenType.IntegerLiteral));
return Stay(EndToken(SyntaxKind.IntegerLiteral));
}
}
@ -669,7 +670,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent();
}
return Stay(EndToken(CSharpTokenType.RealLiteral));
return Stay(EndToken(SyntaxKind.RealLiteral));
}
// CSharp Spec §2.4.4.3
@ -708,21 +709,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Debug.Assert(IsIdentifierStart(CurrentCharacter));
TakeCurrent();
TakeUntil(c => !IsIdentifierPart(c));
CSharpToken token = null;
SyntaxToken token = null;
if (HaveContent)
{
CSharpKeyword keyword;
var type = CSharpTokenType.Identifier;
var type = SyntaxKind.Identifier;
var tokenContent = Buffer.ToString();
if (_keywords.TryGetValue(tokenContent, out keyword))
{
type = CSharpTokenType.Keyword;
type = SyntaxKind.Keyword;
}
token = new CSharpToken(tokenContent, type)
{
Keyword = type == CSharpTokenType.Keyword ? (CSharpKeyword?)keyword : null,
};
token = SyntaxFactory.Token(type, tokenContent);
Buffer.Clear();
CurrentErrors.Clear();
@ -736,7 +734,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null);
}
private StateResult Transition(CSharpTokenizerState state, CSharpToken result)
private StateResult Transition(CSharpTokenizerState state, SyntaxToken result)
{
return Transition((int)state, result);
}
@ -780,6 +778,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return (value >= '0' && value <= '9') || (value >= 'A' && value <= 'F') || (value >= 'a' && value <= 'f');
}
internal static CSharpKeyword? GetTokenKeyword(SyntaxToken token)
{
if (token != null && _keywords.TryGetValue(token.Content, out var keyword))
{
return keyword;
}
return null;
}
private enum CSharpTokenizerState
{
Data,

3
src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs
View File

@ -4,12 +4,13 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CodeBlockEditHandler : SpanEditHandler
{
public CodeBlockEditHandler(Func<string, IEnumerable<IToken>> tokenizer) : base(tokenizer)
public CodeBlockEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer) : base(tokenizer)
{
}

23
src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
@ -17,11 +18,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected override StateResult Dispatch()
{
var result = base.Dispatch();
if (result.Result != null && !_visitedFirstTokenStart && IsValidTokenType(result.Result.Type))
if (result.Result != null && !_visitedFirstTokenStart && IsValidTokenType(result.Result.Kind))
{
_visitedFirstTokenStart = true;
}
else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpTokenType.NewLine)
else if (result.Result != null && _visitedFirstTokenStart && result.Result.Kind == SyntaxKind.NewLine)
{
_visitedFirstTokenLineEnd = true;
}
@ -29,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result;
}
public override CSharpToken NextToken()
public override SyntaxToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
@ -48,15 +49,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token;
}
private bool IsValidTokenType(CSharpTokenType type)
private bool IsValidTokenType(SyntaxKind kind)
{
return type != CSharpTokenType.WhiteSpace &&
type != CSharpTokenType.NewLine &&
type != CSharpTokenType.Comment &&
type != CSharpTokenType.RazorComment &&
type != CSharpTokenType.RazorCommentStar &&
type != CSharpTokenType.RazorCommentTransition &&
type != CSharpTokenType.Transition;
return kind != SyntaxKind.Whitespace &&
kind != SyntaxKind.NewLine &&
kind != SyntaxKind.CSharpComment &&
kind != SyntaxKind.RazorComment &&
kind != SyntaxKind.RazorCommentStar &&
kind != SyntaxKind.RazorCommentTransition &&
kind != SyntaxKind.Transition;
}
}
}

19
src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
@ -16,7 +17,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected override StateResult Dispatch()
{
var result = base.Dispatch();
if (result.Result != null && IsValidTokenType(result.Result.Type))
if (result.Result != null && IsValidTokenType(result.Result.Kind))
{
_visitedFirstTokenStart = true;
}
@ -24,7 +25,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result;
}
public override HtmlToken NextToken()
public override SyntaxToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
@ -43,14 +44,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token;
}
private bool IsValidTokenType(HtmlTokenType type)
private bool IsValidTokenType(SyntaxKind kind)
{
return type != HtmlTokenType.WhiteSpace &&
type != HtmlTokenType.NewLine &&
type != HtmlTokenType.RazorComment &&
type != HtmlTokenType.RazorCommentStar &&
type != HtmlTokenType.RazorCommentTransition &&
type != HtmlTokenType.Transition;
return kind != SyntaxKind.Whitespace &&
kind != SyntaxKind.NewLine &&
kind != SyntaxKind.RazorComment &&
kind != SyntaxKind.RazorCommentStar &&
kind != SyntaxKind.RazorCommentTransition &&
kind != SyntaxKind.Transition;
}
}
}

91
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
View File

@ -3,10 +3,11 @@
using System.Collections.Generic;
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class HtmlLanguageCharacteristics : LanguageCharacteristics<HtmlTokenizer, HtmlToken, HtmlTokenType>
internal class HtmlLanguageCharacteristics : LanguageCharacteristics<HtmlTokenizer>
{
private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics();
@ -19,47 +20,47 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _instance; }
}
public override string GetSample(HtmlTokenType type)
public override string GetSample(SyntaxKind type)
{
switch (type)
{
case HtmlTokenType.Text:
case SyntaxKind.HtmlTextLiteral:
return Resources.HtmlToken_Text;
case HtmlTokenType.WhiteSpace:
case SyntaxKind.Whitespace:
return Resources.HtmlToken_WhiteSpace;
case HtmlTokenType.NewLine:
case SyntaxKind.NewLine:
return Resources.HtmlToken_NewLine;
case HtmlTokenType.OpenAngle:
case SyntaxKind.OpenAngle:
return "<";
case HtmlTokenType.Bang:
case SyntaxKind.Bang:
return "!";
case HtmlTokenType.ForwardSlash:
case SyntaxKind.ForwardSlash:
return "/";
case HtmlTokenType.QuestionMark:
case SyntaxKind.QuestionMark:
return "?";
case HtmlTokenType.DoubleHyphen:
case SyntaxKind.DoubleHyphen:
return "--";
case HtmlTokenType.LeftBracket:
case SyntaxKind.LeftBracket:
return "[";
case HtmlTokenType.CloseAngle:
case SyntaxKind.CloseAngle:
return ">";
case HtmlTokenType.RightBracket:
case SyntaxKind.RightBracket:
return "]";
case HtmlTokenType.Equals:
case SyntaxKind.Equals:
return "=";
case HtmlTokenType.DoubleQuote:
case SyntaxKind.DoubleQuote:
return "\"";
case HtmlTokenType.SingleQuote:
case SyntaxKind.SingleQuote:
return "'";
case HtmlTokenType.Transition:
case SyntaxKind.Transition:
return "@";
case HtmlTokenType.Colon:
case SyntaxKind.Colon:
return ":";
case HtmlTokenType.RazorComment:
case SyntaxKind.RazorComment:
return Resources.HtmlToken_RazorComment;
case HtmlTokenType.RazorCommentStar:
case SyntaxKind.RazorCommentStar:
return "*";
case HtmlTokenType.RazorCommentTransition:
case SyntaxKind.RazorCommentTransition:
return "@";
default:
return Resources.Token_Unknown;
@ -71,57 +72,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source);
}
public override HtmlTokenType FlipBracket(HtmlTokenType bracket)
public override SyntaxKind FlipBracket(SyntaxKind bracket)
{
switch (bracket)
{
case HtmlTokenType.LeftBracket:
return HtmlTokenType.RightBracket;
case HtmlTokenType.OpenAngle:
return HtmlTokenType.CloseAngle;
case HtmlTokenType.RightBracket:
return HtmlTokenType.LeftBracket;
case HtmlTokenType.CloseAngle:
return HtmlTokenType.OpenAngle;
case SyntaxKind.LeftBracket:
return SyntaxKind.RightBracket;
case SyntaxKind.OpenAngle:
return SyntaxKind.CloseAngle;
case SyntaxKind.RightBracket:
return SyntaxKind.LeftBracket;
case SyntaxKind.CloseAngle:
return SyntaxKind.OpenAngle;
default:
Debug.Fail("FlipBracket must be called with a bracket character");
return HtmlTokenType.Unknown;
return SyntaxKind.Unknown;
}
}
public override HtmlToken CreateMarkerToken()
public override SyntaxToken CreateMarkerToken()
{
return new HtmlToken(string.Empty, HtmlTokenType.Unknown);
return SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
}
public override HtmlTokenType GetKnownTokenType(KnownTokenType type)
public override SyntaxKind GetKnownTokenType(KnownTokenType type)
{
switch (type)
{
case KnownTokenType.CommentStart:
return HtmlTokenType.RazorCommentTransition;
return SyntaxKind.RazorCommentTransition;
case KnownTokenType.CommentStar:
return HtmlTokenType.RazorCommentStar;
return SyntaxKind.RazorCommentStar;
case KnownTokenType.CommentBody:
return HtmlTokenType.RazorComment;
return SyntaxKind.RazorComment;
case KnownTokenType.Identifier:
return HtmlTokenType.Text;
return SyntaxKind.HtmlTextLiteral;
case KnownTokenType.Keyword:
return HtmlTokenType.Text;
return SyntaxKind.HtmlTextLiteral;
case KnownTokenType.NewLine:
return HtmlTokenType.NewLine;
return SyntaxKind.NewLine;
case KnownTokenType.Transition:
return HtmlTokenType.Transition;
return SyntaxKind.Transition;
case KnownTokenType.WhiteSpace:
return HtmlTokenType.WhiteSpace;
return SyntaxKind.Whitespace;
default:
return HtmlTokenType.Unknown;
return SyntaxKind.Unknown;
}
}
protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList<RazorDiagnostic> errors)
protected override SyntaxToken CreateToken(string content, SyntaxKind kind, IReadOnlyList<RazorDiagnostic> errors)
{
return new HtmlToken(content, type, errors);
return SyntaxFactory.Token(kind, content, errors);
}
}
}

458
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
View File

File diff suppressed because it is too large Load Diff

77
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlToken.cs
View File

@ -1,77 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class HtmlToken : TokenBase<HtmlTokenType>
{
internal static readonly HtmlToken Hyphen = new HtmlToken("-", HtmlTokenType.Text);
public HtmlToken(string content, HtmlTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public HtmlToken(
string content,
HtmlTokenType type,
IReadOnlyList<RazorDiagnostic> errors)
: base(content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
protected override SyntaxToken GetSyntaxToken()
{
switch (Type)
{
case HtmlTokenType.Text:
return SyntaxFactory.HtmlTextToken(Content, Errors.ToArray());
case HtmlTokenType.WhiteSpace:
return SyntaxFactory.WhitespaceToken(Content, Errors.ToArray());
case HtmlTokenType.NewLine:
return SyntaxFactory.NewLineToken(Content, Errors.ToArray());
case HtmlTokenType.OpenAngle:
return SyntaxFactory.Punctuation(SyntaxKind.OpenAngle, Content, Errors.ToArray());
case HtmlTokenType.Bang:
return SyntaxFactory.Punctuation(SyntaxKind.Bang, Content, Errors.ToArray());
case HtmlTokenType.ForwardSlash:
return SyntaxFactory.Punctuation(SyntaxKind.ForwardSlash, Content, Errors.ToArray());
case HtmlTokenType.QuestionMark:
return SyntaxFactory.Punctuation(SyntaxKind.QuestionMark, Content, Errors.ToArray());
case HtmlTokenType.DoubleHyphen:
return SyntaxFactory.Punctuation(SyntaxKind.DoubleHyphen, Content, Errors.ToArray());
case HtmlTokenType.LeftBracket:
return SyntaxFactory.Punctuation(SyntaxKind.LeftBracket, Content, Errors.ToArray());
case HtmlTokenType.CloseAngle:
return SyntaxFactory.Punctuation(SyntaxKind.CloseAngle, Content, Errors.ToArray());
case HtmlTokenType.RightBracket:
return SyntaxFactory.Punctuation(SyntaxKind.RightBracket, Content, Errors.ToArray());
case HtmlTokenType.Equals:
return SyntaxFactory.Punctuation(SyntaxKind.Equals, Content, Errors.ToArray());
case HtmlTokenType.DoubleQuote:
return SyntaxFactory.Punctuation(SyntaxKind.DoubleQuote, Content, Errors.ToArray());
case HtmlTokenType.SingleQuote:
return SyntaxFactory.Punctuation(SyntaxKind.SingleQuote, Content, Errors.ToArray());
case HtmlTokenType.Transition:
return SyntaxFactory.Punctuation(SyntaxKind.Transition, Content, Errors.ToArray());
case HtmlTokenType.Colon:
return SyntaxFactory.Punctuation(SyntaxKind.Colon, Content, Errors.ToArray());
default:
return SyntaxFactory.UnknownToken(Content, Errors.ToArray());
}
}
}
}

32
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenType.cs
View File

@ -1,32 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
[Flags]
internal enum HtmlTokenType
{
Unknown,
Text, // Text which isn't one of the below
WhiteSpace, // Non-newline Whitespace
NewLine, // Newline
OpenAngle, // <
Bang, // !
ForwardSlash, // /
QuestionMark, // ?
DoubleHyphen, // --
LeftBracket, // [
CloseAngle, // >
RightBracket, // ]
Equals, // =
DoubleQuote, // "
SingleQuote, // '
Transition, // @
Colon,
RazorComment,
RazorCommentStar,
RazorCommentTransition
}
}

93
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs
View File

@ -3,11 +3,12 @@
using System.Collections.Generic;
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization
internal class HtmlTokenizer : Tokenizer<HtmlToken, HtmlTokenType>
internal class HtmlTokenizer : Tokenizer
{
private const char TransitionChar = '@';
@ -21,24 +22,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new HtmlTokenizerState? CurrentState => (HtmlTokenizerState?)base.CurrentState;
public override HtmlTokenType RazorCommentType
public override SyntaxKind RazorCommentKind
{
get { return HtmlTokenType.RazorComment; }
get { return SyntaxKind.RazorComment; }
}
public override HtmlTokenType RazorCommentTransitionType
public override SyntaxKind RazorCommentTransitionKind
{
get { return HtmlTokenType.RazorCommentTransition; }
get { return SyntaxKind.RazorCommentTransition; }
}
public override HtmlTokenType RazorCommentStarType
public override SyntaxKind RazorCommentStarKind
{
get { return HtmlTokenType.RazorCommentStar; }
get { return SyntaxKind.RazorCommentStar; }
}
protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList<RazorDiagnostic> errors)
protected override SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList<RazorDiagnostic> errors)
{
return new HtmlToken(content, type, errors);
return SyntaxFactory.Token(type, content, errors);
}
protected override StateResult Dispatch()
@ -66,7 +67,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Optimize memory allocation by returning constants for the most frequent cases
protected override string GetTokenContent(HtmlTokenType type)
protected override string GetTokenContent(SyntaxKind type)
{
var tokenLength = Buffer.Length;
@ -74,27 +75,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
switch (type)
{
case HtmlTokenType.OpenAngle:
case SyntaxKind.OpenAngle:
return "<";
case HtmlTokenType.Bang:
case SyntaxKind.Bang:
return "!";
case HtmlTokenType.ForwardSlash:
case SyntaxKind.ForwardSlash:
return "/";
case HtmlTokenType.QuestionMark:
case SyntaxKind.QuestionMark:
return "?";
case HtmlTokenType.LeftBracket:
case SyntaxKind.LeftBracket:
return "[";
case HtmlTokenType.CloseAngle:
case SyntaxKind.CloseAngle:
return ">";
case HtmlTokenType.RightBracket:
case SyntaxKind.RightBracket:
return "]";
case HtmlTokenType.Equals:
case SyntaxKind.Equals:
return "=";
case HtmlTokenType.DoubleQuote:
case SyntaxKind.DoubleQuote:
return "\"";
case HtmlTokenType.SingleQuote:
case SyntaxKind.SingleQuote:
return "'";
case HtmlTokenType.WhiteSpace:
case SyntaxKind.Whitespace:
if (Buffer[0] == ' ')
{
return " ";
@ -104,7 +105,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t";
}
break;
case HtmlTokenType.NewLine:
case SyntaxKind.NewLine:
if (Buffer[0] == '\n')
{
return "\n";
@ -113,7 +114,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
if (tokenLength == 2 && type == HtmlTokenType.NewLine)
if (tokenLength == 2 && type == SyntaxKind.NewLine)
{
return "\r\n";
}
@ -139,17 +140,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
HtmlTokenizerState.AfterRazorCommentTransition,
EndToken(HtmlTokenType.RazorCommentTransition));
EndToken(SyntaxKind.RazorCommentTransition));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return Transition(
HtmlTokenizerState.EscapedRazorCommentTransition,
EndToken(HtmlTokenType.Transition));
EndToken(SyntaxKind.Transition));
}
return Stay(EndToken(HtmlTokenType.Transition));
return Stay(EndToken(SyntaxKind.Transition));
}
else if (AtToken())
{
@ -164,7 +165,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult EscapedRazorCommentTransition()
{
TakeCurrent();
return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Transition));
return Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.Transition));
}
private StateResult Text()
@ -190,10 +191,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Output the Text token and return to the Data state to tokenize the next character (if there is one)
return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Text));
return Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.HtmlTextLiteral));
}
private HtmlToken Token()
private SyntaxToken Token()
{
Debug.Assert(AtToken());
var sym = CurrentCharacter;
@ -201,45 +202,45 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
switch (sym)
{
case '<':
return EndToken(HtmlTokenType.OpenAngle);
return EndToken(SyntaxKind.OpenAngle);
case '!':
return EndToken(HtmlTokenType.Bang);
return EndToken(SyntaxKind.Bang);
case '/':
return EndToken(HtmlTokenType.ForwardSlash);
return EndToken(SyntaxKind.ForwardSlash);
case '?':
return EndToken(HtmlTokenType.QuestionMark);
return EndToken(SyntaxKind.QuestionMark);
case '[':
return EndToken(HtmlTokenType.LeftBracket);
return EndToken(SyntaxKind.LeftBracket);
case '>':
return EndToken(HtmlTokenType.CloseAngle);
return EndToken(SyntaxKind.CloseAngle);
case ']':
return EndToken(HtmlTokenType.RightBracket);
return EndToken(SyntaxKind.RightBracket);
case '=':
return EndToken(HtmlTokenType.Equals);
return EndToken(SyntaxKind.Equals);
case '"':
return EndToken(HtmlTokenType.DoubleQuote);
return EndToken(SyntaxKind.DoubleQuote);
case '\'':
return EndToken(HtmlTokenType.SingleQuote);
return EndToken(SyntaxKind.SingleQuote);
case '-':
Debug.Assert(CurrentCharacter == '-');
TakeCurrent();
return EndToken(HtmlTokenType.DoubleHyphen);
return EndToken(SyntaxKind.DoubleHyphen);
default:
Debug.Fail("Unexpected token!");
return EndToken(HtmlTokenType.Unknown);
return EndToken(SyntaxKind.Unknown);
}
}
private HtmlToken Whitespace()
private SyntaxToken Whitespace()
{
while (ParserHelpers.IsWhitespace(CurrentCharacter))
{
TakeCurrent();
}
return EndToken(HtmlTokenType.WhiteSpace);
return EndToken(SyntaxKind.Whitespace);
}
private HtmlToken Newline()
private SyntaxToken Newline()
{
Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter));
// CSharp Spec §2.3.1
@ -249,7 +250,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent();
}
return EndToken(HtmlTokenType.NewLine);
return EndToken(SyntaxKind.NewLine);
}
private bool AtToken()
@ -274,7 +275,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null);
}
private StateResult Transition(HtmlTokenizerState state, HtmlToken result)
private StateResult Transition(HtmlTokenizerState state, SyntaxToken result)
{
return Transition((int)state, result);
}

20
src/Microsoft.AspNetCore.Razor.Language/Legacy/IToken.cs
View File

@ -1,20 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal interface IToken
{
Span Parent { get; set; }
string Content { get; }
SourceLocation Start { get; }
SyntaxKind SyntaxKind { get; }
SyntaxToken SyntaxToken { get; }
}
}

4
src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs
View File

@ -1,10 +1,12 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal interface ITokenizer
{
IToken NextToken();
SyntaxToken NextToken();
}
}

37
src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs
View File

@ -7,6 +7,7 @@ using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax;
using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -16,7 +17,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private readonly ISet<string> _keywords;
private readonly IReadOnlyCollection<string> _readOnlyKeywords;
public ImplicitExpressionEditHandler(Func<string, IEnumerable<IToken>> tokenizer, ISet<string> keywords, bool acceptTrailingDot)
public ImplicitExpressionEditHandler(Func<string, IEnumerable<Syntax.InternalSyntax.SyntaxToken>> tokenizer, ISet<string> keywords, bool acceptTrailingDot)
: base(tokenizer)
{
_keywords = keywords ?? new HashSet<string>();
@ -172,20 +173,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
for (var i = 0; i < target.Tokens.Count; i++)
{
var token = target.Tokens[i] as CSharpToken;
var token = target.Tokens[i];
if (token == null)
{
break;
}
var tokenStartIndex = token.Start.AbsoluteIndex;
var tokenEndIndex = tokenStartIndex + token.Content.Length;
var tokenStartIndex = token.Position;
var tokenEndIndex = token.EndPosition;
// We're looking for the first token that contains the SourceChange.
if (tokenEndIndex > change.Span.AbsoluteIndex)
{
if (tokenEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && token.Type == CSharpTokenType.Identifier)
if (tokenEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && token.Kind == SyntaxKind.Identifier)
{
// The token we're changing happens to be an identifier. Need to check if its transformed state is also one.
// We do this transformation logic to capture the case that the new text change happens to not be an identifier;
@ -200,8 +201,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break;
}
var newToken = (CSharpToken)newTokens.First();
if (newToken.Type == CSharpTokenType.Identifier)
var newToken = newTokens.First();
if (newToken.Kind == SyntaxKind.Identifier)
{
return true;
}
@ -241,8 +242,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var changeStart = change.Span.AbsoluteIndex;
var changeLength = change.Span.Length;
var changeEnd = changeStart + changeLength;
var tokens = target.Tokens.Cast<CSharpToken>().ToArray();
if (!IsInsideParenthesis(changeStart, tokens) || !IsInsideParenthesis(changeEnd, tokens))
if (!IsInsideParenthesis(changeStart, target.Tokens) || !IsInsideParenthesis(changeEnd, target.Tokens))
{
// Either the start or end of the delete does not fall inside of parenthesis, unacceptable inner deletion.
return false;
@ -274,8 +274,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
var tokens = target.Tokens.Cast<CSharpToken>().ToArray();
if (IsInsideParenthesis(change.Span.AbsoluteIndex, tokens))
if (IsInsideParenthesis(change.Span.AbsoluteIndex, target.Tokens))
{
return true;
}
@ -284,7 +283,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Internal for testing
internal static bool IsInsideParenthesis(int position, IReadOnlyList<CSharpToken> tokens)
internal static bool IsInsideParenthesis(int position, IReadOnlyList<SyntaxToken> tokens)
{
var balanceCount = 0;
var foundInsertionPoint = false;
@ -322,9 +321,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Internal for testing
internal static bool ContainsPosition(int position, CSharpToken currentToken)
internal static bool ContainsPosition(int position, SyntaxToken currentToken)
{
var tokenStart = currentToken.Start.AbsoluteIndex;
var tokenStart = currentToken.Position;
if (tokenStart == position)
{
// Token is exactly at the insertion point.
@ -342,14 +341,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Internal for testing
internal static bool TryUpdateBalanceCount(CSharpToken token, ref int count)
internal static bool TryUpdateBalanceCount(SyntaxToken token, ref int count)
{
var updatedCount = count;
if (token.Type == CSharpTokenType.LeftParenthesis)
if (token.Kind == SyntaxKind.LeftParenthesis)
{
updatedCount++;
}
else if (token.Type == CSharpTokenType.RightParenthesis)
else if (token.Kind == SyntaxKind.RightParenthesis)
{
if (updatedCount == 0)
{
@ -358,7 +357,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
updatedCount--;
}
else if (token.Type == CSharpTokenType.StringLiteral)
else if (token.Kind == SyntaxKind.StringLiteral)
{
var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '"')
@ -370,7 +369,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
}
else if (token.Type == CSharpTokenType.CharacterLiteral)
else if (token.Kind == SyntaxKind.CharacterLiteral)
{
var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '\'')

51
src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs
View File

@ -3,30 +3,29 @@
using System;
using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal abstract class LanguageCharacteristics<TTokenizer, TToken, TTokenType>
where TTokenType : struct
where TTokenizer : Tokenizer<TToken, TTokenType>
where TToken : TokenBase<TTokenType>
internal abstract class LanguageCharacteristics<TTokenizer>
where TTokenizer : Tokenizer
{
public abstract string GetSample(TTokenType type);
public abstract string GetSample(SyntaxKind type);
public abstract TTokenizer CreateTokenizer(ITextDocument source);
public abstract TTokenType FlipBracket(TTokenType bracket);
public abstract TToken CreateMarkerToken();
public abstract SyntaxKind FlipBracket(SyntaxKind bracket);
public abstract SyntaxToken CreateMarkerToken();
public virtual IEnumerable<TToken> TokenizeString(string content)
public virtual IEnumerable<SyntaxToken> TokenizeString(string content)
{
return TokenizeString(SourceLocation.Zero, content);
}
public virtual IEnumerable<TToken> TokenizeString(SourceLocation start, string input)
public virtual IEnumerable<SyntaxToken> TokenizeString(SourceLocation start, string input)
{
using (var reader = new SeekableTextReader(input, start.FilePath))
{
var tok = CreateTokenizer(reader);
TToken token;
SyntaxToken token;
while ((token = tok.NextToken()) != null)
{
yield return token;
@ -34,76 +33,76 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
public virtual bool IsWhiteSpace(TToken token)
public virtual bool IsWhiteSpace(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.WhiteSpace);
}
public virtual bool IsNewLine(TToken token)
public virtual bool IsNewLine(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.NewLine);
}
public virtual bool IsIdentifier(TToken token)
public virtual bool IsIdentifier(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.Identifier);
}
public virtual bool IsKeyword(TToken token)
public virtual bool IsKeyword(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.Keyword);
}
public virtual bool IsTransition(TToken token)
public virtual bool IsTransition(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.Transition);
}
public virtual bool IsCommentStart(TToken token)
public virtual bool IsCommentStart(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.CommentStart);
}
public virtual bool IsCommentStar(TToken token)
public virtual bool IsCommentStar(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.CommentStar);
}
public virtual bool IsCommentBody(TToken token)
public virtual bool IsCommentBody(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.CommentBody);
}
public virtual bool IsUnknown(TToken token)
public virtual bool IsUnknown(SyntaxToken token)
{
return IsKnownTokenType(token, KnownTokenType.Unknown);
}
public virtual bool IsKnownTokenType(TToken token, KnownTokenType type)
public virtual bool IsKnownTokenType(SyntaxToken token, KnownTokenType type)
{
return token != null && Equals(token.Type, GetKnownTokenType(type));
return token != null && Equals(token.Kind, GetKnownTokenType(type));
}
public virtual Tuple<TToken, TToken> SplitToken(TToken token, int splitAt, TTokenType leftType)
public virtual Tuple<SyntaxToken, SyntaxToken> SplitToken(SyntaxToken token, int splitAt, SyntaxKind leftType)
{
var left = CreateToken(token.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray);
TToken right = null;
SyntaxToken right = null;
if (splitAt < token.Content.Length)
{
right = CreateToken(token.Content.Substring(splitAt), token.Type, token.Errors);
right = CreateToken(token.Content.Substring(splitAt), token.Kind, token.GetDiagnostics());
}
return Tuple.Create(left, right);
}
public abstract TTokenType GetKnownTokenType(KnownTokenType type);
public abstract SyntaxKind GetKnownTokenType(KnownTokenType type);
public virtual bool KnowsTokenType(KnownTokenType type)
{
return type == KnownTokenType.Unknown || !Equals(GetKnownTokenType(type), GetKnownTokenType(KnownTokenType.Unknown));
}
protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList<RazorDiagnostic> errors);
protected abstract SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList<RazorDiagnostic> errors);
}
}

4
src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs
View File

@ -67,9 +67,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
builder.Kind = SpanKindInternal.Markup;
builder.ChunkGenerator = new MarkupChunkGenerator();
foreach (IToken sym in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content))
foreach (var token in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content))
{
builder.Accept(sym);
builder.Accept(token);
}
}
}

66
src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs
View File

@ -5,13 +5,16 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Microsoft.AspNetCore.Razor.Language.Syntax;
using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class Span : SyntaxTreeNode
{
private static readonly List<SyntaxToken> EmptyTokenList = new List<SyntaxToken>(0);
private static readonly int TypeHashCode = typeof(Span).GetHashCode();
private IReadOnlyList<Syntax.InternalSyntax.SyntaxToken> _greenTokens;
private string _content;
private int? _length;
private SourceLocation _start;
@ -24,7 +27,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public ISpanChunkGenerator ChunkGenerator { get; private set; }
public SpanKindInternal Kind { get; private set; }
public IReadOnlyList<IToken> Tokens { get; private set; }
public IReadOnlyList<SyntaxToken> Tokens { get; private set; }
// Allow test code to re-link spans
public Span Previous { get; internal set; }
@ -32,7 +36,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SpanEditHandler EditHandler { get; private set; }
public HtmlNodeSyntax SyntaxNode { get; private set; }
public SyntaxNode SyntaxNode { get; private set; }
public override bool IsBlock => false;
@ -96,20 +100,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void ReplaceWith(SpanBuilder builder)
{
Kind = builder.Kind;
Tokens = builder.Tokens;
SyntaxNode = builder.SyntaxNode;
for (var i = 0; i <Tokens.Count; i++)
{
Tokens[i].Parent = this;
}
_greenTokens = builder.Tokens;
EditHandler = builder.EditHandler;
ChunkGenerator = builder.ChunkGenerator ?? SpanChunkGenerator.Null;
_start = builder.Start;
SyntaxNode = builder.SyntaxNode?.CreateRed(parent: null, position: _start.AbsoluteIndex);
_content = null;
_length = null;
var tokens = EmptyTokenList;
if (_greenTokens.Count > 0)
{
tokens = new List<SyntaxToken>();
var currentStart = _start.AbsoluteIndex;
for (var i = 0; i < _greenTokens.Count; i++)
{
var token = new SyntaxToken(_greenTokens[i], parent: SyntaxNode, parentSpan: this, position: currentStart);
tokens.Add(token);
currentStart += token.FullWidth;
}
}
Tokens = tokens;
Parent?.ChildChanged();
// Since we took references to the values in SpanBuilder, clear its references out
@ -149,8 +162,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// </summary>
public override bool EquivalentTo(SyntaxTreeNode node)
{
var other = node as Span;
return other != null &&
return node is Span other &&
Kind.Equals(other.Kind) &&
Start.Equals(other.Start) &&
EditHandler.Equals(other.EditHandler) &&
@ -165,12 +177,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj)
{
var other = obj as Span;
return other != null &&
return obj is Span other &&
Kind.Equals(other.Kind) &&
EditHandler.Equals(other.EditHandler) &&
ChunkGenerator.Equals(other.ChunkGenerator) &&
Tokens.SequenceEqual(other.Tokens);
Tokens.SequenceEqual(other.Tokens, SyntaxTokenComparer.Default);
}
public override int GetHashCode()
@ -189,5 +200,28 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var spanBuilder = new SpanBuilder(this);
return spanBuilder.Build();
}
private class SyntaxTokenComparer : IEqualityComparer<SyntaxToken>
{
public static readonly SyntaxTokenComparer Default = new SyntaxTokenComparer();
private SyntaxTokenComparer()
{
}
public bool Equals(SyntaxToken x, SyntaxToken y)
{
return x.IsEquivalentTo(y);
}
public int GetHashCode(SyntaxToken obj)
{
var hash = HashCodeCombiner.Start();
hash.Add(obj.Content, StringComparer.Ordinal);
hash.Add(obj.Kind);
return hash;
}
}
}
}

30
src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs
View File

@ -12,7 +12,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal class SpanBuilder
{
private SourceLocation _start;
private List<IToken> _tokens;
private List<SyntaxToken> _tokens;
private SourceLocationTracker _tracker;
public SpanBuilder(Span original)
@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_start = original.Start;
ChunkGenerator = original.ChunkGenerator;
_tokens = new List<IToken>(original.Tokens);
_tokens = new List<SyntaxToken>(original.Tokens.Select(t =>t.Green));
_tracker = new SourceLocationTracker(original.Start);
}
@ -35,7 +35,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Start = location;
}
public HtmlNodeSyntax SyntaxNode { get; private set; }
public Syntax.GreenNode SyntaxNode { get; private set; }
public ISpanChunkGenerator ChunkGenerator { get; set; }
@ -53,13 +53,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SpanKindInternal Kind { get; set; }
public IReadOnlyList<IToken> Tokens
public IReadOnlyList<SyntaxToken> Tokens
{
get
{
if (_tokens == null)
{
_tokens = new List<IToken>();
_tokens = new List<SyntaxToken>();
}
return _tokens;
@ -73,9 +73,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Need to potentially allocate a new list because Span.ReplaceWith takes ownership
// of the original list.
_tokens = null;
_tokens = new List<IToken>();
_tokens = new List<SyntaxToken>();
EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<IToken>());
EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<SyntaxToken>());
ChunkGenerator = SpanChunkGenerator.Null;
Start = SourceLocation.Undefined;
}
@ -85,12 +85,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
SyntaxNode = GetSyntaxNode(syntaxKind);
var span = new Span(this);
for (var i = 0; i < span.Tokens.Count; i++)
{
var token = span.Tokens[i];
token.Parent = span;
}
return span;
}
@ -100,7 +94,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_tokens?.Clear();
}
public void Accept(IToken token)
public void Accept(SyntaxToken token)
{
if (token == null)
{
@ -116,20 +110,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_tracker.UpdateLocation(token.Content);
}
private HtmlNodeSyntax GetSyntaxNode(SyntaxKind syntaxKind)
private Syntax.GreenNode GetSyntaxNode(SyntaxKind syntaxKind)
{
if (syntaxKind == SyntaxKind.HtmlText)
{
var textTokens = new SyntaxListBuilder<SyntaxToken>(SyntaxListBuilder.Create());
foreach (var token in Tokens)
{
if (token.SyntaxKind == SyntaxKind.Unknown)
if (token.Kind == SyntaxKind.Unknown)
{
Debug.Assert(false, $"Unexpected html token {((HtmlToken)token).Type}");
Debug.Assert(false, $"Unexpected token {token.Kind}");
continue;
}
textTokens.Add(token.SyntaxToken);
textTokens.Add(token);
}
var textResult = textTokens.ToList();
return SyntaxFactory.HtmlText(new SyntaxList<SyntaxToken>(textResult.Node));

13
src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs
View File

@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
@ -11,12 +11,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
private static readonly int TypeHashCode = typeof(SpanEditHandler).GetHashCode();
public SpanEditHandler(Func<string, IEnumerable<IToken>> tokenizer)
public SpanEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer)
: this(tokenizer, AcceptedCharactersInternal.Any)
{
}
public SpanEditHandler(Func<string, IEnumerable<IToken>> tokenizer, AcceptedCharactersInternal accepted)
public SpanEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer, AcceptedCharactersInternal accepted)
{
AcceptedCharacters = accepted;
Tokenizer = tokenizer;
@ -24,9 +24,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public AcceptedCharactersInternal AcceptedCharacters { get; set; }
public Func<string, IEnumerable<IToken>> Tokenizer { get; set; }
public Func<string, IEnumerable<SyntaxToken>> Tokenizer { get; set; }
public static SpanEditHandler CreateDefault(Func<string, IEnumerable<IToken>> tokenizer)
public static SpanEditHandler CreateDefault(Func<string, IEnumerable<SyntaxToken>> tokenizer)
{
return new SpanEditHandler(tokenizer);
}
@ -116,8 +116,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj)
{
var other = obj as SpanEditHandler;
return other != null &&
return obj is SpanEditHandler other &&
GetType() == other.GetType() &&
AcceptedCharacters == other.AcceptedCharacters;
}

63
src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs
View File

@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
@ -150,7 +151,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
};
// Will contain tokens that represent a single attribute value: <input| class="btn"| />
var htmlTokens = span.Tokens.OfType<HtmlToken>().ToArray();
var tokens = span.Tokens;
var capturedAttributeValueStart = false;
var attributeValueStartLocation = span.Start;
@ -165,9 +166,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Iterate down through the tokens to find the name and the start of the value.
// We subtract the tokenOffset so we don't accept an ending quote of a span.
for (var i = 0; i < htmlTokens.Length - tokenOffset; i++)
for (var i = 0; i < tokens.Count - tokenOffset; i++)
{
var token = htmlTokens[i];
var token = tokens[i];
if (afterEquals)
{
@ -186,9 +187,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
attributeValueStartLocation = token.Start;
}
builder.Accept(token);
builder.Accept(token.Green);
}
else if (name == null && HtmlMarkupParser.IsValidAttributeNameToken(token))
else if (name == null && HtmlMarkupParser.IsValidAttributeNameToken(token.Green))
{
// We've captured all leading whitespace prior to the attribute name.
// We're now at: " |asp-for='...'" or " |asp-for=..."
@ -196,10 +197,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var nameBuilder = new StringBuilder();
// Move the indexer past the attribute name tokens.
for (var j = i; j < htmlTokens.Length; j++)
for (var j = i; j < tokens.Count; j++)
{
var nameToken = htmlTokens[j];
if (!HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
var nameToken = tokens[j];
if (!HtmlMarkupParser.IsValidAttributeNameToken(nameToken.Green))
{
break;
}
@ -213,7 +214,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
name = nameBuilder.ToString();
attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, name);
}
else if (token.Type == HtmlTokenType.Equals)
else if (token.Kind == SyntaxKind.Equals)
{
// We've captured all leading whitespace and the attribute name.
// We're now at: " asp-for|='...'" or " asp-for|=..."
@ -227,19 +228,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
do
{
i++; // Start from the token after '='.
} while (i < htmlTokens.Length &&
(htmlTokens[i].Type == HtmlTokenType.WhiteSpace ||
htmlTokens[i].Type == HtmlTokenType.NewLine));
} while (i < tokens.Count &&
(tokens[i].Kind == SyntaxKind.Whitespace ||
tokens[i].Kind == SyntaxKind.NewLine));
// Check for attribute start values, aka single or double quote
if (i < htmlTokens.Length && IsQuote(htmlTokens[i]))
if (i < tokens.Count && IsQuote(tokens[i]))
{
if (htmlTokens[i].Type == HtmlTokenType.SingleQuote)
if (tokens[i].Kind == SyntaxKind.SingleQuote)
{
attributeValueStyle = AttributeStructure.SingleQuotes;
}
tokenStartLocation = htmlTokens[i].Start;
tokenStartLocation = tokens[i].Start;
// If there's a start quote then there must be an end quote to be valid, skip it.
tokenOffset = 1;
@ -260,7 +261,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
afterEquals = true;
}
else if (token.Type == HtmlTokenType.WhiteSpace)
else if (token.Kind == SyntaxKind.Whitespace)
{
// We're at the start of the attribute, this branch may be hit on the first iterations of
// the loop since the parser separates attributes with their spaces included as tokens.
@ -343,9 +344,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var nameTokens = childSpan
.Tokens
.OfType<HtmlToken>()
.SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token)) // Skip prefix
.TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
.SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token.Green)) // Skip prefix
.TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken.Green))
.Select(nameToken => nameToken.Content);
var name = string.Concat(nameTokens);
@ -362,12 +362,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames);
var firstChild = builder.Children[0] as Span;
if (firstChild != null && firstChild.Tokens[0] is HtmlToken)
if (firstChild != null)
{
var htmlToken = firstChild.Tokens[firstChild.Tokens.Count - 1] as HtmlToken;
switch (htmlToken.Type)
var token = firstChild.Tokens[firstChild.Tokens.Count - 1];
switch (token.Kind)
{
case HtmlTokenType.Equals:
case SyntaxKind.Equals:
if (builder.Children.Count == 2 &&
builder.Children[1] is Span value &&
value.Kind == SpanKindInternal.Markup)
@ -385,10 +385,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
result.AttributeStructure = AttributeStructure.DoubleQuotes;
}
break;
case HtmlTokenType.DoubleQuote:
case SyntaxKind.DoubleQuote:
result.AttributeStructure = AttributeStructure.DoubleQuotes;
break;
case HtmlTokenType.SingleQuote:
case SyntaxKind.SingleQuote:
result.AttributeStructure = AttributeStructure.SingleQuotes;
break;
default:
@ -408,8 +408,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// In some malformed cases e.g. <p bar="false', the last Span (false' in the ex.) may contain more
// than a single HTML token. Do not ignore those other tokens.
var tokenCount = endSpan.Tokens.Count();
var endToken = tokenCount == 1 ? (HtmlToken)endSpan.Tokens.First() : null;
var tokenCount = endSpan.Tokens.Count;
var endToken = tokenCount == 1 ? endSpan.Tokens.First() : null;
// Checking to see if it's a quoted attribute, if so we should remove end quote
if (endToken != null && IsQuote(endToken))
@ -614,8 +614,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// expression).
var firstNonWhitespaceToken = span
.Tokens
.OfType<HtmlToken>()
.First(token => token.Type != HtmlTokenType.WhiteSpace && token.Type != HtmlTokenType.NewLine);
.First(token => token.Kind != SyntaxKind.Whitespace && token.Kind != SyntaxKind.NewLine);
var location = new SourceSpan(firstNonWhitespaceToken.Start, attributeName.Length);
return location;
@ -716,10 +715,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return firstBoundAttribute;
}
private static bool IsQuote(HtmlToken htmlToken)
private static bool IsQuote(SyntaxToken token)
{
return htmlToken.Type == HtmlTokenType.DoubleQuote ||
htmlToken.Type == HtmlTokenType.SingleQuote;
return token.Kind == SyntaxKind.DoubleQuote ||
token.Kind == SyntaxKind.SingleQuote;
}
private static void ConfigureNonStringAttribute(SpanBuilder builder, bool isDuplicateAttribute)

39
src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperParseTreeRewriter.cs
View File

@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
@ -372,10 +373,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var childOffset = 0;
if (childSpan.Tokens.Count > 0)
{
var potentialQuote = childSpan.Tokens[childSpan.Tokens.Count - 1] as HtmlToken;
var potentialQuote = childSpan.Tokens[childSpan.Tokens.Count - 1];
if (potentialQuote != null &&
(potentialQuote.Type == HtmlTokenType.DoubleQuote ||
potentialQuote.Type == HtmlTokenType.SingleQuote))
(potentialQuote.Kind == SyntaxKind.DoubleQuote ||
potentialQuote.Kind == SyntaxKind.SingleQuote))
{
childOffset = 1;
}
@ -409,23 +410,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Entire attribute is a string
for (var j = 0; j < endValueMarker; j++)
{
var htmlToken = (HtmlToken)childSpan.Tokens[j];
var token = childSpan.Tokens[j];
if (!afterEquals)
{
afterEquals = htmlToken.Type == HtmlTokenType.Equals;
afterEquals = token.Kind == SyntaxKind.Equals;
continue;
}
if (!atValue)
{
atValue = htmlToken.Type != HtmlTokenType.WhiteSpace &&
htmlToken.Type != HtmlTokenType.NewLine;
atValue = token.Kind != SyntaxKind.Whitespace &&
token.Kind != SyntaxKind.NewLine;
if (atValue)
{
if (htmlToken.Type == HtmlTokenType.DoubleQuote ||
htmlToken.Type == HtmlTokenType.SingleQuote)
if (token.Kind == SyntaxKind.DoubleQuote ||
token.Kind == SyntaxKind.SingleQuote)
{
endValueMarker--;
}
@ -433,14 +434,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Current token is considered the value (unquoted). Add its content to the
// attribute value builder before we move past it.
_attributeValueBuilder.Append(htmlToken.Content);
_attributeValueBuilder.Append(token.Content);
}
}
continue;
}
_attributeValueBuilder.Append(htmlToken.Content);
_attributeValueBuilder.Append(token.Content);
}
}
@ -643,10 +644,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (tagEnd != null && tagEnd.Kind == SpanKindInternal.Markup)
{
var endToken = tagEnd.Tokens.Count > 0 ?
tagEnd.Tokens[tagEnd.Tokens.Count - 1] as HtmlToken :
tagEnd.Tokens[tagEnd.Tokens.Count - 1] :
null;
if (endToken != null && endToken.Type == HtmlTokenType.CloseAngle)
if (endToken != null && endToken.Kind == SyntaxKind.CloseAngle)
{
return false;
}
@ -793,13 +794,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
var childSpan = (Span)child;
HtmlToken textToken = null;
SyntaxToken textToken = null;
for (var i = 0; i < childSpan.Tokens.Count; i++)
{
var token = childSpan.Tokens[i] as HtmlToken;
var token = childSpan.Tokens[i];
if (token != null &&
(token.Type & (HtmlTokenType.WhiteSpace | HtmlTokenType.Text)) == token.Type)
(token.Kind == SyntaxKind.Whitespace || token.Kind == SyntaxKind.HtmlTextLiteral))
{
textToken = token;
break;
@ -811,7 +812,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null;
}
return textToken.Type == HtmlTokenType.WhiteSpace ? null : textToken.Content;
return textToken.Kind == SyntaxKind.Whitespace ? null : textToken.Content;
}
private static bool IsEndTag(Block tagBlock)
@ -821,9 +822,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var childSpan = (Span)tagBlock.Children.First();
// We grab the token that could be forward slash
var relevantToken = (HtmlToken)childSpan.Tokens[childSpan.Tokens.Count == 1 ? 0 : 1];
var relevantToken = childSpan.Tokens[childSpan.Tokens.Count == 1 ? 0 : 1];
return relevantToken.Type == HtmlTokenType.ForwardSlash;
return relevantToken.Kind == SyntaxKind.ForwardSlash;
}
internal static bool IsComment(Span span)

90
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenBase.cs
View File

@ -1,90 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal abstract class TokenBase<TType> : IToken where TType : struct
{
protected TokenBase(
string content,
TType type,
IReadOnlyList<RazorDiagnostic> errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
Content = content;
Type = type;
Errors = errors;
}
public Span Parent { get; set; }
public IReadOnlyList<RazorDiagnostic> Errors { get; }
public string Content { get; }
public TType Type { get; }
public SourceLocation Start
{
get
{
if (Parent == null)
{
return SourceLocation.Undefined;
}
var tracker = new SourceLocationTracker(Parent.Start);
for (var i = 0; i < Parent.Tokens.Count; i++)
{
var token = Parent.Tokens[i];
if (object.ReferenceEquals(this, token))
{
break;
}
tracker.UpdateLocation(token.Content);
}
return tracker.CurrentLocation;
}
}
public SyntaxKind SyntaxKind => SyntaxToken.Kind;
public SyntaxToken SyntaxToken => GetSyntaxToken();
protected abstract SyntaxToken GetSyntaxToken();
public override bool Equals(object obj)
{
return obj is TokenBase<TType> other &&
string.Equals(Content, other.Content, StringComparison.Ordinal) &&
Type.Equals(other.Type);
}
public override int GetHashCode()
{
// Hash code should include only immutable properties.
var hash = HashCodeCombiner.Start();
hash.Add(Content, StringComparer.Ordinal);
hash.Add(Type);
return hash;
}
public override string ToString()
{
return string.Format(CultureInfo.InvariantCulture, "{0} [{1}]", Type, Content);
}
}
}

59
src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs
View File

@ -5,12 +5,11 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal abstract partial class Tokenizer<TToken, TTokenType> : ITokenizer
where TTokenType : struct
where TToken : TokenBase<TTokenType>
internal abstract class Tokenizer : ITokenizer
{
protected Tokenizer(ITextDocument source)
{
@ -31,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected int? CurrentState { get; set; }
protected TToken CurrentToken { get; private set; }
protected SyntaxToken CurrenSyntaxToken { get; private set; }
public ITextDocument Source { get; private set; }
@ -42,9 +41,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return Source.Peek() == -1; }
}
public abstract TTokenType RazorCommentStarType { get; }
public abstract TTokenType RazorCommentType { get; }
public abstract TTokenType RazorCommentTransitionType { get; }
public abstract SyntaxKind RazorCommentStarKind { get; }
public abstract SyntaxKind RazorCommentKind { get; }
public abstract SyntaxKind RazorCommentTransitionKind { get; }
protected bool HaveContent
{
@ -64,16 +63,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SourceLocation CurrentStart { get; private set; }
protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList<RazorDiagnostic> errors);
protected abstract SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList<RazorDiagnostic> errors);
protected abstract StateResult Dispatch();
IToken ITokenizer.NextToken()
SyntaxToken ITokenizer.NextToken()
{
return NextToken();
}
public virtual TToken NextToken()
public virtual SyntaxToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
@ -95,7 +94,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token;
}
protected virtual TToken Turn()
protected virtual SyntaxToken Turn()
{
if (CurrentState != null)
{
@ -105,19 +104,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var next = Dispatch();
CurrentState = next.State;
CurrentToken = next.Result;
CurrenSyntaxToken = next.Result;
}
while (CurrentState != null && CurrentToken == null);
while (CurrentState != null && CurrenSyntaxToken == null);
if (CurrentState == null)
{
return default(TToken); // Terminated
return default(SyntaxToken); // Terminated
}
return CurrentToken;
return CurrenSyntaxToken;
}
return default(TToken);
return default(SyntaxToken);
}
public void Reset()
@ -149,7 +148,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to
/// <see cref="Turn"/> should invoke the provided state.
/// </summary>
protected StateResult Transition(int state, TToken result)
protected StateResult Transition(int state, SyntaxToken result)
{
return new StateResult(state, result);
}
@ -159,7 +158,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new StateResult((int)state, result: null);
}
protected StateResult Transition(RazorCommentTokenizerState state, TToken result)
protected StateResult Transition(RazorCommentTokenizerState state, SyntaxToken result)
{
return new StateResult((int)state, result);
}
@ -180,12 +179,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to
/// <see cref="Turn"/> should re-invoke the current state.
/// </summary>
protected StateResult Stay(TToken result)
protected StateResult Stay(SyntaxToken result)
{
return new StateResult(CurrentState, result);
}
protected TToken Single(TTokenType type)
protected SyntaxToken Single(SyntaxKind type)
{
TakeCurrent();
return EndToken(type);
@ -199,9 +198,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
CurrentStart = CurrentLocation;
}
protected TToken EndToken(TTokenType type)
protected SyntaxToken EndToken(SyntaxKind type)
{
TToken token = null;
SyntaxToken token = null;
if (HaveContent)
{
// Perf: Don't allocate a new errors array unless necessary.
@ -222,7 +221,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token;
}
protected virtual string GetTokenContent(TTokenType type)
protected virtual string GetTokenContent(SyntaxKind type)
{
return Buffer.ToString();
}
@ -278,7 +277,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AssertCurrent('*');
TakeCurrent();
return Transition(1002, EndToken(RazorCommentStarType));
return Transition(1002, EndToken(RazorCommentStarKind));
}
protected StateResult RazorCommentBody()
@ -292,7 +291,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
RazorCommentTokenizerState.StarAfterRazorCommentBody,
EndToken(RazorCommentType));
EndToken(RazorCommentKind));
}
else
{
@ -306,7 +305,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
return Transition(StartState, EndToken(RazorCommentType));
return Transition(StartState, EndToken(RazorCommentKind));
}
protected StateResult StarAfterRazorCommentBody()
@ -315,14 +314,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
TakeCurrent();
return Transition(
RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
EndToken(RazorCommentStarType));
EndToken(RazorCommentStarKind));
}
protected StateResult AtTokenAfterRazorCommentBody()
{
AssertCurrent('@');
TakeCurrent();
return Transition(StartState, EndToken(RazorCommentTransitionType));
return Transition(StartState, EndToken(RazorCommentTransitionKind));
}
/// <summary>
@ -397,7 +396,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected struct StateResult
{
public StateResult(int? state, TToken result)
public StateResult(int? state, SyntaxToken result)
{
State = state;
Result = result;
@ -405,7 +404,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public int? State { get; }
public TToken Result { get; }
public SyntaxToken Result { get; }
}
private static LookaheadToken BeginLookahead(ITextBuffer buffer)

155
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
View File

@ -9,20 +9,18 @@ using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal abstract partial class TokenizerBackedParser<TTokenizer, TToken, TTokenType> : ParserBase
where TTokenType : struct
where TTokenizer : Tokenizer<TToken, TTokenType>
where TToken : TokenBase<TTokenType>
internal abstract class TokenizerBackedParser<TTokenizer> : ParserBase
where TTokenizer : Tokenizer
{
private readonly TokenizerView<TTokenizer, TToken, TTokenType> _tokenizer;
private readonly TokenizerView<TTokenizer> _tokenizer;
protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TToken, TTokenType> language, ParserContext context)
protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer> language, ParserContext context)
: base(context)
{
Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView<TTokenizer, TToken, TTokenType>(languageTokenizer);
_tokenizer = new TokenizerView<TTokenizer>(languageTokenizer);
Span = new SpanBuilder(CurrentLocation);
}
@ -32,14 +30,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected Action<SpanBuilder> SpanConfig { get; set; }
protected TToken CurrentToken
protected SyntaxToken CurrentToken
{
get { return _tokenizer.Current; }
}
protected SyntaxToken CurrentSyntaxToken => CurrentToken?.SyntaxToken;
protected TToken PreviousToken { get; private set; }
protected SyntaxToken PreviousToken { get; private set; }
protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
@ -50,7 +46,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _tokenizer.EndOfFile; }
}
protected LanguageCharacteristics<TTokenizer, TToken, TTokenType> Language { get; }
protected LanguageCharacteristics<TTokenizer> Language { get; }
protected virtual void HandleEmbeddedTransition()
{
@ -63,21 +59,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override void BuildSpan(SpanBuilder span, SourceLocation start, string content)
{
foreach (IToken sym in Language.TokenizeString(start, content))
foreach (var token in Language.TokenizeString(start, content))
{
span.Accept(sym);
span.Accept(token);
}
}
protected void Initialize(SpanBuilder span)
{
if (SpanConfig != null)
{
SpanConfig(span);
}
SpanConfig?.Invoke(span);
}
protected TToken Lookahead(int count)
protected SyntaxToken Lookahead(int count)
{
if (count < 0)
{
@ -89,7 +82,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// We add 1 in order to store the current token.
var tokens = new TToken[count + 1];
var tokens = new SyntaxToken[count + 1];
var currentToken = CurrentToken;
tokens[0] = currentToken;
@ -120,7 +113,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// <param name="condition">A predicate accepting the token being evaluated and the list of tokens which have been looped through.</param>
/// <returns>true, if the condition was met. false - if the condition wasn't met and the last token has already been processed.</returns>
/// <remarks>The list of previous tokens is passed in the reverse order. So the last processed element will be the first one in the list.</remarks>
protected bool LookaheadUntil(Func<TToken, IEnumerable<TToken>, bool> condition)
protected bool LookaheadUntil(Func<SyntaxToken, IEnumerable<SyntaxToken>, bool> condition)
{
if (condition == null)
{
@ -129,7 +122,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var matchFound = false;
var tokens = new List<TToken>();
var tokens = new List<SyntaxToken>();
tokens.Add(CurrentToken);
while (true)
@ -168,14 +161,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Helpers
[Conditional("DEBUG")]
internal void Assert(TTokenType expectedType)
internal void Assert(SyntaxKind expectedType)
{
Debug.Assert(!EndOfFile && TokenTypeEquals(CurrentToken.Type, expectedType));
Debug.Assert(!EndOfFile && TokenKindEquals(CurrentToken.Kind, expectedType));
}
abstract protected bool TokenTypeEquals(TTokenType x, TTokenType y);
protected abstract bool TokenKindEquals(SyntaxKind x, SyntaxKind y);
protected internal void PutBack(TToken token)
protected internal void PutBack(SyntaxToken token)
{
if (token != null)
{
@ -185,7 +178,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// <summary>
/// Put the specified tokens back in the input stream. The provided list MUST be in the ORDER THE TOKENS WERE READ. The
/// list WILL be reversed and the Putback(TToken) will be called on each item.
/// list WILL be reversed and the Putback(SyntaxToken) will be called on each item.
/// </summary>
/// <remarks>
/// If a document contains tokens: a, b, c, d, e, f
@ -194,9 +187,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// that is the correct format for providing to this method. The caller of this method would,
/// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order
/// </remarks>
protected internal void PutBack(IEnumerable<TToken> tokens)
protected internal void PutBack(IEnumerable<SyntaxToken> tokens)
{
foreach (TToken token in tokens.Reverse())
foreach (var token in tokens.Reverse())
{
PutBack(token);
}
@ -212,7 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected internal bool Balance(BalancingModes mode)
{
var left = CurrentToken.Type;
var left = CurrentToken.Kind;
var right = Language.FlipBracket(left);
var start = CurrentStart;
AcceptAndMoveNext();
@ -228,21 +221,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Balance(mode, left, right, start);
}
protected internal bool Balance(BalancingModes mode, TTokenType left, TTokenType right, SourceLocation start)
protected internal bool Balance(BalancingModes mode, SyntaxKind left, SyntaxKind right, SourceLocation start)
{
var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1;
if (!EndOfFile)
{
var syms = new List<TToken>();
var tokens = new List<SyntaxToken>();
do
{
if (IsAtEmbeddedTransition(
(mode & BalancingModes.AllowCommentsAndTemplates) == BalancingModes.AllowCommentsAndTemplates,
(mode & BalancingModes.AllowEmbeddedTransitions) == BalancingModes.AllowEmbeddedTransitions))
{
Accept(syms);
syms.Clear();
Accept(tokens);
tokens.Clear();
HandleEmbeddedTransition();
// Reset backtracking since we've already outputted some spans.
@ -258,7 +251,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
if (nesting > 0)
{
syms.Add(CurrentToken);
tokens.Add(CurrentToken);
}
}
while (nesting > 0 && NextToken());
@ -280,29 +273,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else
{
Accept(syms);
Accept(tokens);
}
}
else
{
// Accept all the tokens we saw
Accept(syms);
Accept(tokens);
}
}
return nesting == 0;
}
protected internal bool NextIs(TTokenType type)
protected internal bool NextIs(SyntaxKind type)
{
return NextIs(sym => sym != null && TokenTypeEquals(type, sym.Type));
return NextIs(token => token != null && TokenKindEquals(type, token.Kind));
}
protected internal bool NextIs(params TTokenType[] types)
protected internal bool NextIs(params SyntaxKind[] types)
{
return NextIs(sym => sym != null && types.Any(t => TokenTypeEquals(t, sym.Type)));
return NextIs(token => token != null && types.Any(t => TokenKindEquals(t, token.Kind)));
}
protected internal bool NextIs(Func<TToken, bool> condition)
protected internal bool NextIs(Func<SyntaxToken, bool> condition)
{
var cur = CurrentToken;
if (NextToken())
@ -322,14 +315,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
protected internal bool Was(TTokenType type)
protected internal bool Was(SyntaxKind type)
{
return PreviousToken != null && TokenTypeEquals(PreviousToken.Type, type);
return PreviousToken != null && TokenKindEquals(PreviousToken.Kind, type);
}
protected internal bool At(TTokenType type)
protected internal bool At(SyntaxKind type)
{
return !EndOfFile && CurrentToken != null && TokenTypeEquals(CurrentToken.Type, type);
return !EndOfFile && CurrentToken != null && TokenKindEquals(CurrentToken.Kind, type);
}
protected internal bool AcceptAndMoveNext()
@ -338,11 +331,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return NextToken();
}
protected TToken AcceptSingleWhiteSpaceCharacter()
protected SyntaxToken AcceptSingleWhiteSpaceCharacter()
{
if (Language.IsWhiteSpace(CurrentToken))
{
Tuple<TToken, TToken> pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
var pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
Accept(pair.Item1);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
NextToken();
@ -351,19 +344,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null;
}
protected internal void Accept(IEnumerable<TToken> tokens)
protected internal void Accept(IEnumerable<SyntaxToken> tokens)
{
foreach (TToken token in tokens)
foreach (var token in tokens)
{
Accept(token);
}
}
protected internal void Accept(TToken token)
protected internal void Accept(SyntaxToken token)
{
if (token != null)
{
foreach (var error in token.Errors)
foreach (var error in token.GetDiagnostics())
{
Context.ErrorSink.OnError(error);
}
@ -372,11 +365,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
protected internal bool AcceptAll(params TTokenType[] types)
protected internal bool AcceptAll(params SyntaxKind[] kinds)
{
foreach (TTokenType type in types)
foreach (var kind in kinds)
{
if (CurrentToken == null || !TokenTypeEquals(CurrentToken.Type, type))
if (CurrentToken == null || !TokenKindEquals(CurrentToken.Kind, kind))
{
return false;
}
@ -442,7 +435,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected IDisposable PushSpanConfig(Action<SpanBuilder, Action<SpanBuilder>> newConfig)
{
Action<SpanBuilder> old = SpanConfig;
var old = SpanConfig;
ConfigureSpan(newConfig);
return new DisposableAction(() => SpanConfig = old);
}
@ -455,7 +448,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected void ConfigureSpan(Action<SpanBuilder, Action<SpanBuilder>> config)
{
Action<SpanBuilder> prev = SpanConfig;
var prev = SpanConfig;
if (config == null)
{
SpanConfig = null;
@ -472,9 +465,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Expected(Language.GetKnownTokenType(type));
}
protected internal void Expected(params TTokenType[] types)
protected internal void Expected(params SyntaxKind[] types)
{
Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Type));
Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Kind));
AcceptAndMoveNext();
}
@ -483,7 +476,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Optional(Language.GetKnownTokenType(type));
}
protected internal bool Optional(TTokenType type)
protected internal bool Optional(SyntaxKind type)
{
if (At(type))
{
@ -503,61 +496,61 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true;
}
protected internal void AcceptWhile(TTokenType type)
protected internal void AcceptWhile(SyntaxKind type)
{
AcceptWhile(sym => TokenTypeEquals(type, sym.Type));
AcceptWhile(token => TokenKindEquals(type, token.Kind));
}
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
protected internal void AcceptWhile(TTokenType type1, TTokenType type2)
protected internal void AcceptWhile(SyntaxKind type1, SyntaxKind type2)
{
AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type));
AcceptWhile(token => TokenKindEquals(type1, token.Kind) || TokenKindEquals(type2, token.Kind));
}
protected internal void AcceptWhile(TTokenType type1, TTokenType type2, TTokenType type3)
protected internal void AcceptWhile(SyntaxKind type1, SyntaxKind type2, SyntaxKind type3)
{
AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type) || TokenTypeEquals(type3, sym.Type));
AcceptWhile(token => TokenKindEquals(type1, token.Kind) || TokenKindEquals(type2, token.Kind) || TokenKindEquals(type3, token.Kind));
}
protected internal void AcceptWhile(params TTokenType[] types)
protected internal void AcceptWhile(params SyntaxKind[] types)
{
AcceptWhile(sym => types.Any(expected => TokenTypeEquals(expected, sym.Type)));
AcceptWhile(token => types.Any(expected => TokenKindEquals(expected, token.Kind)));
}
protected internal void AcceptUntil(TTokenType type)
protected internal void AcceptUntil(SyntaxKind type)
{
AcceptWhile(sym => !TokenTypeEquals(type, sym.Type));
AcceptWhile(token => !TokenKindEquals(type, token.Kind));
}
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
protected internal void AcceptUntil(TTokenType type1, TTokenType type2)
protected internal void AcceptUntil(SyntaxKind type1, SyntaxKind type2)
{
AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type));
AcceptWhile(token => !TokenKindEquals(type1, token.Kind) && !TokenKindEquals(type2, token.Kind));
}
protected internal void AcceptUntil(TTokenType type1, TTokenType type2, TTokenType type3)
protected internal void AcceptUntil(SyntaxKind type1, SyntaxKind type2, SyntaxKind type3)
{
AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type) && !TokenTypeEquals(type3, sym.Type));
AcceptWhile(token => !TokenKindEquals(type1, token.Kind) && !TokenKindEquals(type2, token.Kind) && !TokenKindEquals(type3, token.Kind));
}
protected internal void AcceptUntil(params TTokenType[] types)
protected internal void AcceptUntil(params SyntaxKind[] types)
{
AcceptWhile(sym => types.All(expected => !TokenTypeEquals(expected, sym.Type)));
AcceptWhile(token => types.All(expected => !TokenKindEquals(expected, token.Kind)));
}
protected internal void AcceptWhile(Func<TToken, bool> condition)
protected internal void AcceptWhile(Func<SyntaxToken, bool> condition)
{
Accept(ReadWhileLazy(condition));
}
protected internal IEnumerable<TToken> ReadWhile(Func<TToken, bool> condition)
protected internal IEnumerable<SyntaxToken> ReadWhile(Func<SyntaxToken, bool> condition)
{
return ReadWhileLazy(condition).ToList();
}
protected TToken AcceptWhiteSpaceInLines()
protected SyntaxToken AcceptWhiteSpaceInLines()
{
TToken lastWs = null;
SyntaxToken lastWs = null;
while (Language.IsWhiteSpace(CurrentToken) || Language.IsNewLine(CurrentToken))
{
// Capture the previous whitespace node
@ -591,7 +584,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Don't open this to sub classes because it's lazy but it looks eager.
// You have to advance the Enumerable to read the next characters.
internal IEnumerable<TToken> ReadWhileLazy(Func<TToken, bool> condition)
internal IEnumerable<SyntaxToken> ReadWhileLazy(Func<SyntaxToken, bool> condition)
{
while (EnsureCurrent() && condition(CurrentToken))
{

12
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
View File

@ -1,12 +1,12 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class TokenizerView<TTokenizer, TToken, TTokenType>
where TTokenType : struct
where TTokenizer : Tokenizer<TToken, TTokenType>
where TToken : TokenBase<TTokenType>
internal class TokenizerView<TTokenizer>
where TTokenizer : Tokenizer
{
public TokenizerView(TTokenizer tokenizer)
{
@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public TTokenizer Tokenizer { get; private set; }
public bool EndOfFile { get; private set; }
public TToken Current { get; private set; }
public SyntaxToken Current { get; private set; }
public ITextDocument Source
{
@ -29,7 +29,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return !EndOfFile;
}
public void PutBack(TToken token)
public void PutBack(SyntaxToken token)
{
Source.Position -= token.Content.Length;
Current = null;

64
src/Microsoft.AspNetCore.Razor.Language/Syntax/GreenNode.cs
View File

@ -377,6 +377,70 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
}
#endregion
#region Equivalence
public virtual bool IsEquivalentTo(GreenNode other)
{
if (this == other)
{
return true;
}
if (other == null)
{
return false;
}
return EquivalentToInternal(this, other);
}
private static bool EquivalentToInternal(GreenNode node1, GreenNode node2)
{
if (node1.Kind != node2.Kind)
{
// A single-element list is usually represented as just a single node,
// but can be represented as a List node with one child. Move to that
// child if necessary.
if (node1.IsList && node1.SlotCount == 1)
{
node1 = node1.GetSlot(0);
}
if (node2.IsList && node2.SlotCount == 1)
{
node2 = node2.GetSlot(0);
}
if (node1.Kind != node2.Kind)
{
return false;
}
}
if (node1.FullWidth != node2.FullWidth)
{
return false;
}
var n = node1.SlotCount;
if (n != node2.SlotCount)
{
return false;
}
for (var i = 0; i < n; i++)
{
var node1Child = node1.GetSlot(i);
var node2Child = node2.GetSlot(i);
if (node1Child != null && node2Child != null && !node1Child.IsEquivalentTo(node2Child))
{
return false;
}
}
return true;
}
#endregion
#region Factories
public virtual GreenNode CreateList(IEnumerable<GreenNode> nodes, bool alwaysCreateListNode = false)
{

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/HtmlTextTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class HtmlTextTokenSyntax : SyntaxToken
{
internal HtmlTextTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.HtmlTextTokenSyntax Green => (InternalSyntax.HtmlTextTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.HtmlTextTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as HtmlTextTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.HtmlTextTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as HtmlTextTokenSyntax;
}
}
}

53
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/HtmlTextTokenSyntax.cs
View File

@ -1,53 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class HtmlTextTokenSyntax : SyntaxToken
{
internal HtmlTextTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.HtmlTextLiteralToken, text, null, null, diagnostics, null)
{
}
internal HtmlTextTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.HtmlTextLiteralToken, text, leadingTrivia, trailingTrivia)
{
}
protected HtmlTextTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected HtmlTextTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position)
{
return new Syntax.HtmlTextTokenSyntax(this, parent, position);
}
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new HtmlTextTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

50
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/NewLineTokenSyntax.cs
View File

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class NewLineTokenSyntax : SyntaxToken
{
internal NewLineTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.NewLine, text, null, null, diagnostics, null)
{
}
internal NewLineTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.NewLine, text, leadingTrivia, trailingTrivia)
{
}
protected NewLineTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected NewLineTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.NewLineTokenSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new NewLineTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

45
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/PunctuationSyntax.cs
View File

@ -1,45 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class PunctuationSyntax : SyntaxToken
{
internal PunctuationSyntax(SyntaxKind kind, string name, RazorDiagnostic[] diagnostics)
: this(kind, name, null, null, diagnostics, null)
{
}
internal PunctuationSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: this(kind, name, leadingTrivia, trailingTrivia, null, null)
{
}
internal PunctuationSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.PunctuationSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new PunctuationSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new PunctuationSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new PunctuationSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new PunctuationSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

26
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxFactory.cs
View File

@ -1,6 +1,9 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Collections.Generic;
using System.Linq;
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal static class SyntaxFactory
@ -10,29 +13,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return new HtmlTextSyntax(textTokens.Node);
}
internal static HtmlTextTokenSyntax HtmlTextToken(string text, params RazorDiagnostic[] diagnostics)
internal static SyntaxToken Token(SyntaxKind kind, string content, IEnumerable<RazorDiagnostic> diagnostics)
{
return new HtmlTextTokenSyntax(text, diagnostics);
return Token(kind, content, diagnostics.ToArray());
}
internal static WhitespaceTokenSyntax WhitespaceToken(string text, params RazorDiagnostic[] diagnostics)
internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
{
return new WhitespaceTokenSyntax(text, diagnostics);
}
internal static NewLineTokenSyntax NewLineToken(string text, params RazorDiagnostic[] diagnostics)
{
return new NewLineTokenSyntax(text, diagnostics);
}
internal static PunctuationSyntax Punctuation(SyntaxKind syntaxKind, string text, params RazorDiagnostic[] diagnostics)
{
return new PunctuationSyntax(syntaxKind, text, diagnostics);
}
internal static UnknownTokenSyntax UnknownToken(string text, params RazorDiagnostic[] diagnostics)
{
return new UnknownTokenSyntax(text, diagnostics);
return new SyntaxToken(kind, content, diagnostics);
}
}
}

103
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxToken.cs
View File

@ -4,32 +4,39 @@
using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.AspNetCore.Razor.Language.Legacy;
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal abstract class SyntaxToken : GreenNode
internal class SyntaxToken : GreenNode
{
internal SyntaxToken(SyntaxKind tokenKind, string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(tokenKind, text.Length)
internal SyntaxToken(SyntaxKind kind, string content, RazorDiagnostic[] diagnostics)
: base(kind, content.Length, diagnostics, annotations: null)
{
Text = text;
Content = content;
}
internal SyntaxToken(SyntaxKind kind, string content, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, content.Length)
{
Content = content;
LeadingTrivia = leadingTrivia;
AdjustFlagsAndWidth(leadingTrivia);
TrailingTrivia = trailingTrivia;
AdjustFlagsAndWidth(trailingTrivia);
}
internal SyntaxToken(SyntaxKind tokenKind, string text, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(tokenKind, text.Length, diagnostics, annotations)
internal SyntaxToken(SyntaxKind kind, string content, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, content.Length, diagnostics, annotations)
{
Text = text;
Content = content;
LeadingTrivia = leadingTrivia;
AdjustFlagsAndWidth(leadingTrivia);
TrailingTrivia = trailingTrivia;
AdjustFlagsAndWidth(trailingTrivia);
}
public string Text { get; }
public string Content { get; }
public GreenNode LeadingTrivia { get; }
@ -37,7 +44,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
internal override bool IsToken => true;
public override int Width => Text.Length;
public override int Width => Content.Length;
internal override SyntaxNode CreateRed(SyntaxNode parent, int position)
{
return new Syntax.SyntaxToken(this, parent, position);
}
protected override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
@ -50,7 +62,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
}
}
writer.Write(Text);
writer.Write(Content);
if (trailing)
{
@ -87,14 +99,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return TokenWithLeadingTrivia(trivia);
}
public abstract SyntaxToken TokenWithLeadingTrivia(GreenNode trivia);
public virtual SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new SyntaxToken(Kind, Content, trivia, TrailingTrivia, GetDiagnostics(), GetAnnotations());
}
public sealed override GreenNode WithTrailingTrivia(GreenNode trivia)
{
return TokenWithTrailingTrivia(trivia);
}
public abstract SyntaxToken TokenWithTrailingTrivia(GreenNode trivia);
public virtual SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new SyntaxToken(Kind, Content, LeadingTrivia, trivia, GetDiagnostics(), GetAnnotations());
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new SyntaxToken(Kind, Content, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new SyntaxToken(Kind, Content, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
protected override sealed int GetSlotCount()
{
@ -103,7 +131,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
internal override sealed GreenNode GetSlot(int index)
{
throw new InvalidOperationException();
throw new InvalidOperationException("Tokens don't have slots.");
}
internal override GreenNode Accept(SyntaxVisitor visitor)
@ -111,9 +139,56 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return visitor.VisitSyntaxToken(this);
}
public override bool IsEquivalentTo(GreenNode other)
{
if (!base.IsEquivalentTo(other))
{
return false;
}
var otherToken = (SyntaxToken)other;
if (Content != otherToken.Content)
{
return false;
}
var thisLeading = GetLeadingTrivia();
var otherLeading = otherToken.GetLeadingTrivia();
if (thisLeading != otherLeading)
{
if (thisLeading == null || otherLeading == null)
{
return false;
}
if (!thisLeading.IsEquivalentTo(otherLeading))
{
return false;
}
}
var thisTrailing = GetTrailingTrivia();
var otherTrailing = otherToken.GetTrailingTrivia();
if (thisTrailing != otherTrailing)
{
if (thisTrailing == null || otherTrailing == null)
{
return false;
}
if (!thisTrailing.IsEquivalentTo(otherTrailing))
{
return false;
}
}
return true;
}
public override string ToString()
{
return Text;
return Content;
}
}
}

15
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxTrivia.cs
View File

@ -76,5 +76,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
return new SyntaxTrivia(Kind, Text, GetDiagnostics(), annotations);
}
public override bool IsEquivalentTo(GreenNode other)
{
if (!base.IsEquivalentTo(other))
{
return false;
}
if (Text != ((SyntaxTrivia)other).Text)
{
return false;
}
return true;
}
}
}

50
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/UnknownTokenSyntax.cs
View File

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class UnknownTokenSyntax : SyntaxToken
{
internal UnknownTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.Unknown, text, null, null, diagnostics, null)
{
}
internal UnknownTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.Unknown, text, leadingTrivia, trailingTrivia)
{
}
protected UnknownTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected UnknownTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.UnknownTokenSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new UnknownTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

50
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/WhitespaceTokenSyntax.cs
View File

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class WhitespaceTokenSyntax : SyntaxToken
{
internal WhitespaceTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.Whitespace, text, null, null, diagnostics, null)
{
}
internal WhitespaceTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.Whitespace, text, leadingTrivia, trailingTrivia)
{
}
protected WhitespaceTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected WhitespaceTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.WhitespaceTokenSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new WhitespaceTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/NewLineTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class NewLineTokenSyntax : SyntaxToken
{
internal NewLineTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.NewLineTokenSyntax Green => (InternalSyntax.NewLineTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.NewLineTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as NewLineTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.NewLineTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as NewLineTokenSyntax;
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/PunctuationSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class PunctuationSyntax : SyntaxToken
{
internal PunctuationSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.PunctuationSyntax Green => (InternalSyntax.PunctuationSyntax)base.Green;
public string Punctuation => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.PunctuationSyntax(Kind, Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as PunctuationSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.PunctuationSyntax(Kind, Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as PunctuationSyntax;
}
}
}

13
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxFactory.cs Normal file
View File

@ -0,0 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal static class SyntaxFactory
{
internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
{
return new SyntaxToken(InternalSyntax.SyntaxFactory.Token(kind, content), parent: null, position: 0);
}
}
}

85
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxKind.cs
View File

@ -5,28 +5,93 @@ namespace Microsoft.AspNetCore.Razor.Language
{
internal enum SyntaxKind : byte
{
#region Nodes
HtmlText,
HtmlDocument,
HtmlDeclaration,
#endregion
#region Tokens
// Common
Unknown,
List,
Whitespace,
NewLine,
Colon,
QuestionMark,
RightBracket,
LeftBracket,
Equals,
Transition,
// HTML
HtmlText,
HtmlDocument,
HtmlDeclaration,
HtmlTextLiteralToken,
HtmlTextLiteral,
OpenAngle,
Bang,
ForwardSlash,
QuestionMark,
DoubleHyphen,
LeftBracket,
CloseAngle,
RightBracket,
Equals,
DoubleQuote,
SingleQuote,
Transition,
Colon,
// CSharp literals
Identifier,
Keyword,
IntegerLiteral,
CSharpComment,
RealLiteral,
CharacterLiteral,
StringLiteral,
// CSharp operators
Arrow,
Minus,
Decrement,
MinusAssign,
NotEqual,
Not,
Modulo,
ModuloAssign,
AndAssign,
And,
DoubleAnd,
LeftParenthesis,
RightParenthesis,
Star,
MultiplyAssign,
Comma,
Dot,
Slash,
DivideAssign,
DoubleColon,
Semicolon,
NullCoalesce,
XorAssign,
Xor,
LeftBrace,
OrAssign,
DoubleOr,
Or,
RightBrace,
Tilde,
Plus,
PlusAssign,
Increment,
LessThan,
LessThanEqual,
LeftShift,
LeftShiftAssign,
Assign,
GreaterThan,
GreaterThanEqual,
RightShift,
RightShiftAssign,
Hash,
// Razor specific
RazorComment,
RazorCommentStar,
RazorCommentTransition,
#endregion
}
}

15
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxNode.cs
View File

@ -239,6 +239,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
return Green.GetAnnotations();
}
public bool IsEquivalentTo(SyntaxNode other)
{
if (this == other)
{
return true;
}
if (other == null)
{
return false;
}
return Green.IsEquivalentTo(other.Green);
}
public override string ToString()
{
return Green.ToString();

73
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxToken.cs
View File

@ -3,29 +3,70 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Legacy;
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal abstract class SyntaxToken : SyntaxNode
internal class SyntaxToken : SyntaxNode
{
internal SyntaxToken(GreenNode green, SyntaxNode parent, int position)
: this(green, parent, null, position)
{
}
// Temporary plumbing
internal SyntaxToken(GreenNode green, SyntaxNode parent, Span parentSpan, int position)
: base(green, parent, position)
{
Debug.Assert(parent == null || !parent.Green.IsList, "list cannot be a parent");
Debug.Assert(green == null || green.IsToken, "green must be a token");
ParentSpan = parentSpan;
}
// Temporary plumbing
internal Span ParentSpan { get; }
// Temporary plumbing
internal SourceLocation Start
{
get
{
if (ParentSpan == null)
{
return SourceLocation.Undefined;
}
var tracker = new SourceLocationTracker(ParentSpan.Start);
for (var i = 0; i < ParentSpan.Tokens.Count; i++)
{
var token = ParentSpan.Tokens[i];
if (object.ReferenceEquals(this, token))
{
break;
}
tracker.UpdateLocation(token.Content);
}
return tracker.CurrentLocation;
}
}
internal new InternalSyntax.SyntaxToken Green => (InternalSyntax.SyntaxToken)base.Green;
public string Text => Green.Text;
public string Content => Green.Content;
internal override sealed SyntaxNode GetCachedSlot(int index)
{
throw new InvalidOperationException();
throw new InvalidOperationException("Tokens can't have slots.");
}
internal override sealed SyntaxNode GetNodeSlot(int slot)
{
throw new InvalidOperationException();
throw new InvalidOperationException("Tokens can't have slots.");
}
internal override SyntaxNode Accept(SyntaxVisitor visitor)
@ -33,24 +74,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
return visitor.VisitSyntaxToken(this);
}
internal abstract SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia);
public SyntaxToken WithLeadingTrivia(SyntaxNode trivia)
{
return Green != null
? new SyntaxToken(Green.WithLeadingTrivia(trivia.Green), parent: null, position: 0)
: default(SyntaxToken);
}
internal abstract SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia);
public SyntaxToken WithLeadingTrivia(SyntaxNode trivia) => WithLeadingTriviaCore(trivia);
public SyntaxToken WithTrailingTrivia(SyntaxNode trivia) => WithTrailingTriviaCore(trivia);
public SyntaxToken WithTrailingTrivia(SyntaxNode trivia)
{
return Green != null
? new SyntaxToken(Green.WithTrailingTrivia(trivia.Green), parent: null, position: 0)
: default(SyntaxToken);
}
public SyntaxToken WithLeadingTrivia(IEnumerable<SyntaxTrivia> trivia)
{
var greenList = trivia?.Select(t => t.Green);
return WithLeadingTriviaCore(Green.CreateList(greenList)?.CreateRed());
return WithLeadingTrivia(Green.CreateList(greenList)?.CreateRed());
}
public SyntaxToken WithTrailingTrivia(IEnumerable<SyntaxTrivia> trivia)
{
var greenList = trivia?.Select(t => t.Green);
return WithTrailingTriviaCore(Green.CreateList(greenList)?.CreateRed());
return WithTrailingTrivia(Green.CreateList(greenList)?.CreateRed());
}
public override SyntaxTriviaList GetLeadingTrivia()
@ -85,7 +132,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
public override string ToString()
{
return Text;
return Content;
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/UnknownTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class UnknownTokenSyntax : SyntaxToken
{
internal UnknownTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.UnknownTokenSyntax Green => (InternalSyntax.UnknownTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.UnknownTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as UnknownTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.UnknownTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as UnknownTokenSyntax;
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/WhitespaceTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class WhitespaceTokenSyntax : SyntaxToken
{
internal WhitespaceTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.WhitespaceTokenSyntax Green => (InternalSyntax.WhitespaceTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.WhitespaceTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as WhitespaceTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.WhitespaceTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as WhitespaceTokenSyntax;
}
}
}

12
src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorCompletionFactsService.cs
View File

@ -7,6 +7,7 @@ using System.ComponentModel.Composition;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language;
using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.VisualStudio.Editor.Razor
{
@ -83,16 +84,11 @@ namespace Microsoft.VisualStudio.Editor.Razor
}
// Internal for testing
internal static bool IsDirectiveCompletableToken(IToken token)
internal static bool IsDirectiveCompletableToken(SyntaxToken token)
{
if (!(token is CSharpToken csharpToken))
{
return false;
}
return csharpToken.Type == CSharpTokenType.Identifier ||
return token.Kind == SyntaxKind.Identifier ||
// Marker symbol
csharpToken.Type == CSharpTokenType.Unknown;
token.Kind == SyntaxKind.Unknown;
}
}
}

3
src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
View File

@ -143,8 +143,7 @@ namespace Microsoft.VisualStudio.Editor.Razor
{
return currentChild is Span currentSpan &&
currentSpan.Tokens.Count == 1 &&
currentSpan.Tokens[0] is CSharpToken symbol &&
symbol.Type == CSharpTokenType.LeftBrace;
currentSpan.Tokens[0].Kind == SyntaxKind.LeftBrace;
}
}
}

12
src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
View File

@ -206,17 +206,5 @@ namespace Microsoft.VisualStudio.Editor.Razor
return false;
}
private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.IToken symbol)
{
if (!(symbol is CSharpToken csharpSymbol))
{
return false;
}
return csharpSymbol.Type == CSharpTokenType.Identifier ||
// Marker symbol
csharpSymbol.Type == CSharpTokenType.Unknown;
}
}
}

5
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -13,7 +14,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder()
{
@ -29,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var parentBlock = blockBuilder.Build();
var originalBlockLength = parentBlock.Length;
spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
span.ReplaceWith(spanBuilder);
// Wire up parents now so we can re-trigger ChildChanged to cause cache refresh.

2
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
View File

@ -11,7 +11,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void GetSample_RightShiftAssign_ReturnsCorrectToken()
{
// Arrange & Act
var token = CSharpLanguageCharacteristics.Instance.GetSample(CSharpTokenType.RightShiftAssign);
var token = CSharpLanguageCharacteristics.Instance.GetSample(SyntaxKind.RightShiftAssign);
// Assert
Assert.Equal(">>=", token);

45
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
View File

@ -1,22 +1,23 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
{
private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer(
"@* Foo * Bar * Baz *",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken(" Foo * Bar * Baz *", CSharpTokenType.RazorComment));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz *"));
}
[Fact]
@ -24,11 +25,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo * Bar * Baz *@",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken(" Foo * Bar * Baz ", CSharpTokenType.RazorComment),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz "),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
}
[Fact]
@ -36,59 +37,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo Bar Baz *@",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken(" Foo Bar Baz ", CSharpTokenType.RazorComment),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo Bar Baz "),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{
TestTokenizer("// Foo Bar Baz", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment));
TestTokenizer("// Foo Bar Baz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"));
}
[Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline()
{
TestTokenizer("// Foo Bar Baz\na", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment), IgnoreRemaining);
TestTokenizer("// Foo Bar Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"), IgnoreRemaining);
}
[Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpToken("// Foo/*Bar*/ Baz", CSharpTokenType.Comment), IgnoreRemaining);
TestTokenizer("// Foo/*Bar*/ Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo/*Bar*/ Baz"), IgnoreRemaining);
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment));
TestTokenizer("/* Foo\nBar\nBaz */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"));
}
[Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo\nBar\nBaz */a", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"), IgnoreRemaining);
}
[Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{
TestTokenizer("/* Foo\nBar\nBaz", new CSharpToken("/* Foo\nBar\nBaz", CSharpTokenType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo\nBar\nBaz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz"), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpToken("/* Foo/*\nBar\nBaz*/", CSharpTokenType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo/*\nBar\nBaz*/"), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpToken("/* Foo\nBar\nBaz* */", CSharpTokenType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo\nBar\nBaz* */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz* */"), IgnoreRemaining);
}
}
}

27
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -10,73 +11,73 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact]
public void Simple_Identifier_Is_Recognized()
{
TestTokenizer("foo", new CSharpToken("foo", CSharpTokenType.Identifier));
TestTokenizer("foo", SyntaxFactory.Token(SyntaxKind.Identifier, "foo"));
}
[Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized()
{
TestTokenizer("_foo", new CSharpToken("_foo", CSharpTokenType.Identifier));
TestTokenizer("_foo", SyntaxFactory.Token(SyntaxKind.Identifier, "_foo"));
}
[Fact]
public void Identifier_Can_Contain_Digits()
{
TestTokenizer("foo4", new CSharpToken("foo4", CSharpTokenType.Identifier));
TestTokenizer("foo4", SyntaxFactory.Token(SyntaxKind.Identifier, "foo4"));
}
[Fact]
public void Identifier_Can_Start_With_Titlecase_Letter()
{
TestTokenizer("ῼfoo", new CSharpToken("ῼfoo", CSharpTokenType.Identifier));
TestTokenizer("ῼfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ῼfoo"));
}
[Fact]
public void Identifier_Can_Start_With_Letter_Modifier()
{
TestTokenizer("ᵊfoo", new CSharpToken("ᵊfoo", CSharpTokenType.Identifier));
TestTokenizer("ᵊfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ᵊfoo"));
}
[Fact]
public void Identifier_Can_Start_With_Other_Letter()
{
TestTokenizer("ƻfoo", new CSharpToken("ƻfoo", CSharpTokenType.Identifier));
TestTokenizer("ƻfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ƻfoo"));
}
[Fact]
public void Identifier_Can_Start_With_Number_Letter()
{
TestTokenizer("Ⅽool", new CSharpToken("Ⅽool", CSharpTokenType.Identifier));
TestTokenizer("Ⅽool", SyntaxFactory.Token(SyntaxKind.Identifier, "Ⅽool"));
}
[Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark()
{
TestTokenizer("foo\u0300", new CSharpToken("foo\u0300", CSharpTokenType.Identifier));
TestTokenizer("foo\u0300", SyntaxFactory.Token(SyntaxKind.Identifier, "foo\u0300"));
}
[Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark()
{
TestTokenizer("fooः", new CSharpToken("fooः", CSharpTokenType.Identifier));
TestTokenizer("fooः", SyntaxFactory.Token(SyntaxKind.Identifier, "fooः"));
}
[Fact]
public void Identifier_Can_Contain_Non_English_Digit()
{
TestTokenizer("foo١", new CSharpToken("foo١", CSharpTokenType.Identifier));
TestTokenizer("foo١", SyntaxFactory.Token(SyntaxKind.Identifier, "foo١"));
}
[Fact]
public void Identifier_Can_Contain_Connector_Punctuation()
{
TestTokenizer("foo‿bar", new CSharpToken("foo‿bar", CSharpTokenType.Identifier));
TestTokenizer("foo‿bar", SyntaxFactory.Token(SyntaxKind.Identifier, "foo‿bar"));
}
[Fact]
public void Identifier_Can_Contain_Format_Character()
{
TestTokenizer("foo؃bar", new CSharpToken("foo؃bar", CSharpTokenType.Identifier));
TestTokenizer("foo؃bar", SyntaxFactory.Token(SyntaxKind.Identifier, "foo؃bar"));
}
[Fact]
@ -164,7 +165,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType)
{
TestTokenizer(keyword, new CSharpToken(keyword, CSharpTokenType.Keyword) { Keyword = keywordType });
TestTokenizer(keyword, SyntaxFactory.Token(SyntaxKind.Keyword, keyword));
}
}
}

161
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
View File

@ -2,286 +2,287 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase
{
private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact]
public void Simple_Integer_Literal_Is_Recognized()
{
TestSingleToken("01189998819991197253", CSharpTokenType.IntegerLiteral);
TestSingleToken("01189998819991197253", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized()
{
TestSingleToken("42U", CSharpTokenType.IntegerLiteral);
TestSingleToken("42u", CSharpTokenType.IntegerLiteral);
TestSingleToken("42U", SyntaxKind.IntegerLiteral);
TestSingleToken("42u", SyntaxKind.IntegerLiteral);
TestSingleToken("42L", CSharpTokenType.IntegerLiteral);
TestSingleToken("42l", CSharpTokenType.IntegerLiteral);
TestSingleToken("42L", SyntaxKind.IntegerLiteral);
TestSingleToken("42l", SyntaxKind.IntegerLiteral);
TestSingleToken("42UL", CSharpTokenType.IntegerLiteral);
TestSingleToken("42Ul", CSharpTokenType.IntegerLiteral);
TestSingleToken("42UL", SyntaxKind.IntegerLiteral);
TestSingleToken("42Ul", SyntaxKind.IntegerLiteral);
TestSingleToken("42uL", CSharpTokenType.IntegerLiteral);
TestSingleToken("42ul", CSharpTokenType.IntegerLiteral);
TestSingleToken("42uL", SyntaxKind.IntegerLiteral);
TestSingleToken("42ul", SyntaxKind.IntegerLiteral);
TestSingleToken("42LU", CSharpTokenType.IntegerLiteral);
TestSingleToken("42Lu", CSharpTokenType.IntegerLiteral);
TestSingleToken("42LU", SyntaxKind.IntegerLiteral);
TestSingleToken("42Lu", SyntaxKind.IntegerLiteral);
TestSingleToken("42lU", CSharpTokenType.IntegerLiteral);
TestSingleToken("42lu", CSharpTokenType.IntegerLiteral);
TestSingleToken("42lU", SyntaxKind.IntegerLiteral);
TestSingleToken("42lu", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{
TestTokenizer("42a", new CSharpToken("42", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
TestTokenizer("42a", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "42"), IgnoreRemaining);
}
[Fact]
public void Simple_Hex_Literal_Is_Recognized()
{
TestSingleToken("0x0123456789ABCDEF", CSharpTokenType.IntegerLiteral);
TestSingleToken("0x0123456789ABCDEF", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal()
{
TestSingleToken("0xDEADBEEFU", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFu", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFU", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFu", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFL", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFl", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFL", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFl", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFUL", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFUl", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFUL", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFUl", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFuL", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFul", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFuL", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFul", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFLU", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFLu", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFLU", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFLu", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFlU", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFlu", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFlU", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFlu", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{
TestTokenizer("0xDEADBEEFz", new CSharpToken("0xDEADBEEF", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
TestTokenizer("0xDEADBEEFz", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "0xDEADBEEF"), IgnoreRemaining);
}
[Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{
TestTokenizer("3.a", new CSharpToken("3", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
TestTokenizer("3.a", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "3"), IgnoreRemaining);
}
[Fact]
public void Simple_Real_Literal_Is_Recognized()
{
TestTokenizer("3.14159", new CSharpToken("3.14159", CSharpTokenType.RealLiteral));
TestTokenizer("3.14159", SyntaxFactory.Token(SyntaxKind.RealLiteral, "3.14159"));
}
[Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{
TestTokenizer(".14159", new CSharpToken(".14159", CSharpTokenType.RealLiteral));
TestTokenizer(".14159", SyntaxFactory.Token(SyntaxKind.RealLiteral, ".14159"));
}
[Fact]
public void Integer_With_Real_Type_Suffix_Is_Recognized()
{
TestSingleToken("42F", CSharpTokenType.RealLiteral);
TestSingleToken("42f", CSharpTokenType.RealLiteral);
TestSingleToken("42D", CSharpTokenType.RealLiteral);
TestSingleToken("42d", CSharpTokenType.RealLiteral);
TestSingleToken("42M", CSharpTokenType.RealLiteral);
TestSingleToken("42m", CSharpTokenType.RealLiteral);
TestSingleToken("42F", SyntaxKind.RealLiteral);
TestSingleToken("42f", SyntaxKind.RealLiteral);
TestSingleToken("42D", SyntaxKind.RealLiteral);
TestSingleToken("42d", SyntaxKind.RealLiteral);
TestSingleToken("42M", SyntaxKind.RealLiteral);
TestSingleToken("42m", SyntaxKind.RealLiteral);
}
[Fact]
public void Integer_With_Exponent_Is_Recognized()
{
TestSingleToken("1e10", CSharpTokenType.RealLiteral);
TestSingleToken("1E10", CSharpTokenType.RealLiteral);
TestSingleToken("1e+10", CSharpTokenType.RealLiteral);
TestSingleToken("1E+10", CSharpTokenType.RealLiteral);
TestSingleToken("1e-10", CSharpTokenType.RealLiteral);
TestSingleToken("1E-10", CSharpTokenType.RealLiteral);
TestSingleToken("1e10", SyntaxKind.RealLiteral);
TestSingleToken("1E10", SyntaxKind.RealLiteral);
TestSingleToken("1e+10", SyntaxKind.RealLiteral);
TestSingleToken("1E+10", SyntaxKind.RealLiteral);
TestSingleToken("1e-10", SyntaxKind.RealLiteral);
TestSingleToken("1E-10", SyntaxKind.RealLiteral);
}
[Fact]
public void Real_Number_With_Type_Suffix_Is_Recognized()
{
TestSingleToken("3.14F", CSharpTokenType.RealLiteral);
TestSingleToken("3.14f", CSharpTokenType.RealLiteral);
TestSingleToken("3.14D", CSharpTokenType.RealLiteral);
TestSingleToken("3.14d", CSharpTokenType.RealLiteral);
TestSingleToken("3.14M", CSharpTokenType.RealLiteral);
TestSingleToken("3.14m", CSharpTokenType.RealLiteral);
TestSingleToken("3.14F", SyntaxKind.RealLiteral);
TestSingleToken("3.14f", SyntaxKind.RealLiteral);
TestSingleToken("3.14D", SyntaxKind.RealLiteral);
TestSingleToken("3.14d", SyntaxKind.RealLiteral);
TestSingleToken("3.14M", SyntaxKind.RealLiteral);
TestSingleToken("3.14m", SyntaxKind.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_Is_Recognized()
{
TestSingleToken("3.14E10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14e10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14E+10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14e+10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14E-10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14e-10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14E10", SyntaxKind.RealLiteral);
TestSingleToken("3.14e10", SyntaxKind.RealLiteral);
TestSingleToken("3.14E+10", SyntaxKind.RealLiteral);
TestSingleToken("3.14e+10", SyntaxKind.RealLiteral);
TestSingleToken("3.14E-10", SyntaxKind.RealLiteral);
TestSingleToken("3.14e-10", SyntaxKind.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized()
{
TestSingleToken("3.14E+10F", CSharpTokenType.RealLiteral);
TestSingleToken("3.14E+10F", SyntaxKind.RealLiteral);
}
[Fact]
public void Single_Character_Literal_Is_Recognized()
{
TestSingleToken("'f'", CSharpTokenType.CharacterLiteral);
TestSingleToken("'f'", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Multi_Character_Literal_Is_Recognized()
{
TestSingleToken("'foo'", CSharpTokenType.CharacterLiteral);
TestSingleToken("'foo'", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
TestSingleToken("'foo bar", CSharpTokenType.CharacterLiteral);
TestSingleToken("'foo bar", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Character_Literal_Not_Terminated_By_Escaped_Quote()
{
TestSingleToken("'foo\\'bar'", CSharpTokenType.CharacterLiteral);
TestSingleToken("'foo\\'bar'", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
TestTokenizer("'foo\n", new CSharpToken("'foo", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\n", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("'foo\\\n", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\\n", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer("'foo\\\nflarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\\nflarg", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("'foo\\" + Environment.NewLine, new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\" + Environment.NewLine, SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
TestTokenizer($"'foo\\{Environment.NewLine}flarg", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Allows_Escaped_Escape()
{
TestTokenizer("'foo\\\\'blah", new CSharpToken("'foo\\\\'", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\\\'blah", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\\\'"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Is_Recognized()
{
TestSingleToken("\"foo\"", CSharpTokenType.StringLiteral);
TestSingleToken("\"foo\"", SyntaxKind.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
TestSingleToken("\"foo bar", CSharpTokenType.StringLiteral);
TestSingleToken("\"foo bar", SyntaxKind.StringLiteral);
}
[Fact]
public void String_Literal_Not_Terminated_By_Escaped_Quote()
{
TestSingleToken("\"foo\\\"bar\"", CSharpTokenType.StringLiteral);
TestSingleToken("\"foo\\\"bar\"", SyntaxKind.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
TestTokenizer("\"foo\n", new CSharpToken("\"foo", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\n", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("\"foo\\\n", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\\n", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer("\"foo\\\nflarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\\nflarg", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\" + Environment.NewLine, SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Allows_Escaped_Escape()
{
TestTokenizer("\"foo\\\\\"blah", new CSharpToken("\"foo\\\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\\\\"blah", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\\\\""), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Can_Contain_Newlines()
{
TestSingleToken("@\"foo\nbar\nbaz\"", CSharpTokenType.StringLiteral);
TestSingleToken("@\"foo\nbar\nbaz\"", SyntaxKind.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote()
{
TestSingleToken("@\"foo\"\"bar\"", CSharpTokenType.StringLiteral);
TestSingleToken("@\"foo\"\"bar\"", SyntaxKind.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{
TestTokenizer("@\"foo\\\"bar\"", new CSharpToken("@\"foo\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
TestTokenizer("@\"foo\\\"bar\"", SyntaxFactory.Token(SyntaxKind.StringLiteral, "@\"foo\\\""), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_EOF()
{
TestSingleToken("@\"foo", CSharpTokenType.StringLiteral);
TestSingleToken("@\"foo", SyntaxKind.StringLiteral);
}
}
}

99
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -10,287 +11,287 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact]
public void LeftBrace_Is_Recognized()
{
TestSingleToken("{", CSharpTokenType.LeftBrace);
TestSingleToken("{", SyntaxKind.LeftBrace);
}
[Fact]
public void Plus_Is_Recognized()
{
TestSingleToken("+", CSharpTokenType.Plus);
TestSingleToken("+", SyntaxKind.Plus);
}
[Fact]
public void Assign_Is_Recognized()
{
TestSingleToken("=", CSharpTokenType.Assign);
TestSingleToken("=", SyntaxKind.Assign);
}
[Fact]
public void Arrow_Is_Recognized()
{
TestSingleToken("->", CSharpTokenType.Arrow);
TestSingleToken("->", SyntaxKind.Arrow);
}
[Fact]
public void AndAssign_Is_Recognized()
{
TestSingleToken("&=", CSharpTokenType.AndAssign);
TestSingleToken("&=", SyntaxKind.AndAssign);
}
[Fact]
public void RightBrace_Is_Recognized()
{
TestSingleToken("}", CSharpTokenType.RightBrace);
TestSingleToken("}", SyntaxKind.RightBrace);
}
[Fact]
public void Minus_Is_Recognized()
{
TestSingleToken("-", CSharpTokenType.Minus);
TestSingleToken("-", SyntaxKind.Minus);
}
[Fact]
public void LessThan_Is_Recognized()
{
TestSingleToken("<", CSharpTokenType.LessThan);
TestSingleToken("<", SyntaxKind.LessThan);
}
[Fact]
public void Equals_Is_Recognized()
{
TestSingleToken("==", CSharpTokenType.Equals);
TestSingleToken("==", SyntaxKind.Equals);
}
[Fact]
public void OrAssign_Is_Recognized()
{
TestSingleToken("|=", CSharpTokenType.OrAssign);
TestSingleToken("|=", SyntaxKind.OrAssign);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
TestSingleToken("[", CSharpTokenType.LeftBracket);
TestSingleToken("[", SyntaxKind.LeftBracket);
}
[Fact]
public void Star_Is_Recognized()
{
TestSingleToken("*", CSharpTokenType.Star);
TestSingleToken("*", SyntaxKind.Star);
}
[Fact]
public void GreaterThan_Is_Recognized()
{
TestSingleToken(">", CSharpTokenType.GreaterThan);
TestSingleToken(">", SyntaxKind.GreaterThan);
}
[Fact]
public void NotEqual_Is_Recognized()
{
TestSingleToken("!=", CSharpTokenType.NotEqual);
TestSingleToken("!=", SyntaxKind.NotEqual);
}
[Fact]
public void XorAssign_Is_Recognized()
{
TestSingleToken("^=", CSharpTokenType.XorAssign);
TestSingleToken("^=", SyntaxKind.XorAssign);
}
[Fact]
public void RightBracket_Is_Recognized()
{
TestSingleToken("]", CSharpTokenType.RightBracket);
TestSingleToken("]", SyntaxKind.RightBracket);
}
[Fact]
public void Slash_Is_Recognized()
{
TestSingleToken("/", CSharpTokenType.Slash);
TestSingleToken("/", SyntaxKind.Slash);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
TestSingleToken("?", CSharpTokenType.QuestionMark);
TestSingleToken("?", SyntaxKind.QuestionMark);
}
[Fact]
public void LessThanEqual_Is_Recognized()
{
TestSingleToken("<=", CSharpTokenType.LessThanEqual);
TestSingleToken("<=", SyntaxKind.LessThanEqual);
}
[Fact]
public void LeftShift_Is_Not_Specially_Recognized()
{
TestTokenizer("<<",
new CSharpToken("<", CSharpTokenType.LessThan),
new CSharpToken("<", CSharpTokenType.LessThan));
SyntaxFactory.Token(SyntaxKind.LessThan, "<"),
SyntaxFactory.Token(SyntaxKind.LessThan, "<"));
}
[Fact]
public void LeftParen_Is_Recognized()
{
TestSingleToken("(", CSharpTokenType.LeftParenthesis);
TestSingleToken("(", SyntaxKind.LeftParenthesis);
}
[Fact]
public void Modulo_Is_Recognized()
{
TestSingleToken("%", CSharpTokenType.Modulo);
TestSingleToken("%", SyntaxKind.Modulo);
}
[Fact]
public void NullCoalesce_Is_Recognized()
{
TestSingleToken("??", CSharpTokenType.NullCoalesce);
TestSingleToken("??", SyntaxKind.NullCoalesce);
}
[Fact]
public void GreaterThanEqual_Is_Recognized()
{
TestSingleToken(">=", CSharpTokenType.GreaterThanEqual);
TestSingleToken(">=", SyntaxKind.GreaterThanEqual);
}
[Fact]
public void EqualGreaterThan_Is_Recognized()
{
TestSingleToken("=>", CSharpTokenType.GreaterThanEqual);
TestSingleToken("=>", SyntaxKind.GreaterThanEqual);
}
[Fact]
public void RightParen_Is_Recognized()
{
TestSingleToken(")", CSharpTokenType.RightParenthesis);
TestSingleToken(")", SyntaxKind.RightParenthesis);
}
[Fact]
public void And_Is_Recognized()
{
TestSingleToken("&", CSharpTokenType.And);
TestSingleToken("&", SyntaxKind.And);
}
[Fact]
public void DoubleColon_Is_Recognized()
{
TestSingleToken("::", CSharpTokenType.DoubleColon);
TestSingleToken("::", SyntaxKind.DoubleColon);
}
[Fact]
public void PlusAssign_Is_Recognized()
{
TestSingleToken("+=", CSharpTokenType.PlusAssign);
TestSingleToken("+=", SyntaxKind.PlusAssign);
}
[Fact]
public void Semicolon_Is_Recognized()
{
TestSingleToken(";", CSharpTokenType.Semicolon);
TestSingleToken(";", SyntaxKind.Semicolon);
}
[Fact]
public void Tilde_Is_Recognized()
{
TestSingleToken("~", CSharpTokenType.Tilde);
TestSingleToken("~", SyntaxKind.Tilde);
}
[Fact]
public void DoubleOr_Is_Recognized()
{
TestSingleToken("||", CSharpTokenType.DoubleOr);
TestSingleToken("||", SyntaxKind.DoubleOr);
}
[Fact]
public void ModuloAssign_Is_Recognized()
{
TestSingleToken("%=", CSharpTokenType.ModuloAssign);
TestSingleToken("%=", SyntaxKind.ModuloAssign);
}
[Fact]
public void Colon_Is_Recognized()
{
TestSingleToken(":", CSharpTokenType.Colon);
TestSingleToken(":", SyntaxKind.Colon);
}
[Fact]
public void Not_Is_Recognized()
{
TestSingleToken("!", CSharpTokenType.Not);
TestSingleToken("!", SyntaxKind.Not);
}
[Fact]
public void DoubleAnd_Is_Recognized()
{
TestSingleToken("&&", CSharpTokenType.DoubleAnd);
TestSingleToken("&&", SyntaxKind.DoubleAnd);
}
[Fact]
public void DivideAssign_Is_Recognized()
{
TestSingleToken("/=", CSharpTokenType.DivideAssign);
TestSingleToken("/=", SyntaxKind.DivideAssign);
}
[Fact]
public void Comma_Is_Recognized()
{
TestSingleToken(",", CSharpTokenType.Comma);
TestSingleToken(",", SyntaxKind.Comma);
}
[Fact]
public void Xor_Is_Recognized()
{
TestSingleToken("^", CSharpTokenType.Xor);
TestSingleToken("^", SyntaxKind.Xor);
}
[Fact]
public void Decrement_Is_Recognized()
{
TestSingleToken("--", CSharpTokenType.Decrement);
TestSingleToken("--", SyntaxKind.Decrement);
}
[Fact]
public void MultiplyAssign_Is_Recognized()
{
TestSingleToken("*=", CSharpTokenType.MultiplyAssign);
TestSingleToken("*=", SyntaxKind.MultiplyAssign);
}
[Fact]
public void Dot_Is_Recognized()
{
TestSingleToken(".", CSharpTokenType.Dot);
TestSingleToken(".", SyntaxKind.Dot);
}
[Fact]
public void Or_Is_Recognized()
{
TestSingleToken("|", CSharpTokenType.Or);
TestSingleToken("|", SyntaxKind.Or);
}
[Fact]
public void Increment_Is_Recognized()
{
TestSingleToken("++", CSharpTokenType.Increment);
TestSingleToken("++", SyntaxKind.Increment);
}
[Fact]
public void MinusAssign_Is_Recognized()
{
TestSingleToken("-=", CSharpTokenType.MinusAssign);
TestSingleToken("-=", SyntaxKind.MinusAssign);
}
[Fact]
public void RightShift_Is_Not_Specially_Recognized()
{
TestTokenizer(">>",
new CSharpToken(">", CSharpTokenType.GreaterThan),
new CSharpToken(">", CSharpTokenType.GreaterThan));
SyntaxFactory.Token(SyntaxKind.GreaterThan, ">"),
SyntaxFactory.Token(SyntaxKind.GreaterThan, ">"));
}
[Fact]
public void Hash_Is_Recognized()
{
TestSingleToken("#", CSharpTokenType.Hash);
TestSingleToken("#", SyntaxKind.Hash);
}
}
}

39
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
View File

@ -1,13 +1,14 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerTest : CSharpTokenizerTestBase
{
private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
@ -20,8 +21,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\ra",
new CSharpToken("\r", CSharpTokenType.NewLine),
new CSharpToken("\r", CSharpTokenType.NewLine),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
IgnoreRemaining);
}
@ -30,8 +31,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\n\na",
new CSharpToken("\n", CSharpTokenType.NewLine),
new CSharpToken("\n", CSharpTokenType.NewLine),
SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
IgnoreRemaining);
}
@ -41,8 +42,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// NEL: Unicode "Next Line" U+0085
TestTokenizer(
"\u0085\u0085a",
new CSharpToken("\u0085", CSharpTokenType.NewLine),
new CSharpToken("\u0085", CSharpTokenType.NewLine),
SyntaxFactory.Token(SyntaxKind.NewLine, "\u0085"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\u0085"),
IgnoreRemaining);
}
@ -52,8 +53,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Line Separator" U+2028
TestTokenizer(
"\u2028\u2028a",
new CSharpToken("\u2028", CSharpTokenType.NewLine),
new CSharpToken("\u2028", CSharpTokenType.NewLine),
SyntaxFactory.Token(SyntaxKind.NewLine, "\u2028"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\u2028"),
IgnoreRemaining);
}
@ -63,8 +64,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Paragraph Separator" U+2029
TestTokenizer(
"\u2029\u2029a",
new CSharpToken("\u2029", CSharpTokenType.NewLine),
new CSharpToken("\u2029", CSharpTokenType.NewLine),
SyntaxFactory.Token(SyntaxKind.NewLine, "\u2029"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\u2029"),
IgnoreRemaining);
}
@ -73,8 +74,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n\r\na",
new CSharpToken("\r\n", CSharpTokenType.NewLine),
new CSharpToken("\r\n", CSharpTokenType.NewLine),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
IgnoreRemaining);
}
@ -83,15 +84,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
" \f\t\u000B \n ",
new CSharpToken(" \f\t\u000B ", CSharpTokenType.WhiteSpace),
new CSharpToken("\n", CSharpTokenType.NewLine),
new CSharpToken(" ", CSharpTokenType.WhiteSpace));
SyntaxFactory.Token(SyntaxKind.Whitespace, " \f\t\u000B "),
SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "));
}
[Fact]
public void Transition_Is_Recognized()
{
TestSingleToken("@", CSharpTokenType.Transition);
TestSingleToken("@", SyntaxKind.Transition);
}
[Fact]
@ -99,8 +100,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@(",
new CSharpToken("@", CSharpTokenType.Transition),
new CSharpToken("(", CSharpTokenType.LeftParenthesis));
SyntaxFactory.Token(SyntaxKind.Transition, "@"),
SyntaxFactory.Token(SyntaxKind.LeftParenthesis, "("));
}
}
}

13
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
View File

@ -1,11 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{
private static CSharpToken _ignoreRemaining = new CSharpToken(string.Empty, CSharpTokenType.Unknown);
private static SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
internal override object IgnoreRemaining
{
@ -17,14 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source);
}
internal void TestSingleToken(string text, CSharpTokenType expectedTokenType)
internal void TestSingleToken(string text, SyntaxKind expectedTokenKind)
{
TestTokenizer(text, new CSharpToken(text, expectedTokenType));
}
internal void TestTokenizer(string input, params CSharpToken[] expectedTokens)
{
base.TestTokenizer<CSharpToken, CSharpTokenType>(input, expectedTokens);
TestTokenizer(text, SyntaxFactory.Token(expectedTokenKind, text));
}
}
}

37
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -12,12 +13,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n @something \r\n @this is ignored",
new CSharpToken("\r\n", CSharpTokenType.NewLine),
new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpToken("@", CSharpTokenType.Transition),
new CSharpToken("something", CSharpTokenType.Identifier),
new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpToken("\r\n", CSharpTokenType.NewLine));
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.Transition, "@"),
SyntaxFactory.Token(SyntaxKind.Identifier, "something"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
}
[Fact]
@ -25,18 +26,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@*included*@\r\n @something \"value\"\r\n @this is ignored",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken("included", CSharpTokenType.RazorComment),
new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpToken("\r\n", CSharpTokenType.NewLine),
new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpToken("@", CSharpTokenType.Transition),
new CSharpToken("something", CSharpTokenType.Identifier),
new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpToken("\"value\"", CSharpTokenType.StringLiteral),
new CSharpToken("\r\n", CSharpTokenType.NewLine));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, "included"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.Transition, "@"),
SyntaxFactory.Token(SyntaxKind.Identifier, "something"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"value\""),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
}
internal override object CreateTokenizer(ITextDocument source)

25
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -12,9 +13,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n <div>Ignored</div>",
new HtmlToken("\r\n", HtmlTokenType.NewLine),
new HtmlToken(" ", HtmlTokenType.WhiteSpace),
new HtmlToken("<", HtmlTokenType.OpenAngle));
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
}
[Fact]
@ -22,15 +23,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n @*included*@ <div>Ignored</div>",
new HtmlToken("\r\n", HtmlTokenType.NewLine),
new HtmlToken(" ", HtmlTokenType.WhiteSpace),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken("included", HtmlTokenType.RazorComment),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlToken(" ", HtmlTokenType.WhiteSpace),
new HtmlToken("<", HtmlTokenType.OpenAngle));
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, "included"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
}
internal override object CreateTokenizer(ITextDocument source)

48
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
View File

@ -2,23 +2,24 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
{
public class HtmlMarkupParserTests
{
private static readonly HtmlToken doubleHyphenToken = new HtmlToken("--", HtmlTokenType.DoubleHyphen);
private static readonly SyntaxToken doubleHyphenToken = SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--");
public static IEnumerable<object[]> NonDashTokens
{
get
{
yield return new[] { new HtmlToken("--", HtmlTokenType.DoubleHyphen) };
yield return new[] { new HtmlToken("asdf", HtmlTokenType.Text) };
yield return new[] { new HtmlToken(">", HtmlTokenType.CloseAngle) };
yield return new[] { new HtmlToken("<", HtmlTokenType.OpenAngle) };
yield return new[] { new HtmlToken("!", HtmlTokenType.Bang) };
yield return new[] { SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--") };
yield return new[] { SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf") };
yield return new[] { SyntaxFactory.Token(SyntaxKind.CloseAngle, ">") };
yield return new[] { SyntaxFactory.Token(SyntaxKind.OpenAngle, "<") };
yield return new[] { SyntaxFactory.Token(SyntaxKind.Bang, "!") };
}
}
@ -27,7 +28,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsHyphen_ReturnsFalseForNonDashToken(object token)
{
// Arrange
var convertedToken = (HtmlToken)token;
var convertedToken = (SyntaxToken)token;
// Act & Assert
Assert.False(HtmlMarkupParser.IsHyphen(convertedToken));
@ -37,7 +38,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsHyphen_ReturnsTrueForADashToken()
{
// Arrange
var dashToken = new HtmlToken("-", HtmlTokenType.Text);
var dashToken = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "-");
// Act & Assert
Assert.True(HtmlMarkupParser.IsHyphen(dashToken));
@ -53,9 +54,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
var token = sut.AcceptAllButLastDoubleHyphens();
// Assert
Assert.Equal(doubleHyphenToken, token);
Assert.True(sut.At(HtmlTokenType.CloseAngle));
Assert.Equal(doubleHyphenToken, sut.PreviousToken);
Assert.True(doubleHyphenToken.IsEquivalentTo(token));
Assert.True(sut.At(SyntaxKind.CloseAngle));
Assert.True(doubleHyphenToken.IsEquivalentTo(sut.PreviousToken));
}
[Fact]
@ -68,8 +69,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
var token = sut.AcceptAllButLastDoubleHyphens();
// Assert
Assert.Equal(doubleHyphenToken, token);
Assert.True(sut.At(HtmlTokenType.CloseAngle));
Assert.True(doubleHyphenToken.IsEquivalentTo(token));
Assert.True(sut.At(SyntaxKind.CloseAngle));
Assert.True(HtmlMarkupParser.IsHyphen(sut.PreviousToken));
}
@ -157,8 +158,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsFalseForAllowedContent()
{
// Arrange
var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text);
var sequence = Enumerable.Range((int)'a', 26).Select(item => new HtmlToken(((char)item).ToString(), HtmlTokenType.Text));
var expectedToken1 = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "a");
var sequence = Enumerable.Range((int)'a', 26).Select(item => SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, ((char)item).ToString()));
// Act & Assert
Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -168,8 +169,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsTrueForDisallowedContent()
{
// Arrange
var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text);
var sequence = new[] { new HtmlToken("<", HtmlTokenType.OpenAngle), new HtmlToken("!", HtmlTokenType.Bang), new HtmlToken("-", HtmlTokenType.Text) };
var expectedToken1 = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "a");
var sequence = new[]
{
SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"),
SyntaxFactory.Token(SyntaxKind.Bang, "!"),
SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "-")
};
// Act & Assert
Assert.True(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -179,8 +185,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsFalseForEmptyContent()
{
// Arrange
var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text);
var sequence = Array.Empty<HtmlToken>();
var expectedToken1 = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "a");
var sequence = Array.Empty<SyntaxToken>();
// Act & Assert
Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -188,7 +194,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
private class TestHtmlMarkupParser : HtmlMarkupParser
{
public new HtmlToken PreviousToken
public new SyntaxToken PreviousToken
{
get => base.PreviousToken;
}
@ -203,7 +209,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
this.EnsureCurrent();
}
public new HtmlToken AcceptAllButLastDoubleHyphens()
public new SyntaxToken AcceptAllButLastDoubleHyphens()
{
return base.AcceptAllButLastDoubleHyphens();
}

67
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlTokenizerTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -17,113 +18,113 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void Text_Is_Recognized()
{
TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384",
new HtmlToken("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlTokenType.Text));
SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "foo-9309&smlkmb;::-3029022,.sdkq92384"));
}
[Fact]
public void Whitespace_Is_Recognized()
{
TestTokenizer(" \t\f ",
new HtmlToken(" \t\f ", HtmlTokenType.WhiteSpace));
SyntaxFactory.Token(SyntaxKind.Whitespace, " \t\f "));
}
[Fact]
public void Newline_Is_Recognized()
{
TestTokenizer("\n\r\r\n",
new HtmlToken("\n", HtmlTokenType.NewLine),
new HtmlToken("\r", HtmlTokenType.NewLine),
new HtmlToken("\r\n", HtmlTokenType.NewLine));
SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
}
[Fact]
public void Transition_Is_Not_Recognized_Mid_Text_If_Surrounded_By_Alphanumeric_Characters()
{
TestSingleToken("foo@bar", HtmlTokenType.Text);
TestSingleToken("foo@bar", SyntaxKind.HtmlTextLiteral);
}
[Fact]
public void OpenAngle_Is_Recognized()
{
TestSingleToken("<", HtmlTokenType.OpenAngle);
TestSingleToken("<", SyntaxKind.OpenAngle);
}
[Fact]
public void Bang_Is_Recognized()
{
TestSingleToken("!", HtmlTokenType.Bang);
TestSingleToken("!", SyntaxKind.Bang);
}
[Fact]
public void Solidus_Is_Recognized()
{
TestSingleToken("/", HtmlTokenType.ForwardSlash);
TestSingleToken("/", SyntaxKind.ForwardSlash);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
TestSingleToken("?", HtmlTokenType.QuestionMark);
TestSingleToken("?", SyntaxKind.QuestionMark);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
TestSingleToken("[", HtmlTokenType.LeftBracket);
TestSingleToken("[", SyntaxKind.LeftBracket);
}
[Fact]
public void CloseAngle_Is_Recognized()
{
TestSingleToken(">", HtmlTokenType.CloseAngle);
TestSingleToken(">", SyntaxKind.CloseAngle);
}
[Fact]
public void RightBracket_Is_Recognized()
{
TestSingleToken("]", HtmlTokenType.RightBracket);
TestSingleToken("]", SyntaxKind.RightBracket);
}
[Fact]
public void Equals_Is_Recognized()
{
TestSingleToken("=", HtmlTokenType.Equals);
TestSingleToken("=", SyntaxKind.Equals);
}
[Fact]
public void DoubleQuote_Is_Recognized()
{
TestSingleToken("\"", HtmlTokenType.DoubleQuote);
TestSingleToken("\"", SyntaxKind.DoubleQuote);
}
[Fact]
public void SingleQuote_Is_Recognized()
{
TestSingleToken("'", HtmlTokenType.SingleQuote);
TestSingleToken("'", SyntaxKind.SingleQuote);
}
[Fact]
public void Transition_Is_Recognized()
{
TestSingleToken("@", HtmlTokenType.Transition);
TestSingleToken("@", SyntaxKind.Transition);
}
[Fact]
public void DoubleHyphen_Is_Recognized()
{
TestSingleToken("--", HtmlTokenType.DoubleHyphen);
TestSingleToken("--", SyntaxKind.DoubleHyphen);
}
[Fact]
public void SingleHyphen_Is_Not_Recognized()
{
TestSingleToken("-", HtmlTokenType.Text);
TestSingleToken("-", SyntaxKind.HtmlTextLiteral);
}
[Fact]
public void SingleHyphen_Mid_Text_Is_Not_Recognized_As_Separate_Token()
{
TestSingleToken("foo-bar", HtmlTokenType.Text);
TestSingleToken("foo-bar", SyntaxKind.HtmlTextLiteral);
}
[Fact]
@ -131,9 +132,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo * Bar * Baz *",
new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken(" Foo * Bar * Baz *", HtmlTokenType.RazorComment));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz *"));
}
[Fact]
@ -141,11 +142,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo * Bar * Baz *@",
new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken(" Foo * Bar * Baz ", HtmlTokenType.RazorComment),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz "),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
}
[Fact]
@ -153,11 +154,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo Bar Baz *@",
new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken(" Foo Bar Baz ", HtmlTokenType.RazorComment),
new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition));
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo Bar Baz "),
SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
}
}
}

13
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlTokenizerTestBase.cs
View File

@ -1,11 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public abstract class HtmlTokenizerTestBase : TokenizerTestBase
{
private static HtmlToken _ignoreRemaining = new HtmlToken(string.Empty, HtmlTokenType.Unknown);
private static SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
internal override object IgnoreRemaining
{
@ -17,14 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source);
}
internal void TestSingleToken(string text, HtmlTokenType expectedTokenType)
internal void TestSingleToken(string text, SyntaxKind expectedTokenKind)
{
TestTokenizer(text, new HtmlToken(text, expectedTokenType));
}
internal void TestTokenizer(string input, params HtmlToken[] expectedTokens)
{
base.TestTokenizer<HtmlToken, HtmlTokenType>(input, expectedTokens);
TestTokenizer(text, SyntaxFactory.Token(expectedTokenKind, text));
}
}
}

24
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ImplicitExpressionEditHandlerTest.cs
View File

@ -3,6 +3,7 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -159,7 +160,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_SingleLeftParenthesis_CountsCorrectly()
{
// Arrange
var token = new CSharpToken("(", CSharpTokenType.LeftParenthesis);
var token = SyntaxFactory.Token(SyntaxKind.LeftParenthesis, "(");
var count = 0;
// Act
@ -174,7 +175,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_SingleRightParenthesis_CountsCorrectly()
{
// Arrange
var token = new CSharpToken(")", CSharpTokenType.RightParenthesis);
var token = SyntaxFactory.Token(SyntaxKind.RightParenthesis, ")");
var count = 2;
// Act
@ -189,7 +190,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_IncompleteStringLiteral_CountsCorrectly()
{
// Arrange
var token = new CSharpToken("\"((", CSharpTokenType.StringLiteral);
var token = SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"((");
var count = 2;
// Act
@ -204,7 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_IncompleteCharacterLiteral_CountsCorrectly()
{
// Arrange
var token = new CSharpToken("'((", CSharpTokenType.CharacterLiteral);
var token = SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'((");
var count = 2;
// Act
@ -219,7 +220,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_CompleteStringLiteral_CountsCorrectly()
{
// Arrange
var token = new CSharpToken("\"((\"", CSharpTokenType.StringLiteral);
var token = SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"((\"");
var count = 2;
// Act
@ -234,7 +235,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_CompleteCharacterLiteral_CountsCorrectly()
{
// Arrange
var token = new CSharpToken("'('", CSharpTokenType.CharacterLiteral);
var token = SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'('");
var count = 2;
// Act
@ -249,7 +250,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesis_ReturnsFalse()
{
// Arrange
var token = new CSharpToken(")", CSharpTokenType.RightParenthesis);
var token = SyntaxFactory.Token(SyntaxKind.RightParenthesis, ")");
var count = 0;
// Act
@ -264,7 +265,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesisStringLiteral_ReturnsFalse()
{
// Arrange
var token = new CSharpToken("\")", CSharpTokenType.StringLiteral);
var token = SyntaxFactory.Token(SyntaxKind.StringLiteral, "\")");
var count = 0;
// Act
@ -279,7 +280,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesisCharacterLiteral_ReturnsFalse()
{
// Arrange
var token = new CSharpToken("')", CSharpTokenType.CharacterLiteral);
var token = SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "')");
var count = 0;
// Act
@ -453,11 +454,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return span;
}
private static IReadOnlyList<CSharpToken> GetTokens(SourceLocation start, string content)
private static IReadOnlyList<SyntaxToken> GetTokens(SourceLocation start, string content)
{
var parent = GetSpan(start, content);
var tokens = parent.Tokens.Cast<CSharpToken>().ToArray();
return tokens;
return parent.Tokens;
}
}
}

11
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/SpanTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -12,10 +13,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Arrange
var builder = new SpanBuilder(SourceLocation.Zero);
builder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
builder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = builder.Build();
var newBuilder = new SpanBuilder(SourceLocation.Zero);
newBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
newBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
var originalLength = span.Length;
// Act
@ -33,7 +34,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder()
{
@ -44,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.Parent = block;
var originalBlockLength = block.Length;
var newSpanBuilder = new SpanBuilder(SourceLocation.Zero);
newSpanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
newSpanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
// Act
span.ReplaceWith(newSpanBuilder);
@ -64,7 +65,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind = SpanKindInternal.Transition,
ChunkGenerator = new ExpressionChunkGenerator(),
};
spanBuilder.Accept(new CSharpToken("@", CSharpTokenType.Transition));
spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.Transition, "@"));
var span = spanBuilder.Build();
// Act

50
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/TokenizerLookaheadTest.cs
View File

@ -5,6 +5,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -64,7 +65,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Act
var i = 3;
IEnumerable<HtmlToken> previousTokens = null;
IEnumerable<SyntaxToken> previousTokens = null;
var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{
previousTokens = p;
@ -77,9 +78,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// For the very first element, there will be no previous items, so null is expected
var orderIndex = 0;
Assert.Null(previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlToken("fvd", HtmlTokenType.Text), previousTokens.ElementAt(orderIndex++));
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf"), previousTokens.ElementAt(orderIndex++));
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--"), previousTokens.ElementAt(orderIndex++));
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "fvd"), previousTokens.ElementAt(orderIndex++));
}
[Fact]
@ -89,7 +90,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokenizer = CreateContentTokenizer("asdf--fvd");
// Act
var tokens = new Stack<HtmlToken>();
var tokens = new Stack<SyntaxToken>();
var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{
tokens.Push(s);
@ -99,9 +100,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Assert
Assert.False(tokenFound);
Assert.Equal(3, tokens.Count);
Assert.Equal(new HtmlToken("fvd", HtmlTokenType.Text), tokens.Pop());
Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), tokens.Pop());
Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), tokens.Pop());
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "fvd"), tokens.Pop());
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--"), tokens.Pop());
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf"), tokens.Pop());
}
[Fact]
@ -111,18 +112,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokenizer = CreateContentTokenizer("asdf--fvd");
// Act
var tokens = new Stack<HtmlToken>();
var tokens = new Stack<SyntaxToken>();
var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{
tokens.Push(s);
return s.Type == HtmlTokenType.DoubleHyphen;
return s.Kind == SyntaxKind.DoubleHyphen;
});
// Assert
Assert.True(tokenFound);
Assert.Equal(2, tokens.Count);
Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), tokens.Pop());
Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), tokens.Pop());
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--"), tokens.Pop());
AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf"), tokens.Pop());
}
private static TestTokenizerBackedParser CreateContentTokenizer(string content)
@ -135,7 +136,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return tokenizer;
}
private class ExposedTokenizer : Tokenizer<CSharpToken, CSharpTokenType>
private static void AssertTokenEqual(SyntaxToken expected, SyntaxToken actual)
{
Assert.True(expected.IsEquivalentTo(actual), "Tokens not equal.");
}
private class ExposedTokenizer : Tokenizer
{
public ExposedTokenizer(string input)
: base(new SeekableTextReader(input, filePath: null))
@ -150,7 +156,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
public override CSharpTokenType RazorCommentStarType
public override SyntaxKind RazorCommentStarKind
{
get
{
@ -158,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
public override CSharpTokenType RazorCommentTransitionType
public override SyntaxKind RazorCommentTransitionKind
{
get
{
@ -166,7 +172,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
public override CSharpTokenType RazorCommentType
public override SyntaxKind RazorCommentKind
{
get
{
@ -182,9 +188,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
protected override CSharpToken CreateToken(
protected override SyntaxToken CreateToken(
string content,
CSharpTokenType type,
SyntaxKind type,
IReadOnlyList<RazorDiagnostic> errors)
{
throw new NotImplementedException();
@ -196,9 +202,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
private class TestTokenizerBackedParser : TokenizerBackedParser<HtmlTokenizer, HtmlToken, HtmlTokenType>
private class TestTokenizerBackedParser : TokenizerBackedParser<HtmlTokenizer>
{
internal TestTokenizerBackedParser(LanguageCharacteristics<HtmlTokenizer, HtmlToken, HtmlTokenType> language, ParserContext context) : base(language, context)
internal TestTokenizerBackedParser(LanguageCharacteristics<HtmlTokenizer> language, ParserContext context) : base(language, context)
{
}
@ -207,12 +213,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
throw new NotImplementedException();
}
protected override bool TokenTypeEquals(HtmlTokenType x, HtmlTokenType y)
protected override bool TokenKindEquals(SyntaxKind x, SyntaxKind y)
{
throw new NotImplementedException();
}
internal new bool LookaheadUntil(Func<HtmlToken, IEnumerable<HtmlToken>, bool> condition)
internal new bool LookaheadUntil(Func<SyntaxToken, IEnumerable<SyntaxToken>, bool> condition)
{
return base.LookaheadUntil(condition);
}

13
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/TokenizerTestBase.cs
View File

@ -4,6 +4,7 @@
using System;
using System.Diagnostics;
using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -13,18 +14,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal abstract object IgnoreRemaining { get; }
internal abstract object CreateTokenizer(ITextDocument source);
internal void TestTokenizer<TSymbol, TSymbolType>(string input, params TSymbol[] expectedSymbols)
where TSymbolType : struct
where TSymbol : TokenBase<TSymbolType>
internal void TestTokenizer(string input, params SyntaxToken[] expectedSymbols)
{
// Arrange
var success = true;
var output = new StringBuilder();
using (var source = new SeekableTextReader(input, filePath: null))
{
var tokenizer = (Tokenizer<TSymbol, TSymbolType>)CreateTokenizer(source);
var tokenizer = (Tokenizer)CreateTokenizer(source);
var counter = 0;
TSymbol current = null;
SyntaxToken current = null;
while ((current = tokenizer.NextToken()) != null)
{
if (counter >= expectedSymbols.Length)
@ -34,11 +33,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
output.AppendLine(string.Format("P: Ignored {0}", current));
output.AppendLine(string.Format("P: Ignored |{0}|", current));
}
else
{
if (!Equals(expectedSymbols[counter], current))
if (!expectedSymbols[counter].IsEquivalentTo(current))
{
output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
success = false;

21
test/Microsoft.AspNetCore.Razor.Language.Test/SourceChangeTest.cs
View File

@ -3,6 +3,7 @@
using System;
using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language
@ -104,8 +105,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{
// Arrange
var builder = new SpanBuilder(new SourceLocation(0, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(0, 0, 0), "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(7, 0, 7), "World"));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder);
@ -123,8 +124,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{
// Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder);
@ -142,8 +143,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{
// Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder);
@ -161,8 +162,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{
// Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder);
@ -180,8 +181,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{
// Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder);

8
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/FunctionsDirectiveAutoCompleteAtEOF.stree.txt
View File

@ -1,9 +1,9 @@
Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 11 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[functions];
SyntaxKind.Identifier;[functions];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:1
CSharpTokenType.Unknown;[];
SyntaxKind.Unknown;[];

10
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/FunctionsDirectiveAutoCompleteAtStartOfFile.stree.txt
View File

@ -1,10 +1,10 @@
Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 16 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[functions];
SyntaxKind.Identifier;[functions];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [LFfoo] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:2
CSharpTokenType.NewLine;[LF];
CSharpTokenType.Identifier;[foo];
SyntaxKind.NewLine;[LF];
SyntaxKind.Identifier;[foo];

14
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/SectionDirectiveAutoCompleteAtEOF.stree.txt
View File

@ -1,16 +1,16 @@
Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 17 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[section];
SyntaxKind.Identifier;[section];
Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1
CSharpTokenType.Identifier;[Header];
SyntaxKind.Identifier;[Header];
Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Markup block - Gen<None> - 0 - (17:0,17)
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
HtmlTokenType.Unknown;[];
SyntaxKind.Unknown;[];

30
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/SectionDirectiveAutoCompleteAtStartOfFile.stree.txt
View File

@ -1,29 +1,29 @@
Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 29 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[section];
SyntaxKind.Identifier;[section];
Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1
CSharpTokenType.Identifier;[Header];
SyntaxKind.Identifier;[Header];
Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Markup block - Gen<None> - 12 - (17:0,17)
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
HtmlTokenType.NewLine;[LF];
SyntaxKind.NewLine;[LF];
Tag block - Gen<None> - 3 - (19:1,0)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:Any - (19:1,0) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[p];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[p];
SyntaxKind.CloseAngle;[>];
SyntaxKind.HtmlText - [Foo] - [22..25) - FullWidth: 3 - Slots: 1
SyntaxKind.HtmlTextLiteralToken;[Foo];
SyntaxKind.HtmlTextLiteral;[Foo];
Tag block - Gen<None> - 4 - (25:1,6)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:Any - (25:1,6) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[p];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[p];
SyntaxKind.CloseAngle;[>];

6
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/VerbatimBlockAutoCompleteAtEOF.stree.txt
View File

@ -1,7 +1,7 @@
Statement block - Gen<None> - 2 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1
CSharpTokenType.Unknown;[];
SyntaxKind.Unknown;[];

22
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/VerbatimBlockAutoCompleteAtStartOfFile.stree.txt
View File

@ -1,21 +1,21 @@
Statement block - Gen<None> - 11 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [LF] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1
CSharpTokenType.NewLine;[LF];
SyntaxKind.NewLine;[LF];
Markup block - Gen<None> - 7 - (4:1,0)
Tag block - Gen<None> - 3 - (4:1,0)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (4:1,0) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[p];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[p];
SyntaxKind.CloseAngle;[>];
Tag block - Gen<None> - 4 - (7:1,3)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (7:1,3) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[p];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[p];
SyntaxKind.CloseAngle;[>];
Code span - Gen<Stmt> - [] - SpanEditHandler;Accepts:Any - (11:1,7) - Tokens:1
CSharpTokenType.Unknown;[];
SyntaxKind.Unknown;[];

116
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/AcceptsElseIfWithNoCondition.stree.txt
View File

@ -1,60 +1,60 @@
Statement block - Gen<None> - 106 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if { foo(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:58
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"foo } bar"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[else];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

24
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/AcceptsTrailingDotIntoImplicitExpressionWhenEmbeddedInCode.stree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 17 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
Expression block - Gen<Expr> - 5 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [foo.] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:2
CSharpTokenType.Identifier;[foo];
CSharpTokenType.Dot;[.];
SyntaxKind.Identifier;[foo];
SyntaxKind.Dot;[.];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (15:0,15) - Tokens:2
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

16
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/AllowsEmptyBlockStatement.stree.txt
View File

@ -1,10 +1,10 @@
Statement block - Gen<None> - 13 - (0:0,0)
Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[false];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[false];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/BalancingBracketsIgnoresStringLiteralCharactersAndBrackets.stree.txt
View File

@ -1,19 +1,19 @@
Statement block - Gen<None> - 47 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) {LF // bar } " baz 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:17
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Comment;[// bar } " baz '];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[zoop];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.CSharpComment;[// bar } " baz '];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[zoop];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];

38
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/BalancingBracketsIgnoresStringLiteralCharactersAndBracketsInsideBlockComments.stree.txt
View File

@ -1,21 +1,21 @@
Statement block - Gen<None> - 54 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) {LF /* bar } " */ ' baz } 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:19
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Comment;[/* bar } " */];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.CharacterLiteral;[' baz } '];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[zoop];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.CSharpComment;[/* bar } " */];
SyntaxKind.Whitespace;[ ];
SyntaxKind.CharacterLiteral;[' baz } '];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[zoop];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];

8
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CapturesNewlineAfterUsing.stree.txt
View File

@ -1,6 +1,6 @@
Directive block - Gen<None> - 11 - (0:0,0)
Code span - Gen<Import: Foo;> - [using FooLF] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:4
CSharpTokenType.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.NewLine;[LF];
SyntaxKind.Keyword;[using];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.NewLine;[LF];

48
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlock.stree.txt
View File

@ -1,26 +1,26 @@
Statement block - Gen<None> - 40 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar);] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:24
CSharpTokenType.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[while];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.NotEqual;[!=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
SyntaxKind.Keyword;[do];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[while];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.NotEqual;[!=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];

46
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingSemicolon.stree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 39 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar)] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:23
CSharpTokenType.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[while];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.NotEqual;[!=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.RightParenthesis;[)];
SyntaxKind.Keyword;[do];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[while];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.NotEqual;[!=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.RightParenthesis;[)];

28
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingWhileClauseEntirely.stree.txt
View File

@ -1,16 +1,16 @@
Statement block - Gen<None> - 21 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:14
CSharpTokenType.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[do];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

32
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingWhileCondition.stree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 27 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:16
CSharpTokenType.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[while];
SyntaxKind.Keyword;[do];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[while];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingWhileConditionWithSemicolon.stree.txt
View File

@ -1,19 +1,19 @@
Statement block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while;] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:17
CSharpTokenType.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[while];
CSharpTokenType.Semicolon;[;];
SyntaxKind.Keyword;[do];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[while];
SyntaxKind.Semicolon;[;];

76
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesMarkupInDoWhileBlock.stree.txt
View File

@ -1,50 +1,50 @@
Statement block - Gen<None> - 58 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [do { var foo = bar;] - SpanEditHandler;Accepts:Any - (1:0,1) - Tokens:12
CSharpTokenType.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.Semicolon;[;];
SyntaxKind.Keyword;[do];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
SyntaxKind.Semicolon;[;];
Markup block - Gen<None> - 12 - (20:0,20)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (20:0,20) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 3 - (21:0,21)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[p];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[p];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1
HtmlTokenType.Text;[Foo];
SyntaxKind.HtmlTextLiteral;[Foo];
Tag block - Gen<None> - 4 - (27:0,27)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (27:0,27) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[p];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[p];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (31:0,31) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Code span - Gen<Stmt> - [foo++; } while (foo<bar>);] - SpanEditHandler;Accepts:None - (32:0,32) - Tokens:15
CSharpTokenType.Identifier;[foo];
CSharpTokenType.Increment;[++];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[while];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.LessThan;[<];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.GreaterThan;[>];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
SyntaxKind.Identifier;[foo];
SyntaxKind.Increment;[++];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[while];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.LessThan;[<];
SyntaxKind.Identifier;[bar];
SyntaxKind.GreaterThan;[>];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];

70
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/DoesNotAllowMultipleFinallyBlocks.stree.txt
View File

@ -1,37 +1,37 @@
Statement block - Gen<None> - 55 - (0:0,0)
Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:35
CSharpTokenType.Keyword;[try];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[finally];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[try];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[finally];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

30
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/DoesNotParseOnSwitchCharacterNotFollowedByOpenAngleOrColon.stree.txt
View File

@ -1,17 +1,17 @@
Statement block - Gen<None> - 30 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { @"Foo".ToString(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;[@"Foo"];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[ToString];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;[@"Foo"];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[ToString];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

6
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/DoesntCaptureWhitespaceAfterUsing.stree.txt
View File

@ -1,5 +1,5 @@
Directive block - Gen<None> - 9 - (0:0,0)
Code span - Gen<Import: Foo;> - [using Foo] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:3
CSharpTokenType.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
SyntaxKind.Keyword;[using];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];

22
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/HasErrorsIfNamespaceAliasMissingSemicolon.stree.txt
View File

@ -1,13 +1,13 @@
Directive block - Gen<None> - 29 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz = FooBarBaz;> - [using Foo.Bar.Baz = FooBarBaz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:11
CSharpTokenType.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Baz];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[FooBarBaz];
SyntaxKind.Keyword;[using];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Baz];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[FooBarBaz];

14
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/HasErrorsIfNamespaceImportMissingSemicolon.stree.txt
View File

@ -1,9 +1,9 @@
Directive block - Gen<None> - 17 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Baz];
SyntaxKind.Keyword;[using];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Baz];

38
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithAtDoesntCauseError.stree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpTokenType.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[true];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[true];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
Statement block - Gen<None> - 14 - (12:0,12)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:8
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[false];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[false];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (26:0,26) - Tokens:2
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

74
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithCSharpAt.stree.txt
View File

@ -1,41 +1,41 @@
Statement block - Gen<None> - 49 - (0:0,0)
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.LeftBrace;[{];
SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [ if (true) { var val = @x; if (val != 3) { } } ] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[<null>];AtEOL - (1:0,1) - Tokens:35
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[true];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[val];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Transition;[@];
CSharpTokenType.Identifier;[x];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[val];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.NotEqual;[!=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[3];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[true];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[val];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Transition;[@];
SyntaxKind.Identifier;[x];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[val];
SyntaxKind.Whitespace;[ ];
SyntaxKind.NotEqual;[!=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[3];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
MetaCode span - Gen<None> - [}] - SpanEditHandler;Accepts:None - (48:0,48) - Tokens:1
CSharpTokenType.RightBrace;[}];
SyntaxKind.RightBrace;[}];

62
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithMarkupSetsDotAsMarkup.stree.txt
View File

@ -1,50 +1,50 @@
Statement block - Gen<None> - 51 - (0:0,0)
Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpTokenType.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[true];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[true];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
Statement block - Gen<None> - 37 - (12:0,12)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [if(false) {] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:6
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[false];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[false];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
Markup block - Gen<None> - 24 - (24:0,24)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 5 - (25:0,25)
Markup span - Gen<Markup> - [<div>] - SpanEditHandler;Accepts:None - (25:0,25) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[div];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[div];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (30:0,30) - Tokens:1
HtmlTokenType.Unknown;[];
SyntaxKind.Unknown;[];
Expression block - Gen<Expr> - 10 - (30:0,30)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (30:0,30) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [something] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (31:0,31) - Tokens:1
CSharpTokenType.Identifier;[something];
SyntaxKind.Identifier;[something];
Markup span - Gen<Markup> - [.] - SpanEditHandler;Accepts:Any - (40:0,40) - Tokens:1
HtmlTokenType.Text;[.];
SyntaxKind.HtmlTextLiteral;[.];
Tag block - Gen<None> - 6 - (41:0,41)
Markup span - Gen<Markup> - [</div>] - SpanEditHandler;Accepts:None - (41:0,41) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[div];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[div];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (47:0,47) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Code span - Gen<Stmt> - [}] - SpanEditHandler;Accepts:Any - (48:0,48) - Tokens:1
CSharpTokenType.RightBrace;[}];
SyntaxKind.RightBrace;[}];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (49:0,49) - Tokens:2
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

248
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsersCanNestRecursively.stree.txt
View File

@ -1,179 +1,179 @@
Statement block - Gen<None> - 351 - (0:0,0)
Code span - Gen<Stmt> - [foreach(var c in db.Categories) {LF] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15
CSharpTokenType.Keyword;[foreach];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[c];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[in];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[db];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Categories];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
SyntaxKind.Keyword;[foreach];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[c];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[in];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[db];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Categories];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
Markup block - Gen<None> - 307 - (35:1,0)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (35:1,0) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 5 - (47:1,12)
Markup span - Gen<Markup> - [<div>] - SpanEditHandler;Accepts:None - (47:1,12) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[div];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[div];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (52:1,17) - Tokens:2
HtmlTokenType.NewLine;[LF];
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 4 - (70:2,16)
Markup span - Gen<Markup> - [<h1>] - SpanEditHandler;Accepts:None - (70:2,16) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[h1];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[h1];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (74:2,20) - Tokens:1
HtmlTokenType.Unknown;[];
SyntaxKind.Unknown;[];
Expression block - Gen<Expr> - 7 - (74:2,20)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (74:2,20) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [c.Name] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (75:2,21) - Tokens:3
CSharpTokenType.Identifier;[c];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Name];
SyntaxKind.Identifier;[c];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Name];
Tag block - Gen<None> - 5 - (81:2,27)
Markup span - Gen<Markup> - [</h1>] - SpanEditHandler;Accepts:None - (81:2,27) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[h1];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[h1];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (86:2,32) - Tokens:2
HtmlTokenType.NewLine;[LF];
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 4 - (104:3,16)
Markup span - Gen<Markup> - [<ul>] - SpanEditHandler;Accepts:None - (104:3,16) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[ul];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[ul];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (108:3,20) - Tokens:1
HtmlTokenType.NewLine;[LF];
SyntaxKind.NewLine;[LF];
Statement block - Gen<None> - 189 - (110:4,0)
Code span - Gen<Stmt> - [ ] - SpanEditHandler;Accepts:Any - (110:4,0) - Tokens:1
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (130:4,20) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [foreach(var p in c.Products) {LF] - SpanEditHandler;Accepts:Any - (131:4,21) - Tokens:15
CSharpTokenType.Keyword;[foreach];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[p];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[in];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[c];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Products];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
SyntaxKind.Keyword;[foreach];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[var];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[p];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[in];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[c];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Products];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
Markup block - Gen<None> - 113 - (163:5,0)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (163:5,0) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 4 - (187:5,24)
Markup span - Gen<Markup> - [<li>] - SpanEditHandler;Accepts:None - (187:5,24) - Tokens:3
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[li];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[li];
SyntaxKind.CloseAngle;[>];
Tag block - Gen<None> - 67 - (191:5,28)
Markup span - Gen<Markup> - [<a] - SpanEditHandler;Accepts:Any - (191:5,28) - Tokens:2
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.Text;[a];
SyntaxKind.OpenAngle;[<];
SyntaxKind.HtmlTextLiteral;[a];
Markup block - Gen<Attr:href, href="@(193:5,30),"@(256:5,93)> - 64 - (193:5,30)
Markup span - Gen<None> - [ href="] - SpanEditHandler;Accepts:Any - (193:5,30) - Tokens:4
HtmlTokenType.WhiteSpace;[ ];
HtmlTokenType.Text;[href];
HtmlTokenType.Equals;[=];
HtmlTokenType.DoubleQuote;["];
SyntaxKind.Whitespace;[ ];
SyntaxKind.HtmlTextLiteral;[href];
SyntaxKind.Equals;[=];
SyntaxKind.DoubleQuote;["];
Markup block - Gen<DynAttr:@(200:5,37)> - 56 - (200:5,37)
Expression block - Gen<Expr> - 56 - (200:5,37)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (200:5,37) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [Html.ActionUrl("Products", "Detail", new { id = p.Id })] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (201:5,38) - Tokens:24
CSharpTokenType.Identifier;[Html];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[ActionUrl];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;["Products"];
CSharpTokenType.Comma;[,];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["Detail"];
CSharpTokenType.Comma;[,];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[id];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[p];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Id];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
SyntaxKind.Identifier;[Html];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[ActionUrl];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;["Products"];
SyntaxKind.Comma;[,];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["Detail"];
SyntaxKind.Comma;[,];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[id];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[p];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Id];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
Markup span - Gen<None> - ["] - SpanEditHandler;Accepts:Any - (256:5,93) - Tokens:1
HtmlTokenType.DoubleQuote;["];
SyntaxKind.DoubleQuote;["];
Markup span - Gen<Markup> - [>] - SpanEditHandler;Accepts:None - (257:5,94) - Tokens:1
HtmlTokenType.CloseAngle;[>];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (258:5,95) - Tokens:1
HtmlTokenType.Unknown;[];
SyntaxKind.Unknown;[];
Expression block - Gen<Expr> - 7 - (258:5,95)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (258:5,95) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [p.Name] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (259:5,96) - Tokens:3
CSharpTokenType.Identifier;[p];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[Name];
SyntaxKind.Identifier;[p];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[Name];
Tag block - Gen<None> - 4 - (265:5,102)
Markup span - Gen<Markup> - [</a>] - SpanEditHandler;Accepts:None - (265:5,102) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[a];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[a];
SyntaxKind.CloseAngle;[>];
Tag block - Gen<None> - 5 - (269:5,106)
Markup span - Gen<Markup> - [</li>] - SpanEditHandler;Accepts:None - (269:5,106) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[li];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[li];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:None - (274:5,111) - Tokens:1
HtmlTokenType.NewLine;[LF];
SyntaxKind.NewLine;[LF];
Code span - Gen<Stmt> - [ }LF] - SpanEditHandler;Accepts:None - (276:6,0) - Tokens:3
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.NewLine;[LF];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (299:7,0) - Tokens:1
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 5 - (315:7,16)
Markup span - Gen<Markup> - [</ul>] - SpanEditHandler;Accepts:None - (315:7,16) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[ul];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[ul];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (320:7,21) - Tokens:2
HtmlTokenType.NewLine;[LF];
HtmlTokenType.WhiteSpace;[ ];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 6 - (334:8,12)
Markup span - Gen<Markup> - [</div>] - SpanEditHandler;Accepts:None - (334:8,12) - Tokens:4
HtmlTokenType.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/];
HtmlTokenType.Text;[div];
HtmlTokenType.CloseAngle;[>];
SyntaxKind.OpenAngle;[<];
SyntaxKind.ForwardSlash;[/];
SyntaxKind.HtmlTextLiteral;[div];
SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:None - (340:8,18) - Tokens:1
HtmlTokenType.NewLine;[LF];
SyntaxKind.NewLine;[LF];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:None - (342:9,0) - Tokens:2
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

186
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesElseIfBranchesOfIfStatement.stree.txt
View File

@ -1,95 +1,95 @@
Statement block - Gen<None> - 180 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:93
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"foo } bar"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[else];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"bar } baz"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];

36
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesExpressionOnSwitchCharacterFollowedByIdentifierStart.stree.txt
View File

@ -1,24 +1,24 @@
Statement block - Gen<None> - 25 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
Expression block - Gen<Expr> - 13 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [foo[4].bar()] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:8
CSharpTokenType.Identifier;[foo];
CSharpTokenType.LeftBracket;[[];
CSharpTokenType.IntegerLiteral;[4];
CSharpTokenType.RightBracket;[]];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[bar];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)];
SyntaxKind.Identifier;[foo];
SyntaxKind.LeftBracket;[[];
SyntaxKind.IntegerLiteral;[4];
SyntaxKind.RightBracket;[]];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[bar];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.RightParenthesis;[)];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (23:0,23) - Tokens:2
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesExpressionOnSwitchCharacterFollowedByOpenParen.stree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 24 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Identifier;[foo];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
Expression block - Gen<Expr> - 12 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpTokenType.Transition;[@];
SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [(] - SpanEditHandler;Accepts:None - (11:0,11) - Tokens:1
CSharpTokenType.LeftParenthesis;[(];
SyntaxKind.LeftParenthesis;[(];
Code span - Gen<Expr> - [foo + bar] - SpanEditHandler;Accepts:Any - (12:0,12) - Tokens:5
CSharpTokenType.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Plus;[+];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[bar];
SyntaxKind.Identifier;[foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Plus;[+];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[bar];
MetaCode span - Gen<None> - [)] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:1
CSharpTokenType.RightParenthesis;[)];
SyntaxKind.RightParenthesis;[)];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (22:0,22) - Tokens:2
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];

474
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesMultipleElseIfBranchesOfIfStatement.stree.txt
View File

@ -1,239 +1,239 @@
Statement block - Gen<None> - 459 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:237
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Assign;[=];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.LeftBrace;[{];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ];
CSharpTokenType.Identifier;[Debug];
CSharpTokenType.Dot;[.];
CSharpTokenType.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;];
CSharpTokenType.NewLine;[LF];
CSharpTokenType.RightBrace;[}];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"foo } bar"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[else];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"bar } baz"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[else];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"bar } baz"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[else];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"bar } baz"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[else];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[if];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.Keyword;[int];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[0];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[i];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LessThan;[<];
SyntaxKind.Whitespace;[ ];
SyntaxKind.IntegerLiteral;[10];
SyntaxKind.Semicolon;[;];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Keyword;[new];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Foo];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Bar];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Assign;[=];
SyntaxKind.Whitespace;[ ];
SyntaxKind.StringLiteral;["baz"];
SyntaxKind.Whitespace;[ ];
SyntaxKind.RightBrace;[}];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Whitespace;[ ];
SyntaxKind.LeftBrace;[{];
SyntaxKind.NewLine;[LF];
SyntaxKind.Whitespace;[ ];
SyntaxKind.Identifier;[Debug];
SyntaxKind.Dot;[.];
SyntaxKind.Identifier;[WriteLine];
SyntaxKind.LeftParenthesis;[(];
SyntaxKind.StringLiteral;[@"bar } baz"];
SyntaxKind.RightParenthesis;[)];
SyntaxKind.Semicolon;[;];
SyntaxKind.NewLine;[LF];
SyntaxKind.RightBrace;[}];

Some files were not shown because too many files have changed in this diff Show More