huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated the tokenizer and parser to operate on SyntaxTokens directly (#2540) 

* Updated the tokenizer and parser to operate on SyntaxTokens directly
- The tokenizer and parser now operates directly on SyntaxTokens (Green)
- The SyntaxToken(Red) is now created when the Span is built with the correct parent and position
- All other passes that run after the parsing is complete(TagHelperRewriter etc) will operate on the Red tokens.
- There is now only one type for all SyntaxTokens. They are differentiated by their SyntaxKind.
- Added equivalence checking for tokens
- Updated test code to react
- Regenerated baselines
This commit is contained in:
Ajay Bhargav Baaskaran 2018-08-16 16:11:01 -07:00 committed by GitHub
parent 0c25c2958c
commit db2a142132
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
996 changed files with 16796 additions and 17288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs
View File

@ -595,9 +595,9 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
if (span.Tokens.Count == 1) if (span.Tokens.Count == 1)
{ {
var token = span.Tokens[0] as HtmlToken; var token = span.Tokens[0];
if (token != null && if (token != null &&
token.Type == HtmlTokenType.Unknown && token.Kind == SyntaxKind.Unknown &&
token.Content.Length == 0) token.Content.Length == 0)
{ {
// We don't want to create IR nodes for marker tokens. // We don't want to create IR nodes for marker tokens.

3
src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs
View File

@ -4,12 +4,13 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Legacy; using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language namespace Microsoft.AspNetCore.Razor.Language
{ {
internal class DirectiveTokenEditHandler : SpanEditHandler internal class DirectiveTokenEditHandler : SpanEditHandler
{ {
public DirectiveTokenEditHandler(Func<string, IEnumerable<IToken>> tokenizer) : base(tokenizer) public DirectiveTokenEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer) : base(tokenizer)
{ {
} }

7
src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs
View File

@ -3,6 +3,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Microsoft.Extensions.Internal; using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -11,18 +12,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
private static readonly int TypeHashCode = typeof(AutoCompleteEditHandler).GetHashCode(); private static readonly int TypeHashCode = typeof(AutoCompleteEditHandler).GetHashCode();
public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer) public AutoCompleteEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer)
: base(tokenizer) : base(tokenizer)
{ {
} }
public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer, bool autoCompleteAtEndOfSpan) public AutoCompleteEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer, bool autoCompleteAtEndOfSpan)
: this(tokenizer) : this(tokenizer)
{ {
AutoCompleteAtEndOfSpan = autoCompleteAtEndOfSpan; AutoCompleteAtEndOfSpan = autoCompleteAtEndOfSpan;
} }
public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer, AcceptedCharactersInternal accepted) public AutoCompleteEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer, AcceptedCharactersInternal accepted)
: base(tokenizer, accepted) : base(tokenizer, accepted)
{ {
} }

328
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
View File

@ -5,17 +5,18 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CSharpCodeParser : TokenizerBackedParser<CSharpTokenizer, CSharpToken, CSharpTokenType> internal class CSharpCodeParser : TokenizerBackedParser<CSharpTokenizer>
{ {
private static HashSet<char> InvalidNonWhitespaceNameCharacters = new HashSet<char>(new[] private static HashSet<char> InvalidNonWhitespaceNameCharacters = new HashSet<char>(new[]
{ {
'@', '!', '<', '/', '?', '[', '>', ']', '=', '"', '\'', '*' '@', '!', '<', '/', '?', '[', '>', ']', '=', '"', '\'', '*'
}); });
private static readonly Func<CSharpToken, bool> IsValidStatementSpacingToken = private static readonly Func<SyntaxToken, bool> IsValidStatementSpacingToken =
IsSpacingToken(includeNewLines: true, includeComments: true); IsSpacingToken(includeNewLines: true, includeComments: true);
internal static readonly DirectiveDescriptor AddTagHelperDirectiveDescriptor = DirectiveDescriptor.CreateDirective( internal static readonly DirectiveDescriptor AddTagHelperDirectiveDescriptor = DirectiveDescriptor.CreateDirective(
@ -102,7 +103,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public bool IsNested { get; set; } public bool IsNested { get; set; }
protected override bool TokenTypeEquals(CSharpTokenType x, CSharpTokenType y) => x == y; protected override bool TokenKindEquals(SyntaxKind x, SyntaxKind y) => x == y;
protected void MapDirectives(Action handler, params string[] directives) protected void MapDirectives(Action handler, params string[] directives)
{ {
@ -161,16 +162,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Conditional("DEBUG")] [Conditional("DEBUG")]
internal void Assert(CSharpKeyword expectedKeyword) internal void Assert(CSharpKeyword expectedKeyword)
{ {
Debug.Assert(CurrentToken.Type == CSharpTokenType.Keyword && var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
CurrentToken.Keyword.HasValue && Debug.Assert(CurrentToken.Kind == SyntaxKind.Keyword &&
CurrentToken.Keyword.Value == expectedKeyword); result.HasValue &&
result.Value == expectedKeyword);
} }
protected internal bool At(CSharpKeyword keyword) protected internal bool At(CSharpKeyword keyword)
{ {
return At(CSharpTokenType.Keyword) && var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
CurrentToken.Keyword.HasValue && return At(SyntaxKind.Keyword) &&
CurrentToken.Keyword.Value == keyword; result.HasValue &&
result.Value == keyword;
} }
protected internal bool AcceptIf(CSharpKeyword keyword) protected internal bool AcceptIf(CSharpKeyword keyword)
@ -183,11 +186,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false; return false;
} }
protected static Func<CSharpToken, bool> IsSpacingToken(bool includeNewLines, bool includeComments) protected static Func<SyntaxToken, bool> IsSpacingToken(bool includeNewLines, bool includeComments)
{ {
return token => token.Type == CSharpTokenType.WhiteSpace || return token => token.Kind == SyntaxKind.Whitespace ||
(includeNewLines && token.Type == CSharpTokenType.NewLine) || (includeNewLines && token.Kind == SyntaxKind.NewLine) ||
(includeComments && token.Type == CSharpTokenType.Comment); (includeComments && token.Kind == SyntaxKind.CSharpComment);
} }
public override void ParseBlock() public override void ParseBlock()
@ -209,24 +212,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
var current = CurrentToken; var current = CurrentToken;
if (At(CSharpTokenType.StringLiteral) && if (At(SyntaxKind.StringLiteral) &&
CurrentToken.Content.Length > 0 && CurrentToken.Content.Length > 0 &&
CurrentToken.Content[0] == SyntaxConstants.TransitionCharacter) CurrentToken.Content[0] == SyntaxConstants.TransitionCharacter)
{ {
var split = Language.SplitToken(CurrentToken, 1, CSharpTokenType.Transition); var split = Language.SplitToken(CurrentToken, 1, SyntaxKind.Transition);
current = split.Item1; current = split.Item1;
// Back up to the end of the transition // Back up to the end of the transition
Context.Source.Position -= split.Item2.Content.Length; Context.Source.Position -= split.Item2.Content.Length;
NextToken(); NextToken();
} }
else if (At(CSharpTokenType.Transition)) else if (At(SyntaxKind.Transition))
{ {
NextToken(); NextToken();
} }
// Accept "@" if we see it, but if we don't, that's OK. We assume we were started for a good reason // Accept "@" if we see it, but if we don't, that's OK. We assume we were started for a good reason
if (current.Type == CSharpTokenType.Transition) if (current.Kind == SyntaxKind.Transition)
{ {
if (Span.Tokens.Count > 0) if (Span.Tokens.Count > 0)
{ {
@ -251,9 +254,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.ChunkGenerator = new StatementChunkGenerator(); span.ChunkGenerator = new StatementChunkGenerator();
} }
private void AtTransition(CSharpToken current) private void AtTransition(SyntaxToken current)
{ {
Debug.Assert(current.Type == CSharpTokenType.Transition); Debug.Assert(current.Kind == SyntaxKind.Transition);
Accept(current); Accept(current);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
@ -273,14 +276,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// What type of block is this? // What type of block is this?
if (!EndOfFile) if (!EndOfFile)
{ {
if (CurrentToken.Type == CSharpTokenType.LeftParenthesis) if (CurrentToken.Kind == SyntaxKind.LeftParenthesis)
{ {
Context.Builder.CurrentBlock.Type = BlockKindInternal.Expression; Context.Builder.CurrentBlock.Type = BlockKindInternal.Expression;
Context.Builder.CurrentBlock.ChunkGenerator = new ExpressionChunkGenerator(); Context.Builder.CurrentBlock.ChunkGenerator = new ExpressionChunkGenerator();
ExplicitExpression(); ExplicitExpression();
return; return;
} }
else if (CurrentToken.Type == CSharpTokenType.Identifier) else if (CurrentToken.Kind == SyntaxKind.Identifier)
{ {
if (TryGetDirectiveHandler(CurrentToken.Content, out var handler)) if (TryGetDirectiveHandler(CurrentToken.Content, out var handler))
{ {
@ -306,7 +309,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return; return;
} }
} }
else if (CurrentToken.Type == CSharpTokenType.Keyword) else if (CurrentToken.Kind == SyntaxKind.Keyword)
{ {
if (TryGetDirectiveHandler(CurrentToken.Content, out var handler)) if (TryGetDirectiveHandler(CurrentToken.Content, out var handler))
{ {
@ -320,7 +323,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return; return;
} }
} }
else if (CurrentToken.Type == CSharpTokenType.LeftBrace) else if (CurrentToken.Kind == SyntaxKind.LeftBrace)
{ {
VerbatimBlock(); VerbatimBlock();
return; return;
@ -339,7 +342,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
AcceptedCharacters = AcceptedCharactersInternal.NonWhiteSpace AcceptedCharacters = AcceptedCharactersInternal.NonWhiteSpace
}; };
if (At(CSharpTokenType.WhiteSpace) || At(CSharpTokenType.NewLine)) if (At(SyntaxKind.Whitespace) || At(SyntaxKind.NewLine))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedWhiteSpaceAtStartOfCodeBlock( RazorDiagnosticFactory.CreateParsing_UnexpectedWhiteSpaceAtStartOfCodeBlock(
@ -369,7 +372,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void VerbatimBlock() private void VerbatimBlock()
{ {
Assert(CSharpTokenType.LeftBrace); Assert(SyntaxKind.LeftBrace);
var block = new Block(Resources.BlockName_Code, CurrentStart); var block = new Block(Resources.BlockName_Code, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -385,13 +388,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = new StatementChunkGenerator(); Span.ChunkGenerator = new StatementChunkGenerator();
AddMarkerTokenIfNecessary(); AddMarkerTokenIfNecessary();
if (!At(CSharpTokenType.RightBrace)) if (!At(SyntaxKind.RightBrace))
{ {
editHandler.AutoCompleteString = "}"; editHandler.AutoCompleteString = "}";
} }
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
if (Optional(CSharpTokenType.RightBrace)) if (Optional(SyntaxKind.RightBrace))
{ {
// Set up the "}" span // Set up the "}" span
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
@ -401,8 +404,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!IsNested) if (!IsNested)
{ {
EnsureCurrent(); EnsureCurrent();
if (At(CSharpTokenType.NewLine) || if (At(SyntaxKind.NewLine) ||
(At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.NewLine))) (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.NewLine)))
{ {
Context.NullGenerateWhitespaceAndNewLine = true; Context.NullGenerateWhitespaceAndNewLine = true;
} }
@ -456,13 +459,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
if (!EndOfFile) if (!EndOfFile)
{ {
if (CurrentToken.Type == CSharpTokenType.LeftParenthesis || if (CurrentToken.Kind == SyntaxKind.LeftParenthesis ||
CurrentToken.Type == CSharpTokenType.LeftBracket) CurrentToken.Kind == SyntaxKind.LeftBracket)
{ {
// If we end within "(", whitespace is fine // If we end within "(", whitespace is fine
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
CSharpTokenType right; SyntaxKind right;
bool success; bool success;
using (PushSpanConfig((span, prev) => using (PushSpanConfig((span, prev) =>
@ -471,13 +474,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any; span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
})) }))
{ {
right = Language.FlipBracket(CurrentToken.Type); right = Language.FlipBracket(CurrentToken.Kind);
success = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates); success = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
} }
if (!success) if (!success)
{ {
AcceptUntil(CSharpTokenType.LessThan); AcceptUntil(SyntaxKind.LessThan);
} }
if (At(right)) if (At(right))
{ {
@ -488,22 +491,22 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
return MethodCallOrArrayIndex(acceptedCharacters); return MethodCallOrArrayIndex(acceptedCharacters);
} }
if (At(CSharpTokenType.QuestionMark)) if (At(SyntaxKind.QuestionMark))
{ {
var next = Lookahead(count: 1); var next = Lookahead(count: 1);
if (next != null) if (next != null)
{ {
if (next.Type == CSharpTokenType.Dot) if (next.Kind == SyntaxKind.Dot)
{ {
// Accept null conditional dot operator (?.). // Accept null conditional dot operator (?.).
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptAndMoveNext(); AcceptAndMoveNext();
// If the next piece after the ?. is a keyword or identifier then we want to continue. // If the next piece after the ?. is a keyword or identifier then we want to continue.
return At(CSharpTokenType.Identifier) || At(CSharpTokenType.Keyword); return At(SyntaxKind.Identifier) || At(SyntaxKind.Keyword);
} }
else if (next.Type == CSharpTokenType.LeftBracket) else if (next.Kind == SyntaxKind.LeftBracket)
{ {
// We're at the ? for a null conditional bracket operator (?[). // We're at the ? for a null conditional bracket operator (?[).
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -513,12 +516,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
} }
else if (At(CSharpTokenType.Dot)) else if (At(SyntaxKind.Dot))
{ {
var dot = CurrentToken; var dot = CurrentToken;
if (NextToken()) if (NextToken())
{ {
if (At(CSharpTokenType.Identifier) || At(CSharpTokenType.Keyword)) if (At(SyntaxKind.Identifier) || At(SyntaxKind.Keyword))
{ {
// Accept the dot and return to the start // Accept the dot and return to the start
Accept(dot); Accept(dot);
@ -540,7 +543,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(dot); Accept(dot);
} }
} }
else if (!At(CSharpTokenType.WhiteSpace) && !At(CSharpTokenType.NewLine)) else if (!At(SyntaxKind.Whitespace) && !At(SyntaxKind.NewLine))
{ {
PutCurrentBack(); PutCurrentBack();
} }
@ -587,8 +590,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaptureWhitespaceAtEndOfCodeOnlyLine() private void CaptureWhitespaceAtEndOfCodeOnlyLine()
{ {
var whitespace = ReadWhile(token => token.Type == CSharpTokenType.WhiteSpace); var whitespace = ReadWhile(token => token.Kind == SyntaxKind.Whitespace);
if (At(CSharpTokenType.NewLine)) if (At(SyntaxKind.NewLine))
{ {
Accept(whitespace); Accept(whitespace);
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -610,7 +613,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ExplicitExpression() private void ExplicitExpression()
{ {
var block = new Block(Resources.BlockName_ExplicitExpression, CurrentStart); var block = new Block(Resources.BlockName_ExplicitExpression, CurrentStart);
Assert(CSharpTokenType.LeftParenthesis); Assert(SyntaxKind.LeftParenthesis);
AcceptAndMoveNext(); AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
@ -621,13 +624,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
BalancingModes.BacktrackOnFailure | BalancingModes.BacktrackOnFailure |
BalancingModes.NoErrorOnFailure | BalancingModes.NoErrorOnFailure |
BalancingModes.AllowCommentsAndTemplates, BalancingModes.AllowCommentsAndTemplates,
CSharpTokenType.LeftParenthesis, SyntaxKind.LeftParenthesis,
CSharpTokenType.RightParenthesis, SyntaxKind.RightParenthesis,
block.Start); block.Start);
if (!success) if (!success)
{ {
AcceptUntil(CSharpTokenType.LessThan); AcceptUntil(SyntaxKind.LessThan);
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF( RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF(
new SourceSpan(block.Start, contentLength: 1 /* ( */), block.Name, ")", "(")); new SourceSpan(block.Start, contentLength: 1 /* ( */), block.Name, ")", "("));
@ -636,13 +639,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// If necessary, put an empty-content marker token here // If necessary, put an empty-content marker token here
if (Span.Tokens.Count == 0) if (Span.Tokens.Count == 0)
{ {
Accept(new CSharpToken(string.Empty, CSharpTokenType.Unknown)); Accept(SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty));
} }
// Output the content span and then capture the ")" // Output the content span and then capture the ")"
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
} }
Optional(CSharpTokenType.RightParenthesis); Optional(SyntaxKind.RightParenthesis);
if (!EndOfFile) if (!EndOfFile)
{ {
PutCurrentBack(); PutCurrentBack();
@ -702,7 +705,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// No embedded transitions in C#, so ignore that param // No embedded transitions in C#, so ignore that param
return allowTemplatesAndComments return allowTemplatesAndComments
&& ((Language.IsTransition(CurrentToken) && ((Language.IsTransition(CurrentToken)
&& NextIs(CSharpTokenType.LessThan, CSharpTokenType.Colon, CSharpTokenType.DoubleColon)) && NextIs(SyntaxKind.LessThan, SyntaxKind.Colon, SyntaxKind.DoubleColon))
|| Language.IsCommentStart(CurrentToken)); || Language.IsCommentStart(CurrentToken));
} }
@ -783,12 +786,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaseStatement(bool topLevel) private void CaseStatement(bool topLevel)
{ {
Assert(CSharpTokenType.Keyword); Assert(SyntaxKind.Keyword);
Debug.Assert(CurrentToken.Keyword != null && var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
(CurrentToken.Keyword.Value == CSharpKeyword.Case || Debug.Assert(result.HasValue &&
CurrentToken.Keyword.Value == CSharpKeyword.Default)); (result.Value == CSharpKeyword.Case ||
AcceptUntil(CSharpTokenType.Colon); result.Value == CSharpKeyword.Default));
Optional(CSharpTokenType.Colon); AcceptUntil(SyntaxKind.Colon);
Optional(SyntaxKind.Colon);
} }
private void DoStatement(bool topLevel) private void DoStatement(bool topLevel)
@ -813,7 +817,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Assert(CSharpKeyword.While); Assert(CSharpKeyword.While);
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (AcceptCondition() && Optional(CSharpTokenType.Semicolon)) if (AcceptCondition() && Optional(SyntaxKind.Semicolon))
{ {
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
} }
@ -832,12 +836,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
if (At(CSharpTokenType.LeftParenthesis)) if (At(SyntaxKind.LeftParenthesis))
{ {
// using ( ==> Using Statement // using ( ==> Using Statement
UsingStatement(block); UsingStatement(block);
} }
else if (At(CSharpTokenType.Identifier) || At(CSharpKeyword.Static)) else if (At(SyntaxKind.Identifier) || At(CSharpKeyword.Static))
{ {
// using Identifier ==> Using Declaration // using Identifier ==> Using Declaration
if (!topLevel) if (!topLevel)
@ -865,16 +869,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Context.Builder.CurrentBlock.Type = BlockKindInternal.Directive; Context.Builder.CurrentBlock.Type = BlockKindInternal.Directive;
var start = CurrentStart; var start = CurrentStart;
if (At(CSharpTokenType.Identifier)) if (At(SyntaxKind.Identifier))
{ {
// non-static using // non-static using
NamespaceOrTypeName(); NamespaceOrTypeName();
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (At(CSharpTokenType.Assign)) if (At(SyntaxKind.Assign))
{ {
// Alias // Alias
Accept(whitespace); Accept(whitespace);
Assert(CSharpTokenType.Assign); Assert(SyntaxKind.Assign);
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -904,7 +908,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Optional ";" // Optional ";"
if (EnsureCurrent()) if (EnsureCurrent())
{ {
Optional(CSharpTokenType.Semicolon); Optional(SyntaxKind.Semicolon);
} }
} }
@ -919,16 +923,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var expectingDot = false; var expectingDot = false;
var tokens = ReadWhile(token => var tokens = ReadWhile(token =>
{ {
var type = token.Type; var type = token.Kind;
if ((expectingDot && type == CSharpTokenType.Dot) || if ((expectingDot && type == SyntaxKind.Dot) ||
(!expectingDot && type == CSharpTokenType.Identifier)) (!expectingDot && type == SyntaxKind.Identifier))
{ {
expectingDot = !expectingDot; expectingDot = !expectingDot;
return true; return true;
} }
if (type != CSharpTokenType.WhiteSpace && if (type != SyntaxKind.Whitespace &&
type != CSharpTokenType.NewLine) type != SyntaxKind.NewLine)
{ {
expectingDot = false; expectingDot = false;
currentIdentifierLength += token.Content.Length; currentIdentifierLength += token.Content.Length;
@ -966,69 +970,69 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool NamespaceOrTypeName() protected bool NamespaceOrTypeName()
{ {
if (Optional(CSharpTokenType.LeftParenthesis)) if (Optional(SyntaxKind.LeftParenthesis))
{ {
while (!Optional(CSharpTokenType.RightParenthesis) && !EndOfFile) while (!Optional(SyntaxKind.RightParenthesis) && !EndOfFile)
{ {
Optional(CSharpTokenType.WhiteSpace); Optional(SyntaxKind.Whitespace);
if (!NamespaceOrTypeName()) if (!NamespaceOrTypeName())
{ {
return false; return false;
} }
Optional(CSharpTokenType.WhiteSpace); Optional(SyntaxKind.Whitespace);
Optional(CSharpTokenType.Identifier); Optional(SyntaxKind.Identifier);
Optional(CSharpTokenType.WhiteSpace); Optional(SyntaxKind.Whitespace);
Optional(CSharpTokenType.Comma); Optional(SyntaxKind.Comma);
} }
if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.QuestionMark)) if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.QuestionMark))
{ {
// Only accept the whitespace if we are going to consume the next token. // Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
Optional(CSharpTokenType.QuestionMark); // Nullable Optional(SyntaxKind.QuestionMark); // Nullable
return true; return true;
} }
else if (Optional(CSharpTokenType.Identifier) || Optional(CSharpTokenType.Keyword)) else if (Optional(SyntaxKind.Identifier) || Optional(SyntaxKind.Keyword))
{ {
if (Optional(CSharpTokenType.DoubleColon)) if (Optional(SyntaxKind.DoubleColon))
{ {
if (!Optional(CSharpTokenType.Identifier)) if (!Optional(SyntaxKind.Identifier))
{ {
Optional(CSharpTokenType.Keyword); Optional(SyntaxKind.Keyword);
} }
} }
if (At(CSharpTokenType.LessThan)) if (At(SyntaxKind.LessThan))
{ {
TypeArgumentList(); TypeArgumentList();
} }
if (Optional(CSharpTokenType.Dot)) if (Optional(SyntaxKind.Dot))
{ {
NamespaceOrTypeName(); NamespaceOrTypeName();
} }
if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.QuestionMark)) if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.QuestionMark))
{ {
// Only accept the whitespace if we are going to consume the next token. // Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
Optional(CSharpTokenType.QuestionMark); // Nullable Optional(SyntaxKind.QuestionMark); // Nullable
if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.LeftBracket)) if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.LeftBracket))
{ {
// Only accept the whitespace if we are going to consume the next token. // Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
while (At(CSharpTokenType.LeftBracket)) while (At(SyntaxKind.LeftBracket))
{ {
Balance(BalancingModes.None); Balance(BalancingModes.None);
Optional(CSharpTokenType.RightBracket); Optional(SyntaxKind.RightBracket);
} }
return true; return true;
} }
@ -1040,14 +1044,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TypeArgumentList() private void TypeArgumentList()
{ {
Assert(CSharpTokenType.LessThan); Assert(SyntaxKind.LessThan);
Balance(BalancingModes.None); Balance(BalancingModes.None);
Optional(CSharpTokenType.GreaterThan); Optional(SyntaxKind.GreaterThan);
} }
private void UsingStatement(Block block) private void UsingStatement(Block block)
{ {
Assert(CSharpTokenType.LeftParenthesis); Assert(SyntaxKind.LeftParenthesis);
// Parse condition // Parse condition
if (AcceptCondition()) if (AcceptCondition())
@ -1159,12 +1163,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!EndOfFile) if (!EndOfFile)
{ {
// Check for "{" to make sure we're at a block // Check for "{" to make sure we're at a block
if (!At(CSharpTokenType.LeftBrace)) if (!At(SyntaxKind.LeftBrace))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_SingleLineControlFlowStatementsNotAllowed( RazorDiagnosticFactory.CreateParsing_SingleLineControlFlowStatementsNotAllowed(
new SourceSpan(CurrentStart, CurrentToken.Content.Length), new SourceSpan(CurrentStart, CurrentToken.Content.Length),
Language.GetSample(CSharpTokenType.LeftBrace), Language.GetSample(SyntaxKind.LeftBrace),
CurrentToken.Content)); CurrentToken.Content));
} }
@ -1175,7 +1179,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void UnconditionalBlock() private void UnconditionalBlock()
{ {
Assert(CSharpTokenType.Keyword); Assert(SyntaxKind.Keyword);
var block = new Block(CurrentToken, CurrentStart); var block = new Block(CurrentToken, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -1219,7 +1223,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ConditionalBlock(bool topLevel) private void ConditionalBlock(bool topLevel)
{ {
Assert(CSharpTokenType.Keyword); Assert(SyntaxKind.Keyword);
var block = new Block(CurrentToken, CurrentStart); var block = new Block(CurrentToken, CurrentStart);
ConditionalBlock(block); ConditionalBlock(block);
if (topLevel) if (topLevel)
@ -1243,16 +1247,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool AcceptCondition() private bool AcceptCondition()
{ {
if (At(CSharpTokenType.LeftParenthesis)) if (At(SyntaxKind.LeftParenthesis))
{ {
var complete = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates); var complete = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
if (!complete) if (!complete)
{ {
AcceptUntil(CSharpTokenType.NewLine); AcceptUntil(SyntaxKind.NewLine);
} }
else else
{ {
Optional(CSharpTokenType.RightParenthesis); Optional(SyntaxKind.RightParenthesis);
} }
return complete; return complete;
} }
@ -1280,16 +1284,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return; return;
} }
var type = CurrentToken.Type; var type = CurrentToken.Kind;
var loc = CurrentStart; var loc = CurrentStart;
// Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text // Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text
var isSingleLineMarkup = type == CSharpTokenType.Transition && var isSingleLineMarkup = type == SyntaxKind.Transition &&
(NextIs(CSharpTokenType.Colon, CSharpTokenType.DoubleColon)); (NextIs(SyntaxKind.Colon, SyntaxKind.DoubleColon));
var isMarkup = isSingleLineMarkup || var isMarkup = isSingleLineMarkup ||
type == CSharpTokenType.LessThan || type == SyntaxKind.LessThan ||
(type == CSharpTokenType.Transition && NextIs(CSharpTokenType.LessThan)); (type == SyntaxKind.Transition && NextIs(SyntaxKind.LessThan));
if (Context.DesignTimeMode || !isMarkup) if (Context.DesignTimeMode || !isMarkup)
{ {
@ -1321,7 +1325,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (isMarkup) if (isMarkup)
{ {
if (type == CSharpTokenType.Transition && !isSingleLineMarkup) if (type == SyntaxKind.Transition && !isSingleLineMarkup)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_AtInCodeMustBeFollowedByColonParenOrIdentifierStart( RazorDiagnosticFactory.CreateParsing_AtInCodeMustBeFollowedByColonParenOrIdentifierStart(
@ -1331,7 +1335,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Markup block // Markup block
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
if (Context.DesignTimeMode && CurrentToken != null && if (Context.DesignTimeMode && CurrentToken != null &&
(CurrentToken.Type == CSharpTokenType.LessThan || CurrentToken.Type == CSharpTokenType.Transition)) (CurrentToken.Kind == SyntaxKind.LessThan || CurrentToken.Kind == SyntaxKind.Transition))
{ {
PutCurrentBack(); PutCurrentBack();
} }
@ -1344,33 +1348,33 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
private void HandleStatement(Block block, CSharpTokenType type) private void HandleStatement(Block block, SyntaxKind type)
{ {
switch (type) switch (type)
{ {
case CSharpTokenType.RazorCommentTransition: case SyntaxKind.RazorCommentTransition:
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
RazorComment(); RazorComment();
Statement(block); Statement(block);
break; break;
case CSharpTokenType.LeftBrace: case SyntaxKind.LeftBrace:
// Verbatim Block // Verbatim Block
block = block ?? new Block(Resources.BlockName_Code, CurrentStart); block = block ?? new Block(Resources.BlockName_Code, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
CodeBlock(block); CodeBlock(block);
break; break;
case CSharpTokenType.Keyword: case SyntaxKind.Keyword:
// Keyword block // Keyword block
HandleKeyword(false, StandardStatement); HandleKeyword(false, StandardStatement);
break; break;
case CSharpTokenType.Transition: case SyntaxKind.Transition:
// Embedded Expression block // Embedded Expression block
EmbeddedExpression(); EmbeddedExpression();
break; break;
case CSharpTokenType.RightBrace: case SyntaxKind.RightBrace:
// Possible end of Code Block, just run the continuation // Possible end of Code Block, just run the continuation
break; break;
case CSharpTokenType.Comment: case SyntaxKind.CSharpComment:
AcceptAndMoveNext(); AcceptAndMoveNext();
break; break;
default: default:
@ -1383,11 +1387,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void EmbeddedExpression() private void EmbeddedExpression()
{ {
// First, verify the type of the block // First, verify the type of the block
Assert(CSharpTokenType.Transition); Assert(SyntaxKind.Transition);
var transition = CurrentToken; var transition = CurrentToken;
NextToken(); NextToken();
if (At(CSharpTokenType.Transition)) if (At(SyntaxKind.Transition))
{ {
// Escaped "@" // Escaped "@"
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
@ -1397,14 +1401,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
Assert(CSharpTokenType.Transition); Assert(SyntaxKind.Transition);
AcceptAndMoveNext(); AcceptAndMoveNext();
StandardStatement(); StandardStatement();
} }
else else
{ {
// Throw errors as necessary, but continue parsing // Throw errors as necessary, but continue parsing
if (At(CSharpTokenType.LeftBrace)) if (At(SyntaxKind.LeftBrace))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedNestedCodeBlock( RazorDiagnosticFactory.CreateParsing_UnexpectedNestedCodeBlock(
@ -1428,48 +1432,48 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
var bookmark = CurrentStart.AbsoluteIndex; var bookmark = CurrentStart.AbsoluteIndex;
var read = ReadWhile(token => var read = ReadWhile(token =>
token.Type != CSharpTokenType.Semicolon && token.Kind != SyntaxKind.Semicolon &&
token.Type != CSharpTokenType.RazorCommentTransition && token.Kind != SyntaxKind.RazorCommentTransition &&
token.Type != CSharpTokenType.Transition && token.Kind != SyntaxKind.Transition &&
token.Type != CSharpTokenType.LeftBrace && token.Kind != SyntaxKind.LeftBrace &&
token.Type != CSharpTokenType.LeftParenthesis && token.Kind != SyntaxKind.LeftParenthesis &&
token.Type != CSharpTokenType.LeftBracket && token.Kind != SyntaxKind.LeftBracket &&
token.Type != CSharpTokenType.RightBrace); token.Kind != SyntaxKind.RightBrace);
if (At(CSharpTokenType.LeftBrace) || if (At(SyntaxKind.LeftBrace) ||
At(CSharpTokenType.LeftParenthesis) || At(SyntaxKind.LeftParenthesis) ||
At(CSharpTokenType.LeftBracket)) At(SyntaxKind.LeftBracket))
{ {
Accept(read); Accept(read);
if (Balance(BalancingModes.AllowCommentsAndTemplates | BalancingModes.BacktrackOnFailure)) if (Balance(BalancingModes.AllowCommentsAndTemplates | BalancingModes.BacktrackOnFailure))
{ {
Optional(CSharpTokenType.RightBrace); Optional(SyntaxKind.RightBrace);
} }
else else
{ {
// Recovery // Recovery
AcceptUntil(CSharpTokenType.LessThan, CSharpTokenType.RightBrace); AcceptUntil(SyntaxKind.LessThan, SyntaxKind.RightBrace);
return; return;
} }
} }
else if (At(CSharpTokenType.Transition) && (NextIs(CSharpTokenType.LessThan, CSharpTokenType.Colon))) else if (At(SyntaxKind.Transition) && (NextIs(SyntaxKind.LessThan, SyntaxKind.Colon)))
{ {
Accept(read); Accept(read);
Output(SpanKindInternal.Code); Output(SpanKindInternal.Code);
Template(); Template();
} }
else if (At(CSharpTokenType.RazorCommentTransition)) else if (At(SyntaxKind.RazorCommentTransition))
{ {
Accept(read); Accept(read);
RazorComment(); RazorComment();
} }
else if (At(CSharpTokenType.Semicolon)) else if (At(SyntaxKind.Semicolon))
{ {
Accept(read); Accept(read);
AcceptAndMoveNext(); AcceptAndMoveNext();
return; return;
} }
else if (At(CSharpTokenType.RightBrace)) else if (At(SyntaxKind.RightBrace))
{ {
Accept(read); Accept(read);
return; return;
@ -1478,7 +1482,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
Context.Source.Position = bookmark; Context.Source.Position = bookmark;
NextToken(); NextToken();
AcceptUntil(CSharpTokenType.LessThan, CSharpTokenType.LeftBrace, CSharpTokenType.RightBrace); AcceptUntil(SyntaxKind.LessThan, SyntaxKind.LeftBrace, SyntaxKind.RightBrace);
return; return;
} }
} }
@ -1492,7 +1496,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CodeBlock(bool acceptTerminatingBrace, Block block) private void CodeBlock(bool acceptTerminatingBrace, Block block)
{ {
EnsureCurrent(); EnsureCurrent();
while (!EndOfFile && !At(CSharpTokenType.RightBrace)) while (!EndOfFile && !At(SyntaxKind.RightBrace))
{ {
// Parse a statement, then return here // Parse a statement, then return here
Statement(); Statement();
@ -1507,7 +1511,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
else if (acceptTerminatingBrace) else if (acceptTerminatingBrace)
{ {
Assert(CSharpTokenType.RightBrace); Assert(SyntaxKind.RightBrace);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
@ -1515,8 +1519,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void HandleKeyword(bool topLevel, Action fallback) private void HandleKeyword(bool topLevel, Action fallback)
{ {
Debug.Assert(CurrentToken.Type == CSharpTokenType.Keyword && CurrentToken.Keyword != null); var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
if (_keywordParsers.TryGetValue(CurrentToken.Keyword.Value, out var handler)) Debug.Assert(CurrentToken.Kind == SyntaxKind.Keyword && result.HasValue);
if (_keywordParsers.TryGetValue(result.Value, out var handler))
{ {
handler(topLevel); handler(topLevel);
} }
@ -1526,12 +1531,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
private IEnumerable<CSharpToken> SkipToNextImportantToken() private IEnumerable<SyntaxToken> SkipToNextImportantToken()
{ {
while (!EndOfFile) while (!EndOfFile)
{ {
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
if (At(CSharpTokenType.RazorCommentTransition)) if (At(SyntaxKind.RazorCommentTransition))
{ {
Accept(whitespace); Accept(whitespace);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
@ -1542,7 +1547,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return whitespace; return whitespace;
} }
} }
return Enumerable.Empty<CSharpToken>(); return Enumerable.Empty<SyntaxToken>();
} }
// Common code for Parsers, but FxCop REALLY doesn't like it in the base class.. moving it here for now. // Common code for Parsers, but FxCop REALLY doesn't like it in the base class.. moving it here for now.
@ -1644,8 +1649,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
for (var i = 0; i < descriptor.Tokens.Count; i++) for (var i = 0; i < descriptor.Tokens.Count; i++)
{ {
if (!At(CSharpTokenType.WhiteSpace) && if (!At(SyntaxKind.Whitespace) &&
!At(CSharpTokenType.NewLine) && !At(SyntaxKind.NewLine) &&
!EndOfFile) !EndOfFile)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
@ -1664,7 +1669,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Code, AcceptedCharactersInternal.WhiteSpace); Output(SpanKindInternal.Code, AcceptedCharactersInternal.WhiteSpace);
if (EndOfFile || At(CSharpTokenType.NewLine)) if (EndOfFile || At(SyntaxKind.NewLine))
{ {
// Add a marker token to provide CSharp intellisense when we start typing the directive token. // Add a marker token to provide CSharp intellisense when we start typing the directive token.
AddMarkerTokenIfNecessary(); AddMarkerTokenIfNecessary();
@ -1679,7 +1684,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup, AcceptedCharactersInternal.WhiteSpace); Output(SpanKindInternal.Markup, AcceptedCharactersInternal.WhiteSpace);
} }
if (tokenDescriptor.Optional && (EndOfFile || At(CSharpTokenType.NewLine))) if (tokenDescriptor.Optional && (EndOfFile || At(SyntaxKind.NewLine)))
{ {
break; break;
} }
@ -1718,7 +1723,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break; break;
case DirectiveTokenKind.Member: case DirectiveTokenKind.Member:
if (At(CSharpTokenType.Identifier)) if (At(SyntaxKind.Identifier))
{ {
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
@ -1732,7 +1737,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break; break;
case DirectiveTokenKind.String: case DirectiveTokenKind.String:
if (At(CSharpTokenType.StringLiteral) && CurrentToken.Errors.Count == 0) if (At(SyntaxKind.StringLiteral) && !CurrentToken.ContainsDiagnostics)
{ {
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
@ -1759,13 +1764,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
case DirectiveKind.SingleLine: case DirectiveKind.SingleLine:
Output(SpanKindInternal.None, AcceptedCharactersInternal.WhiteSpace); Output(SpanKindInternal.None, AcceptedCharactersInternal.WhiteSpace);
Optional(CSharpTokenType.Semicolon); Optional(SyntaxKind.Semicolon);
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.WhiteSpace); Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.WhiteSpace);
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
if (At(CSharpTokenType.NewLine)) if (At(SyntaxKind.NewLine))
{ {
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
@ -1816,7 +1821,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
ParseDirectiveBlock(descriptor, parseChildren: (startingBraceLocation) => ParseDirectiveBlock(descriptor, parseChildren: (startingBraceLocation) =>
{ {
NextToken(); NextToken();
Balance(BalancingModes.NoErrorOnFailure, CSharpTokenType.LeftBrace, CSharpTokenType.RightBrace, startingBraceLocation); Balance(BalancingModes.NoErrorOnFailure, SyntaxKind.LeftBrace, SyntaxKind.RightBrace, startingBraceLocation);
Span.ChunkGenerator = new StatementChunkGenerator(); Span.ChunkGenerator = new StatementChunkGenerator();
var existingEditHandler = Span.EditHandler; var existingEditHandler = Span.EditHandler;
Span.EditHandler = new CodeBlockEditHandler(Language.TokenizeString); Span.EditHandler = new CodeBlockEditHandler(Language.TokenizeString);
@ -1868,7 +1873,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnexpectedEOFAfterDirective( RazorDiagnosticFactory.CreateParsing_UnexpectedEOFAfterDirective(
new SourceSpan(CurrentStart, contentLength: 1 /* { */), descriptor.Directive, "{")); new SourceSpan(CurrentStart, contentLength: 1 /* { */), descriptor.Directive, "{"));
} }
else if (!At(CSharpTokenType.LeftBrace)) else if (!At(SyntaxKind.LeftBrace))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral( RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral(
@ -1886,7 +1891,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
parseChildren(startingBraceLocation); parseChildren(startingBraceLocation);
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
if (!Optional(CSharpTokenType.RightBrace)) if (!Optional(SyntaxKind.RightBrace))
{ {
editHandler.AutoCompleteString = "}"; editHandler.AutoCompleteString = "}";
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
@ -1982,7 +1987,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Ex: @addTagHelper "*, Microsoft.AspNetCore.CoolLibrary" // Ex: @addTagHelper "*, Microsoft.AspNetCore.CoolLibrary"
// ^ ^ // ^ ^
// Start End // Start End
if (Span.Tokens.Count == 1 && (Span.Tokens[0] as CSharpToken)?.Type == CSharpTokenType.StringLiteral) if (Span.Tokens.Count == 1 && (Span.Tokens[0] as SyntaxToken)?.Kind == SyntaxKind.StringLiteral)
{ {
offset += Span.Tokens[0].Content.IndexOf(directiveText, StringComparison.Ordinal); offset += Span.Tokens[0].Content.IndexOf(directiveText, StringComparison.Ordinal);
@ -2081,7 +2086,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Conditional("DEBUG")] [Conditional("DEBUG")]
protected void AssertDirective(string directive) protected void AssertDirective(string directive)
{ {
Debug.Assert(CurrentToken.Type == CSharpTokenType.Identifier || CurrentToken.Type == CSharpTokenType.Keyword); Debug.Assert(CurrentToken.Kind == SyntaxKind.Identifier || CurrentToken.Kind == SyntaxKind.Keyword);
Debug.Assert(string.Equals(CurrentToken.Content, directive, StringComparison.Ordinal)); Debug.Assert(string.Equals(CurrentToken.Content, directive, StringComparison.Ordinal));
} }
@ -2108,18 +2113,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex; var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
var foundWhitespace = At(CSharpTokenType.WhiteSpace); var foundWhitespace = At(SyntaxKind.Whitespace);
// If we found whitespace then any content placed within the whitespace MAY cause a destructive change // If we found whitespace then any content placed within the whitespace MAY cause a destructive change
// to the document. We can't accept it. // to the document. We can't accept it.
var acceptedCharacters = foundWhitespace ? AcceptedCharactersInternal.None : AcceptedCharactersInternal.AnyExceptNewline; var acceptedCharacters = foundWhitespace ? AcceptedCharactersInternal.None : AcceptedCharactersInternal.AnyExceptNewline;
Output(SpanKindInternal.MetaCode, acceptedCharacters); Output(SpanKindInternal.MetaCode, acceptedCharacters);
AcceptWhile(CSharpTokenType.WhiteSpace); AcceptWhile(SyntaxKind.Whitespace);
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Markup, acceptedCharacters); Output(SpanKindInternal.Markup, acceptedCharacters);
if (EndOfFile || At(CSharpTokenType.NewLine)) if (EndOfFile || At(SyntaxKind.NewLine))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveMustHaveValue( RazorDiagnosticFactory.CreateParsing_DirectiveMustHaveValue(
@ -2134,7 +2139,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code // Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code
// etc. // etc.
AcceptUntil(CSharpTokenType.NewLine); AcceptUntil(SyntaxKind.NewLine);
// Pull out the value and remove whitespaces and optional quotes // Pull out the value and remove whitespaces and optional quotes
var rawValue = string.Concat(Span.Tokens.Select(s => s.Content)).Trim(); var rawValue = string.Concat(Span.Tokens.Select(s => s.Content)).Trim();
@ -2170,7 +2175,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Start = start; Start = start;
} }
public Block(CSharpToken token, SourceLocation start) public Block(SyntaxToken token, SourceLocation start)
: this(GetName(token), start) : this(GetName(token), start)
{ {
} }
@ -2178,11 +2183,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public string Name { get; set; } public string Name { get; set; }
public SourceLocation Start { get; set; } public SourceLocation Start { get; set; }
private static string GetName(CSharpToken token) private static string GetName(SyntaxToken token)
{ {
if (token.Type == CSharpTokenType.Keyword) var result = CSharpTokenizer.GetTokenKeyword(token);
if (result.HasValue && token.Kind == SyntaxKind.Keyword)
{ {
return CSharpLanguageCharacteristics.GetKeyword(token.Keyword.Value); return CSharpLanguageCharacteristics.GetKeyword(result.Value);
} }
return token.Content; return token.Content;
} }

191
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs
View File

@ -3,64 +3,65 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer, CSharpToken, CSharpTokenType> internal class CSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer>
{ {
private static readonly CSharpLanguageCharacteristics _instance = new CSharpLanguageCharacteristics(); private static readonly CSharpLanguageCharacteristics _instance = new CSharpLanguageCharacteristics();
private static Dictionary<CSharpTokenType, string> _tokenSamples = new Dictionary<CSharpTokenType, string>() private static Dictionary<SyntaxKind, string> _tokenSamples = new Dictionary<SyntaxKind, string>()
{ {
{ CSharpTokenType.Arrow, "->" }, { SyntaxKind.Arrow, "->" },
{ CSharpTokenType.Minus, "-" }, { SyntaxKind.Minus, "-" },
{ CSharpTokenType.Decrement, "--" }, { SyntaxKind.Decrement, "--" },
{ CSharpTokenType.MinusAssign, "-=" }, { SyntaxKind.MinusAssign, "-=" },
{ CSharpTokenType.NotEqual, "!=" }, { SyntaxKind.NotEqual, "!=" },
{ CSharpTokenType.Not, "!" }, { SyntaxKind.Not, "!" },
{ CSharpTokenType.Modulo, "%" }, { SyntaxKind.Modulo, "%" },
{ CSharpTokenType.ModuloAssign, "%=" }, { SyntaxKind.ModuloAssign, "%=" },
{ CSharpTokenType.AndAssign, "&=" }, { SyntaxKind.AndAssign, "&=" },
{ CSharpTokenType.And, "&" }, { SyntaxKind.And, "&" },
{ CSharpTokenType.DoubleAnd, "&&" }, { SyntaxKind.DoubleAnd, "&&" },
{ CSharpTokenType.LeftParenthesis, "(" }, { SyntaxKind.LeftParenthesis, "(" },
{ CSharpTokenType.RightParenthesis, ")" }, { SyntaxKind.RightParenthesis, ")" },
{ CSharpTokenType.Star, "*" }, { SyntaxKind.Star, "*" },
{ CSharpTokenType.MultiplyAssign, "*=" }, { SyntaxKind.MultiplyAssign, "*=" },
{ CSharpTokenType.Comma, "," }, { SyntaxKind.Comma, "," },
{ CSharpTokenType.Dot, "." }, { SyntaxKind.Dot, "." },
{ CSharpTokenType.Slash, "/" }, { SyntaxKind.Slash, "/" },
{ CSharpTokenType.DivideAssign, "/=" }, { SyntaxKind.DivideAssign, "/=" },
{ CSharpTokenType.DoubleColon, "::" }, { SyntaxKind.DoubleColon, "::" },
{ CSharpTokenType.Colon, ":" }, { SyntaxKind.Colon, ":" },
{ CSharpTokenType.Semicolon, ";" }, { SyntaxKind.Semicolon, ";" },
{ CSharpTokenType.QuestionMark, "?" }, { SyntaxKind.QuestionMark, "?" },
{ CSharpTokenType.NullCoalesce, "??" }, { SyntaxKind.NullCoalesce, "??" },
{ CSharpTokenType.RightBracket, "]" }, { SyntaxKind.RightBracket, "]" },
{ CSharpTokenType.LeftBracket, "[" }, { SyntaxKind.LeftBracket, "[" },
{ CSharpTokenType.XorAssign, "^=" }, { SyntaxKind.XorAssign, "^=" },
{ CSharpTokenType.Xor, "^" }, { SyntaxKind.Xor, "^" },
{ CSharpTokenType.LeftBrace, "{" }, { SyntaxKind.LeftBrace, "{" },
{ CSharpTokenType.OrAssign, "|=" }, { SyntaxKind.OrAssign, "|=" },
{ CSharpTokenType.DoubleOr, "||" }, { SyntaxKind.DoubleOr, "||" },
{ CSharpTokenType.Or, "|" }, { SyntaxKind.Or, "|" },
{ CSharpTokenType.RightBrace, "}" }, { SyntaxKind.RightBrace, "}" },
{ CSharpTokenType.Tilde, "~" }, { SyntaxKind.Tilde, "~" },
{ CSharpTokenType.Plus, "+" }, { SyntaxKind.Plus, "+" },
{ CSharpTokenType.PlusAssign, "+=" }, { SyntaxKind.PlusAssign, "+=" },
{ CSharpTokenType.Increment, "++" }, { SyntaxKind.Increment, "++" },
{ CSharpTokenType.LessThan, "<" }, { SyntaxKind.LessThan, "<" },
{ CSharpTokenType.LessThanEqual, "<=" }, { SyntaxKind.LessThanEqual, "<=" },
{ CSharpTokenType.LeftShift, "<<" }, { SyntaxKind.LeftShift, "<<" },
{ CSharpTokenType.LeftShiftAssign, "<<=" }, { SyntaxKind.LeftShiftAssign, "<<=" },
{ CSharpTokenType.Assign, "=" }, { SyntaxKind.Assign, "=" },
{ CSharpTokenType.Equals, "==" }, { SyntaxKind.Equals, "==" },
{ CSharpTokenType.GreaterThan, ">" }, { SyntaxKind.GreaterThan, ">" },
{ CSharpTokenType.GreaterThanEqual, ">=" }, { SyntaxKind.GreaterThanEqual, ">=" },
{ CSharpTokenType.RightShift, ">>" }, { SyntaxKind.RightShift, ">>" },
{ CSharpTokenType.RightShiftAssign, ">>=" }, { SyntaxKind.RightShiftAssign, ">>=" },
{ CSharpTokenType.Hash, "#" }, { SyntaxKind.Hash, "#" },
{ CSharpTokenType.Transition, "@" }, { SyntaxKind.Transition, "@" },
}; };
protected CSharpLanguageCharacteristics() protected CSharpLanguageCharacteristics()
@ -74,35 +75,35 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source); return new CSharpTokenizer(source);
} }
protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList<RazorDiagnostic> errors) protected override SyntaxToken CreateToken(string content, SyntaxKind kind, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new CSharpToken(content, type, errors); return SyntaxFactory.Token(kind, content, errors);
} }
public override string GetSample(CSharpTokenType type) public override string GetSample(SyntaxKind kind)
{ {
string sample; string sample;
if (!_tokenSamples.TryGetValue(type, out sample)) if (!_tokenSamples.TryGetValue(kind, out sample))
{ {
switch (type) switch (kind)
{ {
case CSharpTokenType.Identifier: case SyntaxKind.Identifier:
return Resources.CSharpToken_Identifier; return Resources.CSharpToken_Identifier;
case CSharpTokenType.Keyword: case SyntaxKind.Keyword:
return Resources.CSharpToken_Keyword; return Resources.CSharpToken_Keyword;
case CSharpTokenType.IntegerLiteral: case SyntaxKind.IntegerLiteral:
return Resources.CSharpToken_IntegerLiteral; return Resources.CSharpToken_IntegerLiteral;
case CSharpTokenType.NewLine: case SyntaxKind.NewLine:
return Resources.CSharpToken_Newline; return Resources.CSharpToken_Newline;
case CSharpTokenType.WhiteSpace: case SyntaxKind.Whitespace:
return Resources.CSharpToken_Whitespace; return Resources.CSharpToken_Whitespace;
case CSharpTokenType.Comment: case SyntaxKind.CSharpComment:
return Resources.CSharpToken_Comment; return Resources.CSharpToken_Comment;
case CSharpTokenType.RealLiteral: case SyntaxKind.RealLiteral:
return Resources.CSharpToken_RealLiteral; return Resources.CSharpToken_RealLiteral;
case CSharpTokenType.CharacterLiteral: case SyntaxKind.CharacterLiteral:
return Resources.CSharpToken_CharacterLiteral; return Resources.CSharpToken_CharacterLiteral;
case CSharpTokenType.StringLiteral: case SyntaxKind.StringLiteral:
return Resources.CSharpToken_StringLiteral; return Resources.CSharpToken_StringLiteral;
default: default:
return Resources.Token_Unknown; return Resources.Token_Unknown;
@ -111,59 +112,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return sample; return sample;
} }
public override CSharpToken CreateMarkerToken() public override SyntaxToken CreateMarkerToken()
{ {
return new CSharpToken(string.Empty, CSharpTokenType.Unknown); return SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
} }
public override CSharpTokenType GetKnownTokenType(KnownTokenType type) public override SyntaxKind GetKnownTokenType(KnownTokenType type)
{ {
switch (type) switch (type)
{ {
case KnownTokenType.Identifier: case KnownTokenType.Identifier:
return CSharpTokenType.Identifier; return SyntaxKind.Identifier;
case KnownTokenType.Keyword: case KnownTokenType.Keyword:
return CSharpTokenType.Keyword; return SyntaxKind.Keyword;
case KnownTokenType.NewLine: case KnownTokenType.NewLine:
return CSharpTokenType.NewLine; return SyntaxKind.NewLine;
case KnownTokenType.WhiteSpace: case KnownTokenType.WhiteSpace:
return CSharpTokenType.WhiteSpace; return SyntaxKind.Whitespace;
case KnownTokenType.Transition: case KnownTokenType.Transition:
return CSharpTokenType.Transition; return SyntaxKind.Transition;
case KnownTokenType.CommentStart: case KnownTokenType.CommentStart:
return CSharpTokenType.RazorCommentTransition; return SyntaxKind.RazorCommentTransition;
case KnownTokenType.CommentStar: case KnownTokenType.CommentStar:
return CSharpTokenType.RazorCommentStar; return SyntaxKind.RazorCommentStar;
case KnownTokenType.CommentBody: case KnownTokenType.CommentBody:
return CSharpTokenType.RazorComment; return SyntaxKind.RazorComment;
default: default:
return CSharpTokenType.Unknown; return SyntaxKind.Unknown;
} }
} }
public override CSharpTokenType FlipBracket(CSharpTokenType bracket) public override SyntaxKind FlipBracket(SyntaxKind bracket)
{ {
switch (bracket) switch (bracket)
{ {
case CSharpTokenType.LeftBrace: case SyntaxKind.LeftBrace:
return CSharpTokenType.RightBrace; return SyntaxKind.RightBrace;
case CSharpTokenType.LeftBracket: case SyntaxKind.LeftBracket:
return CSharpTokenType.RightBracket; return SyntaxKind.RightBracket;
case CSharpTokenType.LeftParenthesis: case SyntaxKind.LeftParenthesis:
return CSharpTokenType.RightParenthesis; return SyntaxKind.RightParenthesis;
case CSharpTokenType.LessThan: case SyntaxKind.LessThan:
return CSharpTokenType.GreaterThan; return SyntaxKind.GreaterThan;
case CSharpTokenType.RightBrace: case SyntaxKind.RightBrace:
return CSharpTokenType.LeftBrace; return SyntaxKind.LeftBrace;
case CSharpTokenType.RightBracket: case SyntaxKind.RightBracket:
return CSharpTokenType.LeftBracket; return SyntaxKind.LeftBracket;
case CSharpTokenType.RightParenthesis: case SyntaxKind.RightParenthesis:
return CSharpTokenType.LeftParenthesis; return SyntaxKind.LeftParenthesis;
case CSharpTokenType.GreaterThan: case SyntaxKind.GreaterThan:
return CSharpTokenType.LessThan; return SyntaxKind.LessThan;
default: default:
Debug.Fail("FlipBracket must be called with a bracket character"); Debug.Fail("FlipBracket must be called with a bracket character");
return CSharpTokenType.Unknown; return SyntaxKind.Unknown;
} }
} }

60
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpToken.cs
View File

@ -1,60 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CSharpToken : TokenBase<CSharpTokenType>
{
public CSharpToken(
string content,
CSharpTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpToken(
string content,
CSharpTokenType type,
IReadOnlyList<RazorDiagnostic> errors)
: base(content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpKeyword? Keyword { get; set; }
protected override SyntaxToken GetSyntaxToken()
{
switch (Type)
{
default:
return SyntaxFactory.UnknownToken(Content, Errors.ToArray());
}
}
public override bool Equals(object obj)
{
var other = obj as CSharpToken;
return base.Equals(other) &&
other.Keyword == Keyword;
}
public override int GetHashCode()
{
// Hash code should include only immutable properties.
return base.GetHashCode();
}
}
}

75
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenType.cs
View File

@ -1,75 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal enum CSharpTokenType
{
Unknown,
Identifier,
Keyword,
IntegerLiteral,
NewLine,
WhiteSpace,
Comment,
RealLiteral,
CharacterLiteral,
StringLiteral,
// Operators
Arrow,
Minus,
Decrement,
MinusAssign,
NotEqual,
Not,
Modulo,
ModuloAssign,
AndAssign,
And,
DoubleAnd,
LeftParenthesis,
RightParenthesis,
Star,
MultiplyAssign,
Comma,
Dot,
Slash,
DivideAssign,
DoubleColon,
Colon,
Semicolon,
QuestionMark,
NullCoalesce,
RightBracket,
LeftBracket,
XorAssign,
Xor,
LeftBrace,
OrAssign,
DoubleOr,
Or,
RightBrace,
Tilde,
Plus,
PlusAssign,
Increment,
LessThan,
LessThanEqual,
LeftShift,
LeftShiftAssign,
Assign,
Equals,
GreaterThan,
GreaterThanEqual,
RightShift,
RightShiftAssign,
Hash,
Transition,
// Razor specific
RazorCommentTransition,
RazorCommentStar,
RazorComment
}
}

260
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs
View File

@ -5,12 +5,13 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Globalization; using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CSharpTokenizer : Tokenizer<CSharpToken, CSharpTokenType> internal class CSharpTokenizer : Tokenizer
{ {
private Dictionary<char, Func<CSharpTokenType>> _operatorHandlers; private Dictionary<char, Func<SyntaxKind>> _operatorHandlers;
private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal) private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal)
{ {
@ -100,31 +101,31 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
base.CurrentState = StartState; base.CurrentState = StartState;
_operatorHandlers = new Dictionary<char, Func<CSharpTokenType>>() _operatorHandlers = new Dictionary<char, Func<SyntaxKind>>()
{ {
{ '-', MinusOperator }, { '-', MinusOperator },
{ '<', LessThanOperator }, { '<', LessThanOperator },
{ '>', GreaterThanOperator }, { '>', GreaterThanOperator },
{ '&', CreateTwoCharOperatorHandler(CSharpTokenType.And, '=', CSharpTokenType.AndAssign, '&', CSharpTokenType.DoubleAnd) }, { '&', CreateTwoCharOperatorHandler(SyntaxKind.And, '=', SyntaxKind.AndAssign, '&', SyntaxKind.DoubleAnd) },
{ '|', CreateTwoCharOperatorHandler(CSharpTokenType.Or, '=', CSharpTokenType.OrAssign, '|', CSharpTokenType.DoubleOr) }, { '|', CreateTwoCharOperatorHandler(SyntaxKind.Or, '=', SyntaxKind.OrAssign, '|', SyntaxKind.DoubleOr) },
{ '+', CreateTwoCharOperatorHandler(CSharpTokenType.Plus, '=', CSharpTokenType.PlusAssign, '+', CSharpTokenType.Increment) }, { '+', CreateTwoCharOperatorHandler(SyntaxKind.Plus, '=', SyntaxKind.PlusAssign, '+', SyntaxKind.Increment) },
{ '=', CreateTwoCharOperatorHandler(CSharpTokenType.Assign, '=', CSharpTokenType.Equals, '>', CSharpTokenType.GreaterThanEqual) }, { '=', CreateTwoCharOperatorHandler(SyntaxKind.Assign, '=', SyntaxKind.Equals, '>', SyntaxKind.GreaterThanEqual) },
{ '!', CreateTwoCharOperatorHandler(CSharpTokenType.Not, '=', CSharpTokenType.NotEqual) }, { '!', CreateTwoCharOperatorHandler(SyntaxKind.Not, '=', SyntaxKind.NotEqual) },
{ '%', CreateTwoCharOperatorHandler(CSharpTokenType.Modulo, '=', CSharpTokenType.ModuloAssign) }, { '%', CreateTwoCharOperatorHandler(SyntaxKind.Modulo, '=', SyntaxKind.ModuloAssign) },
{ '*', CreateTwoCharOperatorHandler(CSharpTokenType.Star, '=', CSharpTokenType.MultiplyAssign) }, { '*', CreateTwoCharOperatorHandler(SyntaxKind.Star, '=', SyntaxKind.MultiplyAssign) },
{ ':', CreateTwoCharOperatorHandler(CSharpTokenType.Colon, ':', CSharpTokenType.DoubleColon) }, { ':', CreateTwoCharOperatorHandler(SyntaxKind.Colon, ':', SyntaxKind.DoubleColon) },
{ '?', CreateTwoCharOperatorHandler(CSharpTokenType.QuestionMark, '?', CSharpTokenType.NullCoalesce) }, { '?', CreateTwoCharOperatorHandler(SyntaxKind.QuestionMark, '?', SyntaxKind.NullCoalesce) },
{ '^', CreateTwoCharOperatorHandler(CSharpTokenType.Xor, '=', CSharpTokenType.XorAssign) }, { '^', CreateTwoCharOperatorHandler(SyntaxKind.Xor, '=', SyntaxKind.XorAssign) },
{ '(', () => CSharpTokenType.LeftParenthesis }, { '(', () => SyntaxKind.LeftParenthesis },
{ ')', () => CSharpTokenType.RightParenthesis }, { ')', () => SyntaxKind.RightParenthesis },
{ '{', () => CSharpTokenType.LeftBrace }, { '{', () => SyntaxKind.LeftBrace },
{ '}', () => CSharpTokenType.RightBrace }, { '}', () => SyntaxKind.RightBrace },
{ '[', () => CSharpTokenType.LeftBracket }, { '[', () => SyntaxKind.LeftBracket },
{ ']', () => CSharpTokenType.RightBracket }, { ']', () => SyntaxKind.RightBracket },
{ ',', () => CSharpTokenType.Comma }, { ',', () => SyntaxKind.Comma },
{ ';', () => CSharpTokenType.Semicolon }, { ';', () => SyntaxKind.Semicolon },
{ '~', () => CSharpTokenType.Tilde }, { '~', () => SyntaxKind.Tilde },
{ '#', () => CSharpTokenType.Hash } { '#', () => SyntaxKind.Hash }
}; };
} }
@ -132,11 +133,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState; private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState;
public override CSharpTokenType RazorCommentType => CSharpTokenType.RazorComment; public override SyntaxKind RazorCommentKind => SyntaxKind.RazorComment;
public override CSharpTokenType RazorCommentTransitionType => CSharpTokenType.RazorCommentTransition; public override SyntaxKind RazorCommentTransitionKind => SyntaxKind.RazorCommentTransition;
public override CSharpTokenType RazorCommentStarType => CSharpTokenType.RazorCommentStar; public override SyntaxKind RazorCommentStarKind => SyntaxKind.RazorCommentStar;
protected override StateResult Dispatch() protected override StateResult Dispatch()
{ {
@ -169,7 +170,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Optimize memory allocation by returning constants for the most frequent cases // Optimize memory allocation by returning constants for the most frequent cases
protected override string GetTokenContent(CSharpTokenType type) protected override string GetTokenContent(SyntaxKind type)
{ {
var tokenLength = Buffer.Length; var tokenLength = Buffer.Length;
@ -177,7 +178,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
switch (type) switch (type)
{ {
case CSharpTokenType.IntegerLiteral: case SyntaxKind.IntegerLiteral:
switch (Buffer[0]) switch (Buffer[0])
{ {
case '0': case '0':
@ -202,13 +203,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "9"; return "9";
} }
break; break;
case CSharpTokenType.NewLine: case SyntaxKind.NewLine:
if (Buffer[0] == '\n') if (Buffer[0] == '\n')
{ {
return "\n"; return "\n";
} }
break; break;
case CSharpTokenType.WhiteSpace: case SyntaxKind.Whitespace:
if (Buffer[0] == ' ') if (Buffer[0] == ' ')
{ {
return " "; return " ";
@ -218,57 +219,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t"; return "\t";
} }
break; break;
case CSharpTokenType.Minus: case SyntaxKind.Minus:
return "-"; return "-";
case CSharpTokenType.Not: case SyntaxKind.Not:
return "!"; return "!";
case CSharpTokenType.Modulo: case SyntaxKind.Modulo:
return "%"; return "%";
case CSharpTokenType.And: case SyntaxKind.And:
return "&"; return "&";
case CSharpTokenType.LeftParenthesis: case SyntaxKind.LeftParenthesis:
return "("; return "(";
case CSharpTokenType.RightParenthesis: case SyntaxKind.RightParenthesis:
return ")"; return ")";
case CSharpTokenType.Star: case SyntaxKind.Star:
return "*"; return "*";
case CSharpTokenType.Comma: case SyntaxKind.Comma:
return ","; return ",";
case CSharpTokenType.Dot: case SyntaxKind.Dot:
return "."; return ".";
case CSharpTokenType.Slash: case SyntaxKind.Slash:
return "/"; return "/";
case CSharpTokenType.Colon: case SyntaxKind.Colon:
return ":"; return ":";
case CSharpTokenType.Semicolon: case SyntaxKind.Semicolon:
return ";"; return ";";
case CSharpTokenType.QuestionMark: case SyntaxKind.QuestionMark:
return "?"; return "?";
case CSharpTokenType.RightBracket: case SyntaxKind.RightBracket:
return "]"; return "]";
case CSharpTokenType.LeftBracket: case SyntaxKind.LeftBracket:
return "["; return "[";
case CSharpTokenType.Xor: case SyntaxKind.Xor:
return "^"; return "^";
case CSharpTokenType.LeftBrace: case SyntaxKind.LeftBrace:
return "{"; return "{";
case CSharpTokenType.Or: case SyntaxKind.Or:
return "|"; return "|";
case CSharpTokenType.RightBrace: case SyntaxKind.RightBrace:
return "}"; return "}";
case CSharpTokenType.Tilde: case SyntaxKind.Tilde:
return "~"; return "~";
case CSharpTokenType.Plus: case SyntaxKind.Plus:
return "+"; return "+";
case CSharpTokenType.LessThan: case SyntaxKind.LessThan:
return "<"; return "<";
case CSharpTokenType.Assign: case SyntaxKind.Assign:
return "="; return "=";
case CSharpTokenType.GreaterThan: case SyntaxKind.GreaterThan:
return ">"; return ">";
case CSharpTokenType.Hash: case SyntaxKind.Hash:
return "#"; return "#";
case CSharpTokenType.Transition: case SyntaxKind.Transition:
return "@"; return "@";
} }
@ -277,53 +278,53 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
switch (type) switch (type)
{ {
case CSharpTokenType.NewLine: case SyntaxKind.NewLine:
return "\r\n"; return "\r\n";
case CSharpTokenType.Arrow: case SyntaxKind.Arrow:
return "->"; return "->";
case CSharpTokenType.Decrement: case SyntaxKind.Decrement:
return "--"; return "--";
case CSharpTokenType.MinusAssign: case SyntaxKind.MinusAssign:
return "-="; return "-=";
case CSharpTokenType.NotEqual: case SyntaxKind.NotEqual:
return "!="; return "!=";
case CSharpTokenType.ModuloAssign: case SyntaxKind.ModuloAssign:
return "%="; return "%=";
case CSharpTokenType.AndAssign: case SyntaxKind.AndAssign:
return "&="; return "&=";
case CSharpTokenType.DoubleAnd: case SyntaxKind.DoubleAnd:
return "&&"; return "&&";
case CSharpTokenType.MultiplyAssign: case SyntaxKind.MultiplyAssign:
return "*="; return "*=";
case CSharpTokenType.DivideAssign: case SyntaxKind.DivideAssign:
return "/="; return "/=";
case CSharpTokenType.DoubleColon: case SyntaxKind.DoubleColon:
return "::"; return "::";
case CSharpTokenType.NullCoalesce: case SyntaxKind.NullCoalesce:
return "??"; return "??";
case CSharpTokenType.XorAssign: case SyntaxKind.XorAssign:
return "^="; return "^=";
case CSharpTokenType.OrAssign: case SyntaxKind.OrAssign:
return "|="; return "|=";
case CSharpTokenType.DoubleOr: case SyntaxKind.DoubleOr:
return "||"; return "||";
case CSharpTokenType.PlusAssign: case SyntaxKind.PlusAssign:
return "+="; return "+=";
case CSharpTokenType.Increment: case SyntaxKind.Increment:
return "++"; return "++";
case CSharpTokenType.LessThanEqual: case SyntaxKind.LessThanEqual:
return "<="; return "<=";
case CSharpTokenType.LeftShift: case SyntaxKind.LeftShift:
return "<<"; return "<<";
case CSharpTokenType.Equals: case SyntaxKind.Equals:
return "=="; return "==";
case CSharpTokenType.GreaterThanEqual: case SyntaxKind.GreaterThanEqual:
if (Buffer[0] == '=') if (Buffer[0] == '=')
{ {
return "=>"; return "=>";
} }
return ">="; return ">=";
case CSharpTokenType.RightShift: case SyntaxKind.RightShift:
return ">>"; return ">>";
@ -333,9 +334,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
switch (type) switch (type)
{ {
case CSharpTokenType.LeftShiftAssign: case SyntaxKind.LeftShiftAssign:
return "<<="; return "<<=";
case CSharpTokenType.RightShiftAssign: case SyntaxKind.RightShiftAssign:
return ">>="; return ">>=";
} }
} }
@ -343,9 +344,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return base.GetTokenContent(type); return base.GetTokenContent(type);
} }
protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList<RazorDiagnostic> errors) protected override SyntaxToken CreateToken(string content, SyntaxKind kind, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new CSharpToken(content, type, errors); return SyntaxFactory.Token(kind, content, errors);
} }
private StateResult Data() private StateResult Data()
@ -359,13 +360,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); TakeCurrent();
} }
return Stay(EndToken(CSharpTokenType.NewLine)); return Stay(EndToken(SyntaxKind.NewLine));
} }
else if (ParserHelpers.IsWhitespace(CurrentCharacter)) else if (ParserHelpers.IsWhitespace(CurrentCharacter))
{ {
// CSharp Spec §2.3.3 // CSharp Spec §2.3.3
TakeUntil(c => !ParserHelpers.IsWhitespace(c)); TakeUntil(c => !ParserHelpers.IsWhitespace(c));
return Stay(EndToken(CSharpTokenType.WhiteSpace)); return Stay(EndToken(SyntaxKind.Whitespace));
} }
else if (IsIdentifierStart(CurrentCharacter)) else if (IsIdentifierStart(CurrentCharacter))
{ {
@ -390,7 +391,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return RealLiteral(); return RealLiteral();
} }
return Stay(Single(CSharpTokenType.Dot)); return Stay(Single(SyntaxKind.Dot));
case '/': case '/':
TakeCurrent(); TakeCurrent();
if (CurrentCharacter == '/') if (CurrentCharacter == '/')
@ -406,11 +407,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else if (CurrentCharacter == '=') else if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return Stay(EndToken(CSharpTokenType.DivideAssign)); return Stay(EndToken(SyntaxKind.DivideAssign));
} }
else else
{ {
return Stay(EndToken(CSharpTokenType.Slash)); return Stay(EndToken(SyntaxKind.Slash));
} }
default: default:
return Stay(EndToken(Operator())); return Stay(EndToken(Operator()));
@ -429,78 +430,78 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return Transition( return Transition(
CSharpTokenizerState.AfterRazorCommentTransition, CSharpTokenizerState.AfterRazorCommentTransition,
EndToken(CSharpTokenType.RazorCommentTransition)); EndToken(SyntaxKind.RazorCommentTransition));
} }
else if (CurrentCharacter == '@') else if (CurrentCharacter == '@')
{ {
// Could be escaped comment transition // Could be escaped comment transition
return Transition( return Transition(
CSharpTokenizerState.EscapedRazorCommentTransition, CSharpTokenizerState.EscapedRazorCommentTransition,
EndToken(CSharpTokenType.Transition)); EndToken(SyntaxKind.Transition));
} }
return Stay(EndToken(CSharpTokenType.Transition)); return Stay(EndToken(SyntaxKind.Transition));
} }
private StateResult EscapedRazorCommentTransition() private StateResult EscapedRazorCommentTransition()
{ {
TakeCurrent(); TakeCurrent();
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Transition)); return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.Transition));
} }
private CSharpTokenType Operator() private SyntaxKind Operator()
{ {
var first = CurrentCharacter; var first = CurrentCharacter;
TakeCurrent(); TakeCurrent();
Func<CSharpTokenType> handler; Func<SyntaxKind> handler;
if (_operatorHandlers.TryGetValue(first, out handler)) if (_operatorHandlers.TryGetValue(first, out handler))
{ {
return handler(); return handler();
} }
return CSharpTokenType.Unknown; return SyntaxKind.Unknown;
} }
private CSharpTokenType LessThanOperator() private SyntaxKind LessThanOperator()
{ {
if (CurrentCharacter == '=') if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return CSharpTokenType.LessThanEqual; return SyntaxKind.LessThanEqual;
} }
return CSharpTokenType.LessThan; return SyntaxKind.LessThan;
} }
private CSharpTokenType GreaterThanOperator() private SyntaxKind GreaterThanOperator()
{ {
if (CurrentCharacter == '=') if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return CSharpTokenType.GreaterThanEqual; return SyntaxKind.GreaterThanEqual;
} }
return CSharpTokenType.GreaterThan; return SyntaxKind.GreaterThan;
} }
private CSharpTokenType MinusOperator() private SyntaxKind MinusOperator()
{ {
if (CurrentCharacter == '>') if (CurrentCharacter == '>')
{ {
TakeCurrent(); TakeCurrent();
return CSharpTokenType.Arrow; return SyntaxKind.Arrow;
} }
else if (CurrentCharacter == '-') else if (CurrentCharacter == '-')
{ {
TakeCurrent(); TakeCurrent();
return CSharpTokenType.Decrement; return SyntaxKind.Decrement;
} }
else if (CurrentCharacter == '=') else if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return CSharpTokenType.MinusAssign; return SyntaxKind.MinusAssign;
} }
return CSharpTokenType.Minus; return SyntaxKind.Minus;
} }
private Func<CSharpTokenType> CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char second, CSharpTokenType typeIfBoth) private Func<SyntaxKind> CreateTwoCharOperatorHandler(SyntaxKind typeIfOnlyFirst, char second, SyntaxKind typeIfBoth)
{ {
return () => return () =>
{ {
@ -513,7 +514,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}; };
} }
private Func<CSharpTokenType> CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char option1, CSharpTokenType typeIfOption1, char option2, CSharpTokenType typeIfOption2) private Func<SyntaxKind> CreateTwoCharOperatorHandler(SyntaxKind typeIfOnlyFirst, char option1, SyntaxKind typeIfOption1, char option2, SyntaxKind typeIfOption2)
{ {
return () => return () =>
{ {
@ -550,14 +551,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral( RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */))); new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
} }
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.StringLiteral)); return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.StringLiteral));
} }
private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', CSharpTokenType.CharacterLiteral); private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', SyntaxKind.CharacterLiteral);
private StateResult QuotedStringLiteral() => QuotedLiteral('\"', CSharpTokenType.StringLiteral); private StateResult QuotedStringLiteral() => QuotedLiteral('\"', SyntaxKind.StringLiteral);
private StateResult QuotedLiteral(char quote, CSharpTokenType literalType) private StateResult QuotedLiteral(char quote, SyntaxKind literalType)
{ {
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c)); TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\') if (CurrentCharacter == '\\')
@ -594,7 +595,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated( RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */))); new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment)); return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment));
} }
if (CurrentCharacter == '*') if (CurrentCharacter == '*')
{ {
@ -602,7 +603,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (CurrentCharacter == '/') if (CurrentCharacter == '/')
{ {
TakeCurrent(); TakeCurrent();
return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment)); return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment));
} }
} }
return Stay(); return Stay();
@ -612,7 +613,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult SingleLineComment() private StateResult SingleLineComment()
{ {
TakeUntil(c => ParserHelpers.IsNewLine(c)); TakeUntil(c => ParserHelpers.IsNewLine(c));
return Stay(EndToken(CSharpTokenType.Comment)); return Stay(EndToken(SyntaxKind.CSharpComment));
} }
// CSharp Spec §2.4.4 // CSharp Spec §2.4.4
@ -632,7 +633,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeUntil(c => !IsHexDigit(c)); TakeUntil(c => !IsHexDigit(c));
TakeIntegerSuffix(); TakeIntegerSuffix();
return Stay(EndToken(CSharpTokenType.IntegerLiteral)); return Stay(EndToken(SyntaxKind.IntegerLiteral));
} }
private StateResult DecimalLiteral() private StateResult DecimalLiteral()
@ -650,7 +651,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else else
{ {
TakeIntegerSuffix(); TakeIntegerSuffix();
return Stay(EndToken(CSharpTokenType.IntegerLiteral)); return Stay(EndToken(SyntaxKind.IntegerLiteral));
} }
} }
@ -669,7 +670,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); TakeCurrent();
} }
return Stay(EndToken(CSharpTokenType.RealLiteral)); return Stay(EndToken(SyntaxKind.RealLiteral));
} }
// CSharp Spec §2.4.4.3 // CSharp Spec §2.4.4.3
@ -708,21 +709,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Debug.Assert(IsIdentifierStart(CurrentCharacter)); Debug.Assert(IsIdentifierStart(CurrentCharacter));
TakeCurrent(); TakeCurrent();
TakeUntil(c => !IsIdentifierPart(c)); TakeUntil(c => !IsIdentifierPart(c));
CSharpToken token = null; SyntaxToken token = null;
if (HaveContent) if (HaveContent)
{ {
CSharpKeyword keyword; CSharpKeyword keyword;
var type = CSharpTokenType.Identifier; var type = SyntaxKind.Identifier;
var tokenContent = Buffer.ToString(); var tokenContent = Buffer.ToString();
if (_keywords.TryGetValue(tokenContent, out keyword)) if (_keywords.TryGetValue(tokenContent, out keyword))
{ {
type = CSharpTokenType.Keyword; type = SyntaxKind.Keyword;
} }
token = new CSharpToken(tokenContent, type) token = SyntaxFactory.Token(type, tokenContent);
{
Keyword = type == CSharpTokenType.Keyword ? (CSharpKeyword?)keyword : null,
};
Buffer.Clear(); Buffer.Clear();
CurrentErrors.Clear(); CurrentErrors.Clear();
@ -736,7 +734,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null); return Transition((int)state, result: null);
} }
private StateResult Transition(CSharpTokenizerState state, CSharpToken result) private StateResult Transition(CSharpTokenizerState state, SyntaxToken result)
{ {
return Transition((int)state, result); return Transition((int)state, result);
} }
@ -780,6 +778,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return (value >= '0' && value <= '9') || (value >= 'A' && value <= 'F') || (value >= 'a' && value <= 'f'); return (value >= '0' && value <= '9') || (value >= 'A' && value <= 'F') || (value >= 'a' && value <= 'f');
} }
internal static CSharpKeyword? GetTokenKeyword(SyntaxToken token)
{
if (token != null && _keywords.TryGetValue(token.Content, out var keyword))
{
return keyword;
}
return null;
}
private enum CSharpTokenizerState private enum CSharpTokenizerState
{ {
Data, Data,

3
src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs
View File

@ -4,12 +4,13 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization; using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CodeBlockEditHandler : SpanEditHandler internal class CodeBlockEditHandler : SpanEditHandler
{ {
public CodeBlockEditHandler(Func<string, IEnumerable<IToken>> tokenizer) : base(tokenizer) public CodeBlockEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer) : base(tokenizer)
{ {
} }

23
src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Diagnostics; using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
@ -17,11 +18,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected override StateResult Dispatch() protected override StateResult Dispatch()
{ {
var result = base.Dispatch(); var result = base.Dispatch();
if (result.Result != null && !_visitedFirstTokenStart && IsValidTokenType(result.Result.Type)) if (result.Result != null && !_visitedFirstTokenStart && IsValidTokenType(result.Result.Kind))
{ {
_visitedFirstTokenStart = true; _visitedFirstTokenStart = true;
} }
else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpTokenType.NewLine) else if (result.Result != null && _visitedFirstTokenStart && result.Result.Kind == SyntaxKind.NewLine)
{ {
_visitedFirstTokenLineEnd = true; _visitedFirstTokenLineEnd = true;
} }
@ -29,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result; return result;
} }
public override CSharpToken NextToken() public override SyntaxToken NextToken()
{ {
// Post-Condition: Buffer should be empty at the start of Next() // Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
@ -48,15 +49,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token; return token;
} }
private bool IsValidTokenType(CSharpTokenType type) private bool IsValidTokenType(SyntaxKind kind)
{ {
return type != CSharpTokenType.WhiteSpace && return kind != SyntaxKind.Whitespace &&
type != CSharpTokenType.NewLine && kind != SyntaxKind.NewLine &&
type != CSharpTokenType.Comment && kind != SyntaxKind.CSharpComment &&
type != CSharpTokenType.RazorComment && kind != SyntaxKind.RazorComment &&
type != CSharpTokenType.RazorCommentStar && kind != SyntaxKind.RazorCommentStar &&
type != CSharpTokenType.RazorCommentTransition && kind != SyntaxKind.RazorCommentTransition &&
type != CSharpTokenType.Transition; kind != SyntaxKind.Transition;
} }
} }
} }

19
src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Diagnostics; using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
@ -16,7 +17,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected override StateResult Dispatch() protected override StateResult Dispatch()
{ {
var result = base.Dispatch(); var result = base.Dispatch();
if (result.Result != null && IsValidTokenType(result.Result.Type)) if (result.Result != null && IsValidTokenType(result.Result.Kind))
{ {
_visitedFirstTokenStart = true; _visitedFirstTokenStart = true;
} }
@ -24,7 +25,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result; return result;
} }
public override HtmlToken NextToken() public override SyntaxToken NextToken()
{ {
// Post-Condition: Buffer should be empty at the start of Next() // Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
@ -43,14 +44,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token; return token;
} }
private bool IsValidTokenType(HtmlTokenType type) private bool IsValidTokenType(SyntaxKind kind)
{ {
return type != HtmlTokenType.WhiteSpace && return kind != SyntaxKind.Whitespace &&
type != HtmlTokenType.NewLine && kind != SyntaxKind.NewLine &&
type != HtmlTokenType.RazorComment && kind != SyntaxKind.RazorComment &&
type != HtmlTokenType.RazorCommentStar && kind != SyntaxKind.RazorCommentStar &&
type != HtmlTokenType.RazorCommentTransition && kind != SyntaxKind.RazorCommentTransition &&
type != HtmlTokenType.Transition; kind != SyntaxKind.Transition;
} }
} }
} }

91
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
View File

@ -3,10 +3,11 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class HtmlLanguageCharacteristics : LanguageCharacteristics<HtmlTokenizer, HtmlToken, HtmlTokenType> internal class HtmlLanguageCharacteristics : LanguageCharacteristics<HtmlTokenizer>
{ {
private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics(); private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics();
@ -19,47 +20,47 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _instance; } get { return _instance; }
} }
public override string GetSample(HtmlTokenType type) public override string GetSample(SyntaxKind type)
{ {
switch (type) switch (type)
{ {
case HtmlTokenType.Text: case SyntaxKind.HtmlTextLiteral:
return Resources.HtmlToken_Text; return Resources.HtmlToken_Text;
case HtmlTokenType.WhiteSpace: case SyntaxKind.Whitespace:
return Resources.HtmlToken_WhiteSpace; return Resources.HtmlToken_WhiteSpace;
case HtmlTokenType.NewLine: case SyntaxKind.NewLine:
return Resources.HtmlToken_NewLine; return Resources.HtmlToken_NewLine;
case HtmlTokenType.OpenAngle: case SyntaxKind.OpenAngle:
return "<"; return "<";
case HtmlTokenType.Bang: case SyntaxKind.Bang:
return "!"; return "!";
case HtmlTokenType.ForwardSlash: case SyntaxKind.ForwardSlash:
return "/"; return "/";
case HtmlTokenType.QuestionMark: case SyntaxKind.QuestionMark:
return "?"; return "?";
case HtmlTokenType.DoubleHyphen: case SyntaxKind.DoubleHyphen:
return "--"; return "--";
case HtmlTokenType.LeftBracket: case SyntaxKind.LeftBracket:
return "["; return "[";
case HtmlTokenType.CloseAngle: case SyntaxKind.CloseAngle:
return ">"; return ">";
case HtmlTokenType.RightBracket: case SyntaxKind.RightBracket:
return "]"; return "]";
case HtmlTokenType.Equals: case SyntaxKind.Equals:
return "="; return "=";
case HtmlTokenType.DoubleQuote: case SyntaxKind.DoubleQuote:
return "\""; return "\"";
case HtmlTokenType.SingleQuote: case SyntaxKind.SingleQuote:
return "'"; return "'";
case HtmlTokenType.Transition: case SyntaxKind.Transition:
return "@"; return "@";
case HtmlTokenType.Colon: case SyntaxKind.Colon:
return ":"; return ":";
case HtmlTokenType.RazorComment: case SyntaxKind.RazorComment:
return Resources.HtmlToken_RazorComment; return Resources.HtmlToken_RazorComment;
case HtmlTokenType.RazorCommentStar: case SyntaxKind.RazorCommentStar:
return "*"; return "*";
case HtmlTokenType.RazorCommentTransition: case SyntaxKind.RazorCommentTransition:
return "@"; return "@";
default: default:
return Resources.Token_Unknown; return Resources.Token_Unknown;
@ -71,57 +72,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source); return new HtmlTokenizer(source);
} }
public override HtmlTokenType FlipBracket(HtmlTokenType bracket) public override SyntaxKind FlipBracket(SyntaxKind bracket)
{ {
switch (bracket) switch (bracket)
{ {
case HtmlTokenType.LeftBracket: case SyntaxKind.LeftBracket:
return HtmlTokenType.RightBracket; return SyntaxKind.RightBracket;
case HtmlTokenType.OpenAngle: case SyntaxKind.OpenAngle:
return HtmlTokenType.CloseAngle; return SyntaxKind.CloseAngle;
case HtmlTokenType.RightBracket: case SyntaxKind.RightBracket:
return HtmlTokenType.LeftBracket; return SyntaxKind.LeftBracket;
case HtmlTokenType.CloseAngle: case SyntaxKind.CloseAngle:
return HtmlTokenType.OpenAngle; return SyntaxKind.OpenAngle;
default: default:
Debug.Fail("FlipBracket must be called with a bracket character"); Debug.Fail("FlipBracket must be called with a bracket character");
return HtmlTokenType.Unknown; return SyntaxKind.Unknown;
} }
} }
public override HtmlToken CreateMarkerToken() public override SyntaxToken CreateMarkerToken()
{ {
return new HtmlToken(string.Empty, HtmlTokenType.Unknown); return SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
} }
public override HtmlTokenType GetKnownTokenType(KnownTokenType type) public override SyntaxKind GetKnownTokenType(KnownTokenType type)
{ {
switch (type) switch (type)
{ {
case KnownTokenType.CommentStart: case KnownTokenType.CommentStart:
return HtmlTokenType.RazorCommentTransition; return SyntaxKind.RazorCommentTransition;
case KnownTokenType.CommentStar: case KnownTokenType.CommentStar:
return HtmlTokenType.RazorCommentStar; return SyntaxKind.RazorCommentStar;
case KnownTokenType.CommentBody: case KnownTokenType.CommentBody:
return HtmlTokenType.RazorComment; return SyntaxKind.RazorComment;
case KnownTokenType.Identifier: case KnownTokenType.Identifier:
return HtmlTokenType.Text; return SyntaxKind.HtmlTextLiteral;
case KnownTokenType.Keyword: case KnownTokenType.Keyword:
return HtmlTokenType.Text; return SyntaxKind.HtmlTextLiteral;
case KnownTokenType.NewLine: case KnownTokenType.NewLine:
return HtmlTokenType.NewLine; return SyntaxKind.NewLine;
case KnownTokenType.Transition: case KnownTokenType.Transition:
return HtmlTokenType.Transition; return SyntaxKind.Transition;
case KnownTokenType.WhiteSpace: case KnownTokenType.WhiteSpace:
return HtmlTokenType.WhiteSpace; return SyntaxKind.Whitespace;
default: default:
return HtmlTokenType.Unknown; return SyntaxKind.Unknown;
} }
} }
protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList<RazorDiagnostic> errors) protected override SyntaxToken CreateToken(string content, SyntaxKind kind, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new HtmlToken(content, type, errors); return SyntaxFactory.Token(kind, content, errors);
} }
} }
} }

458
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
View File

File diff suppressed because it is too large Load Diff

77
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlToken.cs
View File

@ -1,77 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class HtmlToken : TokenBase<HtmlTokenType>
{
internal static readonly HtmlToken Hyphen = new HtmlToken("-", HtmlTokenType.Text);
public HtmlToken(string content, HtmlTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public HtmlToken(
string content,
HtmlTokenType type,
IReadOnlyList<RazorDiagnostic> errors)
: base(content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
protected override SyntaxToken GetSyntaxToken()
{
switch (Type)
{
case HtmlTokenType.Text:
return SyntaxFactory.HtmlTextToken(Content, Errors.ToArray());
case HtmlTokenType.WhiteSpace:
return SyntaxFactory.WhitespaceToken(Content, Errors.ToArray());
case HtmlTokenType.NewLine:
return SyntaxFactory.NewLineToken(Content, Errors.ToArray());
case HtmlTokenType.OpenAngle:
return SyntaxFactory.Punctuation(SyntaxKind.OpenAngle, Content, Errors.ToArray());
case HtmlTokenType.Bang:
return SyntaxFactory.Punctuation(SyntaxKind.Bang, Content, Errors.ToArray());
case HtmlTokenType.ForwardSlash:
return SyntaxFactory.Punctuation(SyntaxKind.ForwardSlash, Content, Errors.ToArray());
case HtmlTokenType.QuestionMark:
return SyntaxFactory.Punctuation(SyntaxKind.QuestionMark, Content, Errors.ToArray());
case HtmlTokenType.DoubleHyphen:
return SyntaxFactory.Punctuation(SyntaxKind.DoubleHyphen, Content, Errors.ToArray());
case HtmlTokenType.LeftBracket:
return SyntaxFactory.Punctuation(SyntaxKind.LeftBracket, Content, Errors.ToArray());
case HtmlTokenType.CloseAngle:
return SyntaxFactory.Punctuation(SyntaxKind.CloseAngle, Content, Errors.ToArray());
case HtmlTokenType.RightBracket:
return SyntaxFactory.Punctuation(SyntaxKind.RightBracket, Content, Errors.ToArray());
case HtmlTokenType.Equals:
return SyntaxFactory.Punctuation(SyntaxKind.Equals, Content, Errors.ToArray());
case HtmlTokenType.DoubleQuote:
return SyntaxFactory.Punctuation(SyntaxKind.DoubleQuote, Content, Errors.ToArray());
case HtmlTokenType.SingleQuote:
return SyntaxFactory.Punctuation(SyntaxKind.SingleQuote, Content, Errors.ToArray());
case HtmlTokenType.Transition:
return SyntaxFactory.Punctuation(SyntaxKind.Transition, Content, Errors.ToArray());
case HtmlTokenType.Colon:
return SyntaxFactory.Punctuation(SyntaxKind.Colon, Content, Errors.ToArray());
default:
return SyntaxFactory.UnknownToken(Content, Errors.ToArray());
}
}
}
}

32
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenType.cs
View File

@ -1,32 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
[Flags]
internal enum HtmlTokenType
{
Unknown,
Text, // Text which isn't one of the below
WhiteSpace, // Non-newline Whitespace
NewLine, // Newline
OpenAngle, // <
Bang, // !
ForwardSlash, // /
QuestionMark, // ?
DoubleHyphen, // --
LeftBracket, // [
CloseAngle, // >
RightBracket, // ]
Equals, // =
DoubleQuote, // "
SingleQuote, // '
Transition, // @
Colon,
RazorComment,
RazorCommentStar,
RazorCommentTransition
}
}

93
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs
View File

@ -3,11 +3,12 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization // Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization
internal class HtmlTokenizer : Tokenizer<HtmlToken, HtmlTokenType> internal class HtmlTokenizer : Tokenizer
{ {
private const char TransitionChar = '@'; private const char TransitionChar = '@';
@ -21,24 +22,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new HtmlTokenizerState? CurrentState => (HtmlTokenizerState?)base.CurrentState; private new HtmlTokenizerState? CurrentState => (HtmlTokenizerState?)base.CurrentState;
public override HtmlTokenType RazorCommentType public override SyntaxKind RazorCommentKind
{ {
get { return HtmlTokenType.RazorComment; } get { return SyntaxKind.RazorComment; }
} }
public override HtmlTokenType RazorCommentTransitionType public override SyntaxKind RazorCommentTransitionKind
{ {
get { return HtmlTokenType.RazorCommentTransition; } get { return SyntaxKind.RazorCommentTransition; }
} }
public override HtmlTokenType RazorCommentStarType public override SyntaxKind RazorCommentStarKind
{ {
get { return HtmlTokenType.RazorCommentStar; } get { return SyntaxKind.RazorCommentStar; }
} }
protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList<RazorDiagnostic> errors) protected override SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new HtmlToken(content, type, errors); return SyntaxFactory.Token(type, content, errors);
} }
protected override StateResult Dispatch() protected override StateResult Dispatch()
@ -66,7 +67,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Optimize memory allocation by returning constants for the most frequent cases // Optimize memory allocation by returning constants for the most frequent cases
protected override string GetTokenContent(HtmlTokenType type) protected override string GetTokenContent(SyntaxKind type)
{ {
var tokenLength = Buffer.Length; var tokenLength = Buffer.Length;
@ -74,27 +75,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
switch (type) switch (type)
{ {
case HtmlTokenType.OpenAngle: case SyntaxKind.OpenAngle:
return "<"; return "<";
case HtmlTokenType.Bang: case SyntaxKind.Bang:
return "!"; return "!";
case HtmlTokenType.ForwardSlash: case SyntaxKind.ForwardSlash:
return "/"; return "/";
case HtmlTokenType.QuestionMark: case SyntaxKind.QuestionMark:
return "?"; return "?";
case HtmlTokenType.LeftBracket: case SyntaxKind.LeftBracket:
return "["; return "[";
case HtmlTokenType.CloseAngle: case SyntaxKind.CloseAngle:
return ">"; return ">";
case HtmlTokenType.RightBracket: case SyntaxKind.RightBracket:
return "]"; return "]";
case HtmlTokenType.Equals: case SyntaxKind.Equals:
return "="; return "=";
case HtmlTokenType.DoubleQuote: case SyntaxKind.DoubleQuote:
return "\""; return "\"";
case HtmlTokenType.SingleQuote: case SyntaxKind.SingleQuote:
return "'"; return "'";
case HtmlTokenType.WhiteSpace: case SyntaxKind.Whitespace:
if (Buffer[0] == ' ') if (Buffer[0] == ' ')
{ {
return " "; return " ";
@ -104,7 +105,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t"; return "\t";
} }
break; break;
case HtmlTokenType.NewLine: case SyntaxKind.NewLine:
if (Buffer[0] == '\n') if (Buffer[0] == '\n')
{ {
return "\n"; return "\n";
@ -113,7 +114,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
if (tokenLength == 2 && type == HtmlTokenType.NewLine) if (tokenLength == 2 && type == SyntaxKind.NewLine)
{ {
return "\r\n"; return "\r\n";
} }
@ -139,17 +140,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return Transition( return Transition(
HtmlTokenizerState.AfterRazorCommentTransition, HtmlTokenizerState.AfterRazorCommentTransition,
EndToken(HtmlTokenType.RazorCommentTransition)); EndToken(SyntaxKind.RazorCommentTransition));
} }
else if (CurrentCharacter == '@') else if (CurrentCharacter == '@')
{ {
// Could be escaped comment transition // Could be escaped comment transition
return Transition( return Transition(
HtmlTokenizerState.EscapedRazorCommentTransition, HtmlTokenizerState.EscapedRazorCommentTransition,
EndToken(HtmlTokenType.Transition)); EndToken(SyntaxKind.Transition));
} }
return Stay(EndToken(HtmlTokenType.Transition)); return Stay(EndToken(SyntaxKind.Transition));
} }
else if (AtToken()) else if (AtToken())
{ {
@ -164,7 +165,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult EscapedRazorCommentTransition() private StateResult EscapedRazorCommentTransition()
{ {
TakeCurrent(); TakeCurrent();
return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Transition)); return Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.Transition));
} }
private StateResult Text() private StateResult Text()
@ -190,10 +191,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Output the Text token and return to the Data state to tokenize the next character (if there is one) // Output the Text token and return to the Data state to tokenize the next character (if there is one)
return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Text)); return Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.HtmlTextLiteral));
} }
private HtmlToken Token() private SyntaxToken Token()
{ {
Debug.Assert(AtToken()); Debug.Assert(AtToken());
var sym = CurrentCharacter; var sym = CurrentCharacter;
@ -201,45 +202,45 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
switch (sym) switch (sym)
{ {
case '<': case '<':
return EndToken(HtmlTokenType.OpenAngle); return EndToken(SyntaxKind.OpenAngle);
case '!': case '!':
return EndToken(HtmlTokenType.Bang); return EndToken(SyntaxKind.Bang);
case '/': case '/':
return EndToken(HtmlTokenType.ForwardSlash); return EndToken(SyntaxKind.ForwardSlash);
case '?': case '?':
return EndToken(HtmlTokenType.QuestionMark); return EndToken(SyntaxKind.QuestionMark);
case '[': case '[':
return EndToken(HtmlTokenType.LeftBracket); return EndToken(SyntaxKind.LeftBracket);
case '>': case '>':
return EndToken(HtmlTokenType.CloseAngle); return EndToken(SyntaxKind.CloseAngle);
case ']': case ']':
return EndToken(HtmlTokenType.RightBracket); return EndToken(SyntaxKind.RightBracket);
case '=': case '=':
return EndToken(HtmlTokenType.Equals); return EndToken(SyntaxKind.Equals);
case '"': case '"':
return EndToken(HtmlTokenType.DoubleQuote); return EndToken(SyntaxKind.DoubleQuote);
case '\'': case '\'':
return EndToken(HtmlTokenType.SingleQuote); return EndToken(SyntaxKind.SingleQuote);
case '-': case '-':
Debug.Assert(CurrentCharacter == '-'); Debug.Assert(CurrentCharacter == '-');
TakeCurrent(); TakeCurrent();
return EndToken(HtmlTokenType.DoubleHyphen); return EndToken(SyntaxKind.DoubleHyphen);
default: default:
Debug.Fail("Unexpected token!"); Debug.Fail("Unexpected token!");
return EndToken(HtmlTokenType.Unknown); return EndToken(SyntaxKind.Unknown);
} }
} }
private HtmlToken Whitespace() private SyntaxToken Whitespace()
{ {
while (ParserHelpers.IsWhitespace(CurrentCharacter)) while (ParserHelpers.IsWhitespace(CurrentCharacter))
{ {
TakeCurrent(); TakeCurrent();
} }
return EndToken(HtmlTokenType.WhiteSpace); return EndToken(SyntaxKind.Whitespace);
} }
private HtmlToken Newline() private SyntaxToken Newline()
{ {
Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter)); Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter));
// CSharp Spec §2.3.1 // CSharp Spec §2.3.1
@ -249,7 +250,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); TakeCurrent();
} }
return EndToken(HtmlTokenType.NewLine); return EndToken(SyntaxKind.NewLine);
} }
private bool AtToken() private bool AtToken()
@ -274,7 +275,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null); return Transition((int)state, result: null);
} }
private StateResult Transition(HtmlTokenizerState state, HtmlToken result) private StateResult Transition(HtmlTokenizerState state, SyntaxToken result)
{ {
return Transition((int)state, result); return Transition((int)state, result);
} }

20
src/Microsoft.AspNetCore.Razor.Language/Legacy/IToken.cs
View File

@ -1,20 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal interface IToken
{
Span Parent { get; set; }
string Content { get; }
SourceLocation Start { get; }
SyntaxKind SyntaxKind { get; }
SyntaxToken SyntaxToken { get; }
}
}

4
src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs
View File

@ -1,10 +1,12 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal interface ITokenizer internal interface ITokenizer
{ {
IToken NextToken(); SyntaxToken NextToken();
} }
} }

37
src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs
View File

@ -7,6 +7,7 @@ using System.Diagnostics;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax;
using Microsoft.Extensions.Internal; using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -16,7 +17,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private readonly ISet<string> _keywords; private readonly ISet<string> _keywords;
private readonly IReadOnlyCollection<string> _readOnlyKeywords; private readonly IReadOnlyCollection<string> _readOnlyKeywords;
public ImplicitExpressionEditHandler(Func<string, IEnumerable<IToken>> tokenizer, ISet<string> keywords, bool acceptTrailingDot) public ImplicitExpressionEditHandler(Func<string, IEnumerable<Syntax.InternalSyntax.SyntaxToken>> tokenizer, ISet<string> keywords, bool acceptTrailingDot)
: base(tokenizer) : base(tokenizer)
{ {
_keywords = keywords ?? new HashSet<string>(); _keywords = keywords ?? new HashSet<string>();
@ -172,20 +173,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
for (var i = 0; i < target.Tokens.Count; i++) for (var i = 0; i < target.Tokens.Count; i++)
{ {
var token = target.Tokens[i] as CSharpToken; var token = target.Tokens[i];
if (token == null) if (token == null)
{ {
break; break;
} }
var tokenStartIndex = token.Start.AbsoluteIndex; var tokenStartIndex = token.Position;
var tokenEndIndex = tokenStartIndex + token.Content.Length; var tokenEndIndex = token.EndPosition;
// We're looking for the first token that contains the SourceChange. // We're looking for the first token that contains the SourceChange.
if (tokenEndIndex > change.Span.AbsoluteIndex) if (tokenEndIndex > change.Span.AbsoluteIndex)
{ {
if (tokenEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && token.Type == CSharpTokenType.Identifier) if (tokenEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && token.Kind == SyntaxKind.Identifier)
{ {
// The token we're changing happens to be an identifier. Need to check if its transformed state is also one. // The token we're changing happens to be an identifier. Need to check if its transformed state is also one.
// We do this transformation logic to capture the case that the new text change happens to not be an identifier; // We do this transformation logic to capture the case that the new text change happens to not be an identifier;
@ -200,8 +201,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break; break;
} }
var newToken = (CSharpToken)newTokens.First(); var newToken = newTokens.First();
if (newToken.Type == CSharpTokenType.Identifier) if (newToken.Kind == SyntaxKind.Identifier)
{ {
return true; return true;
} }
@ -241,8 +242,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var changeStart = change.Span.AbsoluteIndex; var changeStart = change.Span.AbsoluteIndex;
var changeLength = change.Span.Length; var changeLength = change.Span.Length;
var changeEnd = changeStart + changeLength; var changeEnd = changeStart + changeLength;
var tokens = target.Tokens.Cast<CSharpToken>().ToArray(); if (!IsInsideParenthesis(changeStart, target.Tokens) || !IsInsideParenthesis(changeEnd, target.Tokens))
if (!IsInsideParenthesis(changeStart, tokens) || !IsInsideParenthesis(changeEnd, tokens))
{ {
// Either the start or end of the delete does not fall inside of parenthesis, unacceptable inner deletion. // Either the start or end of the delete does not fall inside of parenthesis, unacceptable inner deletion.
return false; return false;
@ -274,8 +274,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false; return false;
} }
var tokens = target.Tokens.Cast<CSharpToken>().ToArray(); if (IsInsideParenthesis(change.Span.AbsoluteIndex, target.Tokens))
if (IsInsideParenthesis(change.Span.AbsoluteIndex, tokens))
{ {
return true; return true;
} }
@ -284,7 +283,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Internal for testing // Internal for testing
internal static bool IsInsideParenthesis(int position, IReadOnlyList<CSharpToken> tokens) internal static bool IsInsideParenthesis(int position, IReadOnlyList<SyntaxToken> tokens)
{ {
var balanceCount = 0; var balanceCount = 0;
var foundInsertionPoint = false; var foundInsertionPoint = false;
@ -322,9 +321,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Internal for testing // Internal for testing
internal static bool ContainsPosition(int position, CSharpToken currentToken) internal static bool ContainsPosition(int position, SyntaxToken currentToken)
{ {
var tokenStart = currentToken.Start.AbsoluteIndex; var tokenStart = currentToken.Position;
if (tokenStart == position) if (tokenStart == position)
{ {
// Token is exactly at the insertion point. // Token is exactly at the insertion point.
@ -342,14 +341,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Internal for testing // Internal for testing
internal static bool TryUpdateBalanceCount(CSharpToken token, ref int count) internal static bool TryUpdateBalanceCount(SyntaxToken token, ref int count)
{ {
var updatedCount = count; var updatedCount = count;
if (token.Type == CSharpTokenType.LeftParenthesis) if (token.Kind == SyntaxKind.LeftParenthesis)
{ {
updatedCount++; updatedCount++;
} }
else if (token.Type == CSharpTokenType.RightParenthesis) else if (token.Kind == SyntaxKind.RightParenthesis)
{ {
if (updatedCount == 0) if (updatedCount == 0)
{ {
@ -358,7 +357,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
updatedCount--; updatedCount--;
} }
else if (token.Type == CSharpTokenType.StringLiteral) else if (token.Kind == SyntaxKind.StringLiteral)
{ {
var content = token.Content; var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '"') if (content.Length > 0 && content[content.Length - 1] != '"')
@ -370,7 +369,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
} }
else if (token.Type == CSharpTokenType.CharacterLiteral) else if (token.Kind == SyntaxKind.CharacterLiteral)
{ {
var content = token.Content; var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '\'') if (content.Length > 0 && content[content.Length - 1] != '\'')

51
src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs
View File

@ -3,30 +3,29 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract class LanguageCharacteristics<TTokenizer, TToken, TTokenType> internal abstract class LanguageCharacteristics<TTokenizer>
where TTokenType : struct where TTokenizer : Tokenizer
where TTokenizer : Tokenizer<TToken, TTokenType>
where TToken : TokenBase<TTokenType>
{ {
public abstract string GetSample(TTokenType type); public abstract string GetSample(SyntaxKind type);
public abstract TTokenizer CreateTokenizer(ITextDocument source); public abstract TTokenizer CreateTokenizer(ITextDocument source);
public abstract TTokenType FlipBracket(TTokenType bracket); public abstract SyntaxKind FlipBracket(SyntaxKind bracket);
public abstract TToken CreateMarkerToken(); public abstract SyntaxToken CreateMarkerToken();
public virtual IEnumerable<TToken> TokenizeString(string content) public virtual IEnumerable<SyntaxToken> TokenizeString(string content)
{ {
return TokenizeString(SourceLocation.Zero, content); return TokenizeString(SourceLocation.Zero, content);
} }
public virtual IEnumerable<TToken> TokenizeString(SourceLocation start, string input) public virtual IEnumerable<SyntaxToken> TokenizeString(SourceLocation start, string input)
{ {
using (var reader = new SeekableTextReader(input, start.FilePath)) using (var reader = new SeekableTextReader(input, start.FilePath))
{ {
var tok = CreateTokenizer(reader); var tok = CreateTokenizer(reader);
TToken token; SyntaxToken token;
while ((token = tok.NextToken()) != null) while ((token = tok.NextToken()) != null)
{ {
yield return token; yield return token;
@ -34,76 +33,76 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public virtual bool IsWhiteSpace(TToken token) public virtual bool IsWhiteSpace(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.WhiteSpace); return IsKnownTokenType(token, KnownTokenType.WhiteSpace);
} }
public virtual bool IsNewLine(TToken token) public virtual bool IsNewLine(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.NewLine); return IsKnownTokenType(token, KnownTokenType.NewLine);
} }
public virtual bool IsIdentifier(TToken token) public virtual bool IsIdentifier(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.Identifier); return IsKnownTokenType(token, KnownTokenType.Identifier);
} }
public virtual bool IsKeyword(TToken token) public virtual bool IsKeyword(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.Keyword); return IsKnownTokenType(token, KnownTokenType.Keyword);
} }
public virtual bool IsTransition(TToken token) public virtual bool IsTransition(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.Transition); return IsKnownTokenType(token, KnownTokenType.Transition);
} }
public virtual bool IsCommentStart(TToken token) public virtual bool IsCommentStart(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.CommentStart); return IsKnownTokenType(token, KnownTokenType.CommentStart);
} }
public virtual bool IsCommentStar(TToken token) public virtual bool IsCommentStar(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.CommentStar); return IsKnownTokenType(token, KnownTokenType.CommentStar);
} }
public virtual bool IsCommentBody(TToken token) public virtual bool IsCommentBody(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.CommentBody); return IsKnownTokenType(token, KnownTokenType.CommentBody);
} }
public virtual bool IsUnknown(TToken token) public virtual bool IsUnknown(SyntaxToken token)
{ {
return IsKnownTokenType(token, KnownTokenType.Unknown); return IsKnownTokenType(token, KnownTokenType.Unknown);
} }
public virtual bool IsKnownTokenType(TToken token, KnownTokenType type) public virtual bool IsKnownTokenType(SyntaxToken token, KnownTokenType type)
{ {
return token != null && Equals(token.Type, GetKnownTokenType(type)); return token != null && Equals(token.Kind, GetKnownTokenType(type));
} }
public virtual Tuple<TToken, TToken> SplitToken(TToken token, int splitAt, TTokenType leftType) public virtual Tuple<SyntaxToken, SyntaxToken> SplitToken(SyntaxToken token, int splitAt, SyntaxKind leftType)
{ {
var left = CreateToken(token.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray); var left = CreateToken(token.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray);
TToken right = null; SyntaxToken right = null;
if (splitAt < token.Content.Length) if (splitAt < token.Content.Length)
{ {
right = CreateToken(token.Content.Substring(splitAt), token.Type, token.Errors); right = CreateToken(token.Content.Substring(splitAt), token.Kind, token.GetDiagnostics());
} }
return Tuple.Create(left, right); return Tuple.Create(left, right);
} }
public abstract TTokenType GetKnownTokenType(KnownTokenType type); public abstract SyntaxKind GetKnownTokenType(KnownTokenType type);
public virtual bool KnowsTokenType(KnownTokenType type) public virtual bool KnowsTokenType(KnownTokenType type)
{ {
return type == KnownTokenType.Unknown || !Equals(GetKnownTokenType(type), GetKnownTokenType(KnownTokenType.Unknown)); return type == KnownTokenType.Unknown || !Equals(GetKnownTokenType(type), GetKnownTokenType(KnownTokenType.Unknown));
} }
protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList<RazorDiagnostic> errors); protected abstract SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList<RazorDiagnostic> errors);
} }
} }

4
src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs
View File

@ -67,9 +67,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
builder.Kind = SpanKindInternal.Markup; builder.Kind = SpanKindInternal.Markup;
builder.ChunkGenerator = new MarkupChunkGenerator(); builder.ChunkGenerator = new MarkupChunkGenerator();
foreach (IToken sym in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content)) foreach (var token in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content))
{ {
builder.Accept(sym); builder.Accept(token);
} }
} }
} }

66
src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs
View File

@ -5,13 +5,16 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax; using Microsoft.AspNetCore.Razor.Language.Syntax;
using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class Span : SyntaxTreeNode internal class Span : SyntaxTreeNode
{ {
private static readonly List<SyntaxToken> EmptyTokenList = new List<SyntaxToken>(0);
private static readonly int TypeHashCode = typeof(Span).GetHashCode(); private static readonly int TypeHashCode = typeof(Span).GetHashCode();
private IReadOnlyList<Syntax.InternalSyntax.SyntaxToken> _greenTokens;
private string _content; private string _content;
private int? _length; private int? _length;
private SourceLocation _start; private SourceLocation _start;
@ -24,7 +27,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public ISpanChunkGenerator ChunkGenerator { get; private set; } public ISpanChunkGenerator ChunkGenerator { get; private set; }
public SpanKindInternal Kind { get; private set; } public SpanKindInternal Kind { get; private set; }
public IReadOnlyList<IToken> Tokens { get; private set; }
public IReadOnlyList<SyntaxToken> Tokens { get; private set; }
// Allow test code to re-link spans // Allow test code to re-link spans
public Span Previous { get; internal set; } public Span Previous { get; internal set; }
@ -32,7 +36,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SpanEditHandler EditHandler { get; private set; } public SpanEditHandler EditHandler { get; private set; }
public HtmlNodeSyntax SyntaxNode { get; private set; } public SyntaxNode SyntaxNode { get; private set; }
public override bool IsBlock => false; public override bool IsBlock => false;
@ -96,20 +100,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void ReplaceWith(SpanBuilder builder) public void ReplaceWith(SpanBuilder builder)
{ {
Kind = builder.Kind; Kind = builder.Kind;
Tokens = builder.Tokens; _greenTokens = builder.Tokens;
SyntaxNode = builder.SyntaxNode;
for (var i = 0; i <Tokens.Count; i++)
{
Tokens[i].Parent = this;
}
EditHandler = builder.EditHandler; EditHandler = builder.EditHandler;
ChunkGenerator = builder.ChunkGenerator ?? SpanChunkGenerator.Null; ChunkGenerator = builder.ChunkGenerator ?? SpanChunkGenerator.Null;
_start = builder.Start; _start = builder.Start;
SyntaxNode = builder.SyntaxNode?.CreateRed(parent: null, position: _start.AbsoluteIndex);
_content = null; _content = null;
_length = null; _length = null;
var tokens = EmptyTokenList;
if (_greenTokens.Count > 0)
{
tokens = new List<SyntaxToken>();
var currentStart = _start.AbsoluteIndex;
for (var i = 0; i < _greenTokens.Count; i++)
{
var token = new SyntaxToken(_greenTokens[i], parent: SyntaxNode, parentSpan: this, position: currentStart);
tokens.Add(token);
currentStart += token.FullWidth;
}
}
Tokens = tokens;
Parent?.ChildChanged(); Parent?.ChildChanged();
// Since we took references to the values in SpanBuilder, clear its references out // Since we took references to the values in SpanBuilder, clear its references out
@ -149,8 +162,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// </summary> /// </summary>
public override bool EquivalentTo(SyntaxTreeNode node) public override bool EquivalentTo(SyntaxTreeNode node)
{ {
var other = node as Span; return node is Span other &&
return other != null &&
Kind.Equals(other.Kind) && Kind.Equals(other.Kind) &&
Start.Equals(other.Start) && Start.Equals(other.Start) &&
EditHandler.Equals(other.EditHandler) && EditHandler.Equals(other.EditHandler) &&
@ -165,12 +177,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var other = obj as Span; return obj is Span other &&
return other != null &&
Kind.Equals(other.Kind) && Kind.Equals(other.Kind) &&
EditHandler.Equals(other.EditHandler) && EditHandler.Equals(other.EditHandler) &&
ChunkGenerator.Equals(other.ChunkGenerator) && ChunkGenerator.Equals(other.ChunkGenerator) &&
Tokens.SequenceEqual(other.Tokens); Tokens.SequenceEqual(other.Tokens, SyntaxTokenComparer.Default);
} }
public override int GetHashCode() public override int GetHashCode()
@ -189,5 +200,28 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var spanBuilder = new SpanBuilder(this); var spanBuilder = new SpanBuilder(this);
return spanBuilder.Build(); return spanBuilder.Build();
} }
private class SyntaxTokenComparer : IEqualityComparer<SyntaxToken>
{
public static readonly SyntaxTokenComparer Default = new SyntaxTokenComparer();
private SyntaxTokenComparer()
{
}
public bool Equals(SyntaxToken x, SyntaxToken y)
{
return x.IsEquivalentTo(y);
}
public int GetHashCode(SyntaxToken obj)
{
var hash = HashCodeCombiner.Start();
hash.Add(obj.Content, StringComparer.Ordinal);
hash.Add(obj.Kind);
return hash;
}
}
} }
} }

30
src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs
View File

@ -12,7 +12,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal class SpanBuilder internal class SpanBuilder
{ {
private SourceLocation _start; private SourceLocation _start;
private List<IToken> _tokens; private List<SyntaxToken> _tokens;
private SourceLocationTracker _tracker; private SourceLocationTracker _tracker;
public SpanBuilder(Span original) public SpanBuilder(Span original)
@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_start = original.Start; _start = original.Start;
ChunkGenerator = original.ChunkGenerator; ChunkGenerator = original.ChunkGenerator;
_tokens = new List<IToken>(original.Tokens); _tokens = new List<SyntaxToken>(original.Tokens.Select(t =>t.Green));
_tracker = new SourceLocationTracker(original.Start); _tracker = new SourceLocationTracker(original.Start);
} }
@ -35,7 +35,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Start = location; Start = location;
} }
public HtmlNodeSyntax SyntaxNode { get; private set; } public Syntax.GreenNode SyntaxNode { get; private set; }
public ISpanChunkGenerator ChunkGenerator { get; set; } public ISpanChunkGenerator ChunkGenerator { get; set; }
@ -53,13 +53,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SpanKindInternal Kind { get; set; } public SpanKindInternal Kind { get; set; }
public IReadOnlyList<IToken> Tokens public IReadOnlyList<SyntaxToken> Tokens
{ {
get get
{ {
if (_tokens == null) if (_tokens == null)
{ {
_tokens = new List<IToken>(); _tokens = new List<SyntaxToken>();
} }
return _tokens; return _tokens;
@ -73,9 +73,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Need to potentially allocate a new list because Span.ReplaceWith takes ownership // Need to potentially allocate a new list because Span.ReplaceWith takes ownership
// of the original list. // of the original list.
_tokens = null; _tokens = null;
_tokens = new List<IToken>(); _tokens = new List<SyntaxToken>();
EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<IToken>()); EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<SyntaxToken>());
ChunkGenerator = SpanChunkGenerator.Null; ChunkGenerator = SpanChunkGenerator.Null;
Start = SourceLocation.Undefined; Start = SourceLocation.Undefined;
} }
@ -85,12 +85,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
SyntaxNode = GetSyntaxNode(syntaxKind); SyntaxNode = GetSyntaxNode(syntaxKind);
var span = new Span(this); var span = new Span(this);
for (var i = 0; i < span.Tokens.Count; i++)
{
var token = span.Tokens[i];
token.Parent = span;
}
return span; return span;
} }
@ -100,7 +94,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_tokens?.Clear(); _tokens?.Clear();
} }
public void Accept(IToken token) public void Accept(SyntaxToken token)
{ {
if (token == null) if (token == null)
{ {
@ -116,20 +110,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_tracker.UpdateLocation(token.Content); _tracker.UpdateLocation(token.Content);
} }
private HtmlNodeSyntax GetSyntaxNode(SyntaxKind syntaxKind) private Syntax.GreenNode GetSyntaxNode(SyntaxKind syntaxKind)
{ {
if (syntaxKind == SyntaxKind.HtmlText) if (syntaxKind == SyntaxKind.HtmlText)
{ {
var textTokens = new SyntaxListBuilder<SyntaxToken>(SyntaxListBuilder.Create()); var textTokens = new SyntaxListBuilder<SyntaxToken>(SyntaxListBuilder.Create());
foreach (var token in Tokens) foreach (var token in Tokens)
{ {
if (token.SyntaxKind == SyntaxKind.Unknown) if (token.Kind == SyntaxKind.Unknown)
{ {
Debug.Assert(false, $"Unexpected html token {((HtmlToken)token).Type}"); Debug.Assert(false, $"Unexpected token {token.Kind}");
continue; continue;
} }
textTokens.Add(token.SyntaxToken); textTokens.Add(token);
} }
var textResult = textTokens.ToList(); var textResult = textTokens.ToList();
return SyntaxFactory.HtmlText(new SyntaxList<SyntaxToken>(textResult.Node)); return SyntaxFactory.HtmlText(new SyntaxList<SyntaxToken>(textResult.Node));

13
src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs
View File

@ -3,7 +3,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
@ -11,12 +11,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
private static readonly int TypeHashCode = typeof(SpanEditHandler).GetHashCode(); private static readonly int TypeHashCode = typeof(SpanEditHandler).GetHashCode();
public SpanEditHandler(Func<string, IEnumerable<IToken>> tokenizer) public SpanEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer)
: this(tokenizer, AcceptedCharactersInternal.Any) : this(tokenizer, AcceptedCharactersInternal.Any)
{ {
} }
public SpanEditHandler(Func<string, IEnumerable<IToken>> tokenizer, AcceptedCharactersInternal accepted) public SpanEditHandler(Func<string, IEnumerable<SyntaxToken>> tokenizer, AcceptedCharactersInternal accepted)
{ {
AcceptedCharacters = accepted; AcceptedCharacters = accepted;
Tokenizer = tokenizer; Tokenizer = tokenizer;
@ -24,9 +24,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public AcceptedCharactersInternal AcceptedCharacters { get; set; } public AcceptedCharactersInternal AcceptedCharacters { get; set; }
public Func<string, IEnumerable<IToken>> Tokenizer { get; set; } public Func<string, IEnumerable<SyntaxToken>> Tokenizer { get; set; }
public static SpanEditHandler CreateDefault(Func<string, IEnumerable<IToken>> tokenizer) public static SpanEditHandler CreateDefault(Func<string, IEnumerable<SyntaxToken>> tokenizer)
{ {
return new SpanEditHandler(tokenizer); return new SpanEditHandler(tokenizer);
} }
@ -116,8 +116,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var other = obj as SpanEditHandler; return obj is SpanEditHandler other &&
return other != null &&
GetType() == other.GetType() && GetType() == other.GetType() &&
AcceptedCharacters == other.AcceptedCharacters; AcceptedCharacters == other.AcceptedCharacters;
} }

63
src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs
View File

@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
@ -150,7 +151,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}; };
// Will contain tokens that represent a single attribute value: <input| class="btn"| /> // Will contain tokens that represent a single attribute value: <input| class="btn"| />
var htmlTokens = span.Tokens.OfType<HtmlToken>().ToArray(); var tokens = span.Tokens;
var capturedAttributeValueStart = false; var capturedAttributeValueStart = false;
var attributeValueStartLocation = span.Start; var attributeValueStartLocation = span.Start;
@ -165,9 +166,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Iterate down through the tokens to find the name and the start of the value. // Iterate down through the tokens to find the name and the start of the value.
// We subtract the tokenOffset so we don't accept an ending quote of a span. // We subtract the tokenOffset so we don't accept an ending quote of a span.
for (var i = 0; i < htmlTokens.Length - tokenOffset; i++) for (var i = 0; i < tokens.Count - tokenOffset; i++)
{ {
var token = htmlTokens[i]; var token = tokens[i];
if (afterEquals) if (afterEquals)
{ {
@ -186,9 +187,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
attributeValueStartLocation = token.Start; attributeValueStartLocation = token.Start;
} }
builder.Accept(token); builder.Accept(token.Green);
} }
else if (name == null && HtmlMarkupParser.IsValidAttributeNameToken(token)) else if (name == null && HtmlMarkupParser.IsValidAttributeNameToken(token.Green))
{ {
// We've captured all leading whitespace prior to the attribute name. // We've captured all leading whitespace prior to the attribute name.
// We're now at: " |asp-for='...'" or " |asp-for=..." // We're now at: " |asp-for='...'" or " |asp-for=..."
@ -196,10 +197,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var nameBuilder = new StringBuilder(); var nameBuilder = new StringBuilder();
// Move the indexer past the attribute name tokens. // Move the indexer past the attribute name tokens.
for (var j = i; j < htmlTokens.Length; j++) for (var j = i; j < tokens.Count; j++)
{ {
var nameToken = htmlTokens[j]; var nameToken = tokens[j];
if (!HtmlMarkupParser.IsValidAttributeNameToken(nameToken)) if (!HtmlMarkupParser.IsValidAttributeNameToken(nameToken.Green))
{ {
break; break;
} }
@ -213,7 +214,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
name = nameBuilder.ToString(); name = nameBuilder.ToString();
attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, name); attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, name);
} }
else if (token.Type == HtmlTokenType.Equals) else if (token.Kind == SyntaxKind.Equals)
{ {
// We've captured all leading whitespace and the attribute name. // We've captured all leading whitespace and the attribute name.
// We're now at: " asp-for|='...'" or " asp-for|=..." // We're now at: " asp-for|='...'" or " asp-for|=..."
@ -227,19 +228,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
do do
{ {
i++; // Start from the token after '='. i++; // Start from the token after '='.
} while (i < htmlTokens.Length && } while (i < tokens.Count &&
(htmlTokens[i].Type == HtmlTokenType.WhiteSpace || (tokens[i].Kind == SyntaxKind.Whitespace ||
htmlTokens[i].Type == HtmlTokenType.NewLine)); tokens[i].Kind == SyntaxKind.NewLine));
// Check for attribute start values, aka single or double quote // Check for attribute start values, aka single or double quote
if (i < htmlTokens.Length && IsQuote(htmlTokens[i])) if (i < tokens.Count && IsQuote(tokens[i]))
{ {
if (htmlTokens[i].Type == HtmlTokenType.SingleQuote) if (tokens[i].Kind == SyntaxKind.SingleQuote)
{ {
attributeValueStyle = AttributeStructure.SingleQuotes; attributeValueStyle = AttributeStructure.SingleQuotes;
} }
tokenStartLocation = htmlTokens[i].Start; tokenStartLocation = tokens[i].Start;
// If there's a start quote then there must be an end quote to be valid, skip it. // If there's a start quote then there must be an end quote to be valid, skip it.
tokenOffset = 1; tokenOffset = 1;
@ -260,7 +261,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
afterEquals = true; afterEquals = true;
} }
else if (token.Type == HtmlTokenType.WhiteSpace) else if (token.Kind == SyntaxKind.Whitespace)
{ {
// We're at the start of the attribute, this branch may be hit on the first iterations of // We're at the start of the attribute, this branch may be hit on the first iterations of
// the loop since the parser separates attributes with their spaces included as tokens. // the loop since the parser separates attributes with their spaces included as tokens.
@ -343,9 +344,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var nameTokens = childSpan var nameTokens = childSpan
.Tokens .Tokens
.OfType<HtmlToken>() .SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token.Green)) // Skip prefix
.SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token)) // Skip prefix .TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken.Green))
.TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
.Select(nameToken => nameToken.Content); .Select(nameToken => nameToken.Content);
var name = string.Concat(nameTokens); var name = string.Concat(nameTokens);
@ -362,12 +362,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames); var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames);
var firstChild = builder.Children[0] as Span; var firstChild = builder.Children[0] as Span;
if (firstChild != null && firstChild.Tokens[0] is HtmlToken) if (firstChild != null)
{ {
var htmlToken = firstChild.Tokens[firstChild.Tokens.Count - 1] as HtmlToken; var token = firstChild.Tokens[firstChild.Tokens.Count - 1];
switch (htmlToken.Type) switch (token.Kind)
{ {
case HtmlTokenType.Equals: case SyntaxKind.Equals:
if (builder.Children.Count == 2 && if (builder.Children.Count == 2 &&
builder.Children[1] is Span value && builder.Children[1] is Span value &&
value.Kind == SpanKindInternal.Markup) value.Kind == SpanKindInternal.Markup)
@ -385,10 +385,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
result.AttributeStructure = AttributeStructure.DoubleQuotes; result.AttributeStructure = AttributeStructure.DoubleQuotes;
} }
break; break;
case HtmlTokenType.DoubleQuote: case SyntaxKind.DoubleQuote:
result.AttributeStructure = AttributeStructure.DoubleQuotes; result.AttributeStructure = AttributeStructure.DoubleQuotes;
break; break;
case HtmlTokenType.SingleQuote: case SyntaxKind.SingleQuote:
result.AttributeStructure = AttributeStructure.SingleQuotes; result.AttributeStructure = AttributeStructure.SingleQuotes;
break; break;
default: default:
@ -408,8 +408,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// In some malformed cases e.g. <p bar="false', the last Span (false' in the ex.) may contain more // In some malformed cases e.g. <p bar="false', the last Span (false' in the ex.) may contain more
// than a single HTML token. Do not ignore those other tokens. // than a single HTML token. Do not ignore those other tokens.
var tokenCount = endSpan.Tokens.Count(); var tokenCount = endSpan.Tokens.Count;
var endToken = tokenCount == 1 ? (HtmlToken)endSpan.Tokens.First() : null; var endToken = tokenCount == 1 ? endSpan.Tokens.First() : null;
// Checking to see if it's a quoted attribute, if so we should remove end quote // Checking to see if it's a quoted attribute, if so we should remove end quote
if (endToken != null && IsQuote(endToken)) if (endToken != null && IsQuote(endToken))
@ -614,8 +614,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// expression). // expression).
var firstNonWhitespaceToken = span var firstNonWhitespaceToken = span
.Tokens .Tokens
.OfType<HtmlToken>() .First(token => token.Kind != SyntaxKind.Whitespace && token.Kind != SyntaxKind.NewLine);
.First(token => token.Type != HtmlTokenType.WhiteSpace && token.Type != HtmlTokenType.NewLine);
var location = new SourceSpan(firstNonWhitespaceToken.Start, attributeName.Length); var location = new SourceSpan(firstNonWhitespaceToken.Start, attributeName.Length);
return location; return location;
@ -716,10 +715,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return firstBoundAttribute; return firstBoundAttribute;
} }
private static bool IsQuote(HtmlToken htmlToken) private static bool IsQuote(SyntaxToken token)
{ {
return htmlToken.Type == HtmlTokenType.DoubleQuote || return token.Kind == SyntaxKind.DoubleQuote ||
htmlToken.Type == HtmlTokenType.SingleQuote; token.Kind == SyntaxKind.SingleQuote;
} }
private static void ConfigureNonStringAttribute(SpanBuilder builder, bool isDuplicateAttribute) private static void ConfigureNonStringAttribute(SpanBuilder builder, bool isDuplicateAttribute)

39
src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperParseTreeRewriter.cs
View File

@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
@ -372,10 +373,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var childOffset = 0; var childOffset = 0;
if (childSpan.Tokens.Count > 0) if (childSpan.Tokens.Count > 0)
{ {
var potentialQuote = childSpan.Tokens[childSpan.Tokens.Count - 1] as HtmlToken; var potentialQuote = childSpan.Tokens[childSpan.Tokens.Count - 1];
if (potentialQuote != null && if (potentialQuote != null &&
(potentialQuote.Type == HtmlTokenType.DoubleQuote || (potentialQuote.Kind == SyntaxKind.DoubleQuote ||
potentialQuote.Type == HtmlTokenType.SingleQuote)) potentialQuote.Kind == SyntaxKind.SingleQuote))
{ {
childOffset = 1; childOffset = 1;
} }
@ -409,23 +410,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Entire attribute is a string // Entire attribute is a string
for (var j = 0; j < endValueMarker; j++) for (var j = 0; j < endValueMarker; j++)
{ {
var htmlToken = (HtmlToken)childSpan.Tokens[j]; var token = childSpan.Tokens[j];
if (!afterEquals) if (!afterEquals)
{ {
afterEquals = htmlToken.Type == HtmlTokenType.Equals; afterEquals = token.Kind == SyntaxKind.Equals;
continue; continue;
} }
if (!atValue) if (!atValue)
{ {
atValue = htmlToken.Type != HtmlTokenType.WhiteSpace && atValue = token.Kind != SyntaxKind.Whitespace &&
htmlToken.Type != HtmlTokenType.NewLine; token.Kind != SyntaxKind.NewLine;
if (atValue) if (atValue)
{ {
if (htmlToken.Type == HtmlTokenType.DoubleQuote || if (token.Kind == SyntaxKind.DoubleQuote ||
htmlToken.Type == HtmlTokenType.SingleQuote) token.Kind == SyntaxKind.SingleQuote)
{ {
endValueMarker--; endValueMarker--;
} }
@ -433,14 +434,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Current token is considered the value (unquoted). Add its content to the // Current token is considered the value (unquoted). Add its content to the
// attribute value builder before we move past it. // attribute value builder before we move past it.
_attributeValueBuilder.Append(htmlToken.Content); _attributeValueBuilder.Append(token.Content);
} }
} }
continue; continue;
} }
_attributeValueBuilder.Append(htmlToken.Content); _attributeValueBuilder.Append(token.Content);
} }
} }
@ -643,10 +644,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (tagEnd != null && tagEnd.Kind == SpanKindInternal.Markup) if (tagEnd != null && tagEnd.Kind == SpanKindInternal.Markup)
{ {
var endToken = tagEnd.Tokens.Count > 0 ? var endToken = tagEnd.Tokens.Count > 0 ?
tagEnd.Tokens[tagEnd.Tokens.Count - 1] as HtmlToken : tagEnd.Tokens[tagEnd.Tokens.Count - 1] :
null; null;
if (endToken != null && endToken.Type == HtmlTokenType.CloseAngle) if (endToken != null && endToken.Kind == SyntaxKind.CloseAngle)
{ {
return false; return false;
} }
@ -793,13 +794,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
var childSpan = (Span)child; var childSpan = (Span)child;
HtmlToken textToken = null; SyntaxToken textToken = null;
for (var i = 0; i < childSpan.Tokens.Count; i++) for (var i = 0; i < childSpan.Tokens.Count; i++)
{ {
var token = childSpan.Tokens[i] as HtmlToken; var token = childSpan.Tokens[i];
if (token != null && if (token != null &&
(token.Type & (HtmlTokenType.WhiteSpace | HtmlTokenType.Text)) == token.Type) (token.Kind == SyntaxKind.Whitespace || token.Kind == SyntaxKind.HtmlTextLiteral))
{ {
textToken = token; textToken = token;
break; break;
@ -811,7 +812,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null; return null;
} }
return textToken.Type == HtmlTokenType.WhiteSpace ? null : textToken.Content; return textToken.Kind == SyntaxKind.Whitespace ? null : textToken.Content;
} }
private static bool IsEndTag(Block tagBlock) private static bool IsEndTag(Block tagBlock)
@ -821,9 +822,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var childSpan = (Span)tagBlock.Children.First(); var childSpan = (Span)tagBlock.Children.First();
// We grab the token that could be forward slash // We grab the token that could be forward slash
var relevantToken = (HtmlToken)childSpan.Tokens[childSpan.Tokens.Count == 1 ? 0 : 1]; var relevantToken = childSpan.Tokens[childSpan.Tokens.Count == 1 ? 0 : 1];
return relevantToken.Type == HtmlTokenType.ForwardSlash; return relevantToken.Kind == SyntaxKind.ForwardSlash;
} }
internal static bool IsComment(Span span) internal static bool IsComment(Span span)

90
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenBase.cs
View File

@ -1,90 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal abstract class TokenBase<TType> : IToken where TType : struct
{
protected TokenBase(
string content,
TType type,
IReadOnlyList<RazorDiagnostic> errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
Content = content;
Type = type;
Errors = errors;
}
public Span Parent { get; set; }
public IReadOnlyList<RazorDiagnostic> Errors { get; }
public string Content { get; }
public TType Type { get; }
public SourceLocation Start
{
get
{
if (Parent == null)
{
return SourceLocation.Undefined;
}
var tracker = new SourceLocationTracker(Parent.Start);
for (var i = 0; i < Parent.Tokens.Count; i++)
{
var token = Parent.Tokens[i];
if (object.ReferenceEquals(this, token))
{
break;
}
tracker.UpdateLocation(token.Content);
}
return tracker.CurrentLocation;
}
}
public SyntaxKind SyntaxKind => SyntaxToken.Kind;
public SyntaxToken SyntaxToken => GetSyntaxToken();
protected abstract SyntaxToken GetSyntaxToken();
public override bool Equals(object obj)
{
return obj is TokenBase<TType> other &&
string.Equals(Content, other.Content, StringComparison.Ordinal) &&
Type.Equals(other.Type);
}
public override int GetHashCode()
{
// Hash code should include only immutable properties.
var hash = HashCodeCombiner.Start();
hash.Add(Content, StringComparer.Ordinal);
hash.Add(Type);
return hash;
}
public override string ToString()
{
return string.Format(CultureInfo.InvariantCulture, "{0} [{1}]", Type, Content);
}
}
}

59
src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs
View File

@ -5,12 +5,11 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Text; using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract partial class Tokenizer<TToken, TTokenType> : ITokenizer internal abstract class Tokenizer : ITokenizer
where TTokenType : struct
where TToken : TokenBase<TTokenType>
{ {
protected Tokenizer(ITextDocument source) protected Tokenizer(ITextDocument source)
{ {
@ -31,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected int? CurrentState { get; set; } protected int? CurrentState { get; set; }
protected TToken CurrentToken { get; private set; } protected SyntaxToken CurrenSyntaxToken { get; private set; }
public ITextDocument Source { get; private set; } public ITextDocument Source { get; private set; }
@ -42,9 +41,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return Source.Peek() == -1; } get { return Source.Peek() == -1; }
} }
public abstract TTokenType RazorCommentStarType { get; } public abstract SyntaxKind RazorCommentStarKind { get; }
public abstract TTokenType RazorCommentType { get; } public abstract SyntaxKind RazorCommentKind { get; }
public abstract TTokenType RazorCommentTransitionType { get; } public abstract SyntaxKind RazorCommentTransitionKind { get; }
protected bool HaveContent protected bool HaveContent
{ {
@ -64,16 +63,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SourceLocation CurrentStart { get; private set; } public SourceLocation CurrentStart { get; private set; }
protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList<RazorDiagnostic> errors); protected abstract SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList<RazorDiagnostic> errors);
protected abstract StateResult Dispatch(); protected abstract StateResult Dispatch();
IToken ITokenizer.NextToken() SyntaxToken ITokenizer.NextToken()
{ {
return NextToken(); return NextToken();
} }
public virtual TToken NextToken() public virtual SyntaxToken NextToken()
{ {
// Post-Condition: Buffer should be empty at the start of Next() // Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
@ -95,7 +94,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token; return token;
} }
protected virtual TToken Turn() protected virtual SyntaxToken Turn()
{ {
if (CurrentState != null) if (CurrentState != null)
{ {
@ -105,19 +104,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var next = Dispatch(); var next = Dispatch();
CurrentState = next.State; CurrentState = next.State;
CurrentToken = next.Result; CurrenSyntaxToken = next.Result;
} }
while (CurrentState != null && CurrentToken == null); while (CurrentState != null && CurrenSyntaxToken == null);
if (CurrentState == null) if (CurrentState == null)
{ {
return default(TToken); // Terminated return default(SyntaxToken); // Terminated
} }
return CurrentToken; return CurrenSyntaxToken;
} }
return default(TToken); return default(SyntaxToken);
} }
public void Reset() public void Reset()
@ -149,7 +148,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to /// Returns a result containing the specified output and indicating that the next call to
/// <see cref="Turn"/> should invoke the provided state. /// <see cref="Turn"/> should invoke the provided state.
/// </summary> /// </summary>
protected StateResult Transition(int state, TToken result) protected StateResult Transition(int state, SyntaxToken result)
{ {
return new StateResult(state, result); return new StateResult(state, result);
} }
@ -159,7 +158,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new StateResult((int)state, result: null); return new StateResult((int)state, result: null);
} }
protected StateResult Transition(RazorCommentTokenizerState state, TToken result) protected StateResult Transition(RazorCommentTokenizerState state, SyntaxToken result)
{ {
return new StateResult((int)state, result); return new StateResult((int)state, result);
} }
@ -180,12 +179,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to /// Returns a result containing the specified output and indicating that the next call to
/// <see cref="Turn"/> should re-invoke the current state. /// <see cref="Turn"/> should re-invoke the current state.
/// </summary> /// </summary>
protected StateResult Stay(TToken result) protected StateResult Stay(SyntaxToken result)
{ {
return new StateResult(CurrentState, result); return new StateResult(CurrentState, result);
} }
protected TToken Single(TTokenType type) protected SyntaxToken Single(SyntaxKind type)
{ {
TakeCurrent(); TakeCurrent();
return EndToken(type); return EndToken(type);
@ -199,9 +198,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
CurrentStart = CurrentLocation; CurrentStart = CurrentLocation;
} }
protected TToken EndToken(TTokenType type) protected SyntaxToken EndToken(SyntaxKind type)
{ {
TToken token = null; SyntaxToken token = null;
if (HaveContent) if (HaveContent)
{ {
// Perf: Don't allocate a new errors array unless necessary. // Perf: Don't allocate a new errors array unless necessary.
@ -222,7 +221,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token; return token;
} }
protected virtual string GetTokenContent(TTokenType type) protected virtual string GetTokenContent(SyntaxKind type)
{ {
return Buffer.ToString(); return Buffer.ToString();
} }
@ -278,7 +277,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AssertCurrent('*'); AssertCurrent('*');
TakeCurrent(); TakeCurrent();
return Transition(1002, EndToken(RazorCommentStarType)); return Transition(1002, EndToken(RazorCommentStarKind));
} }
protected StateResult RazorCommentBody() protected StateResult RazorCommentBody()
@ -292,7 +291,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return Transition( return Transition(
RazorCommentTokenizerState.StarAfterRazorCommentBody, RazorCommentTokenizerState.StarAfterRazorCommentBody,
EndToken(RazorCommentType)); EndToken(RazorCommentKind));
} }
else else
{ {
@ -306,7 +305,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
return Transition(StartState, EndToken(RazorCommentType)); return Transition(StartState, EndToken(RazorCommentKind));
} }
protected StateResult StarAfterRazorCommentBody() protected StateResult StarAfterRazorCommentBody()
@ -315,14 +314,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
TakeCurrent(); TakeCurrent();
return Transition( return Transition(
RazorCommentTokenizerState.AtTokenAfterRazorCommentBody, RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
EndToken(RazorCommentStarType)); EndToken(RazorCommentStarKind));
} }
protected StateResult AtTokenAfterRazorCommentBody() protected StateResult AtTokenAfterRazorCommentBody()
{ {
AssertCurrent('@'); AssertCurrent('@');
TakeCurrent(); TakeCurrent();
return Transition(StartState, EndToken(RazorCommentTransitionType)); return Transition(StartState, EndToken(RazorCommentTransitionKind));
} }
/// <summary> /// <summary>
@ -397,7 +396,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected struct StateResult protected struct StateResult
{ {
public StateResult(int? state, TToken result) public StateResult(int? state, SyntaxToken result)
{ {
State = state; State = state;
Result = result; Result = result;
@ -405,7 +404,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public int? State { get; } public int? State { get; }
public TToken Result { get; } public SyntaxToken Result { get; }
} }
private static LookaheadToken BeginLookahead(ITextBuffer buffer) private static LookaheadToken BeginLookahead(ITextBuffer buffer)

155
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
View File

@ -9,20 +9,18 @@ using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract partial class TokenizerBackedParser<TTokenizer, TToken, TTokenType> : ParserBase internal abstract class TokenizerBackedParser<TTokenizer> : ParserBase
where TTokenType : struct where TTokenizer : Tokenizer
where TTokenizer : Tokenizer<TToken, TTokenType>
where TToken : TokenBase<TTokenType>
{ {
private readonly TokenizerView<TTokenizer, TToken, TTokenType> _tokenizer; private readonly TokenizerView<TTokenizer> _tokenizer;
protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TToken, TTokenType> language, ParserContext context) protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer> language, ParserContext context)
: base(context) : base(context)
{ {
Language = language; Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source); var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView<TTokenizer, TToken, TTokenType>(languageTokenizer); _tokenizer = new TokenizerView<TTokenizer>(languageTokenizer);
Span = new SpanBuilder(CurrentLocation); Span = new SpanBuilder(CurrentLocation);
} }
@ -32,14 +30,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected Action<SpanBuilder> SpanConfig { get; set; } protected Action<SpanBuilder> SpanConfig { get; set; }
protected TToken CurrentToken protected SyntaxToken CurrentToken
{ {
get { return _tokenizer.Current; } get { return _tokenizer.Current; }
} }
protected SyntaxToken CurrentSyntaxToken => CurrentToken?.SyntaxToken; protected SyntaxToken PreviousToken { get; private set; }
protected TToken PreviousToken { get; private set; }
protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation; protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
@ -50,7 +46,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _tokenizer.EndOfFile; } get { return _tokenizer.EndOfFile; }
} }
protected LanguageCharacteristics<TTokenizer, TToken, TTokenType> Language { get; } protected LanguageCharacteristics<TTokenizer> Language { get; }
protected virtual void HandleEmbeddedTransition() protected virtual void HandleEmbeddedTransition()
{ {
@ -63,21 +59,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override void BuildSpan(SpanBuilder span, SourceLocation start, string content) public override void BuildSpan(SpanBuilder span, SourceLocation start, string content)
{ {
foreach (IToken sym in Language.TokenizeString(start, content)) foreach (var token in Language.TokenizeString(start, content))
{ {
span.Accept(sym); span.Accept(token);
} }
} }
protected void Initialize(SpanBuilder span) protected void Initialize(SpanBuilder span)
{ {
if (SpanConfig != null) SpanConfig?.Invoke(span);
{
SpanConfig(span);
}
} }
protected TToken Lookahead(int count) protected SyntaxToken Lookahead(int count)
{ {
if (count < 0) if (count < 0)
{ {
@ -89,7 +82,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// We add 1 in order to store the current token. // We add 1 in order to store the current token.
var tokens = new TToken[count + 1]; var tokens = new SyntaxToken[count + 1];
var currentToken = CurrentToken; var currentToken = CurrentToken;
tokens[0] = currentToken; tokens[0] = currentToken;
@ -120,7 +113,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// <param name="condition">A predicate accepting the token being evaluated and the list of tokens which have been looped through.</param> /// <param name="condition">A predicate accepting the token being evaluated and the list of tokens which have been looped through.</param>
/// <returns>true, if the condition was met. false - if the condition wasn't met and the last token has already been processed.</returns> /// <returns>true, if the condition was met. false - if the condition wasn't met and the last token has already been processed.</returns>
/// <remarks>The list of previous tokens is passed in the reverse order. So the last processed element will be the first one in the list.</remarks> /// <remarks>The list of previous tokens is passed in the reverse order. So the last processed element will be the first one in the list.</remarks>
protected bool LookaheadUntil(Func<TToken, IEnumerable<TToken>, bool> condition) protected bool LookaheadUntil(Func<SyntaxToken, IEnumerable<SyntaxToken>, bool> condition)
{ {
if (condition == null) if (condition == null)
{ {
@ -129,7 +122,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var matchFound = false; var matchFound = false;
var tokens = new List<TToken>(); var tokens = new List<SyntaxToken>();
tokens.Add(CurrentToken); tokens.Add(CurrentToken);
while (true) while (true)
@ -168,14 +161,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Helpers // Helpers
[Conditional("DEBUG")] [Conditional("DEBUG")]
internal void Assert(TTokenType expectedType) internal void Assert(SyntaxKind expectedType)
{ {
Debug.Assert(!EndOfFile && TokenTypeEquals(CurrentToken.Type, expectedType)); Debug.Assert(!EndOfFile && TokenKindEquals(CurrentToken.Kind, expectedType));
} }
abstract protected bool TokenTypeEquals(TTokenType x, TTokenType y); protected abstract bool TokenKindEquals(SyntaxKind x, SyntaxKind y);
protected internal void PutBack(TToken token) protected internal void PutBack(SyntaxToken token)
{ {
if (token != null) if (token != null)
{ {
@ -185,7 +178,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// <summary> /// <summary>
/// Put the specified tokens back in the input stream. The provided list MUST be in the ORDER THE TOKENS WERE READ. The /// Put the specified tokens back in the input stream. The provided list MUST be in the ORDER THE TOKENS WERE READ. The
/// list WILL be reversed and the Putback(TToken) will be called on each item. /// list WILL be reversed and the Putback(SyntaxToken) will be called on each item.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// If a document contains tokens: a, b, c, d, e, f /// If a document contains tokens: a, b, c, d, e, f
@ -194,9 +187,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// that is the correct format for providing to this method. The caller of this method would, /// that is the correct format for providing to this method. The caller of this method would,
/// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order /// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order
/// </remarks> /// </remarks>
protected internal void PutBack(IEnumerable<TToken> tokens) protected internal void PutBack(IEnumerable<SyntaxToken> tokens)
{ {
foreach (TToken token in tokens.Reverse()) foreach (var token in tokens.Reverse())
{ {
PutBack(token); PutBack(token);
} }
@ -212,7 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected internal bool Balance(BalancingModes mode) protected internal bool Balance(BalancingModes mode)
{ {
var left = CurrentToken.Type; var left = CurrentToken.Kind;
var right = Language.FlipBracket(left); var right = Language.FlipBracket(left);
var start = CurrentStart; var start = CurrentStart;
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -228,21 +221,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Balance(mode, left, right, start); return Balance(mode, left, right, start);
} }
protected internal bool Balance(BalancingModes mode, TTokenType left, TTokenType right, SourceLocation start) protected internal bool Balance(BalancingModes mode, SyntaxKind left, SyntaxKind right, SourceLocation start)
{ {
var startPosition = CurrentStart.AbsoluteIndex; var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1; var nesting = 1;
if (!EndOfFile) if (!EndOfFile)
{ {
var syms = new List<TToken>(); var tokens = new List<SyntaxToken>();
do do
{ {
if (IsAtEmbeddedTransition( if (IsAtEmbeddedTransition(
(mode & BalancingModes.AllowCommentsAndTemplates) == BalancingModes.AllowCommentsAndTemplates, (mode & BalancingModes.AllowCommentsAndTemplates) == BalancingModes.AllowCommentsAndTemplates,
(mode & BalancingModes.AllowEmbeddedTransitions) == BalancingModes.AllowEmbeddedTransitions)) (mode & BalancingModes.AllowEmbeddedTransitions) == BalancingModes.AllowEmbeddedTransitions))
{ {
Accept(syms); Accept(tokens);
syms.Clear(); tokens.Clear();
HandleEmbeddedTransition(); HandleEmbeddedTransition();
// Reset backtracking since we've already outputted some spans. // Reset backtracking since we've already outputted some spans.
@ -258,7 +251,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
if (nesting > 0) if (nesting > 0)
{ {
syms.Add(CurrentToken); tokens.Add(CurrentToken);
} }
} }
while (nesting > 0 && NextToken()); while (nesting > 0 && NextToken());
@ -280,29 +273,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
else else
{ {
Accept(syms); Accept(tokens);
} }
} }
else else
{ {
// Accept all the tokens we saw // Accept all the tokens we saw
Accept(syms); Accept(tokens);
} }
} }
return nesting == 0; return nesting == 0;
} }
protected internal bool NextIs(TTokenType type) protected internal bool NextIs(SyntaxKind type)
{ {
return NextIs(sym => sym != null && TokenTypeEquals(type, sym.Type)); return NextIs(token => token != null && TokenKindEquals(type, token.Kind));
} }
protected internal bool NextIs(params TTokenType[] types) protected internal bool NextIs(params SyntaxKind[] types)
{ {
return NextIs(sym => sym != null && types.Any(t => TokenTypeEquals(t, sym.Type))); return NextIs(token => token != null && types.Any(t => TokenKindEquals(t, token.Kind)));
} }
protected internal bool NextIs(Func<TToken, bool> condition) protected internal bool NextIs(Func<SyntaxToken, bool> condition)
{ {
var cur = CurrentToken; var cur = CurrentToken;
if (NextToken()) if (NextToken())
@ -322,14 +315,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false; return false;
} }
protected internal bool Was(TTokenType type) protected internal bool Was(SyntaxKind type)
{ {
return PreviousToken != null && TokenTypeEquals(PreviousToken.Type, type); return PreviousToken != null && TokenKindEquals(PreviousToken.Kind, type);
} }
protected internal bool At(TTokenType type) protected internal bool At(SyntaxKind type)
{ {
return !EndOfFile && CurrentToken != null && TokenTypeEquals(CurrentToken.Type, type); return !EndOfFile && CurrentToken != null && TokenKindEquals(CurrentToken.Kind, type);
} }
protected internal bool AcceptAndMoveNext() protected internal bool AcceptAndMoveNext()
@ -338,11 +331,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return NextToken(); return NextToken();
} }
protected TToken AcceptSingleWhiteSpaceCharacter() protected SyntaxToken AcceptSingleWhiteSpaceCharacter()
{ {
if (Language.IsWhiteSpace(CurrentToken)) if (Language.IsWhiteSpace(CurrentToken))
{ {
Tuple<TToken, TToken> pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace)); var pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
Accept(pair.Item1); Accept(pair.Item1);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
NextToken(); NextToken();
@ -351,19 +344,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null; return null;
} }
protected internal void Accept(IEnumerable<TToken> tokens) protected internal void Accept(IEnumerable<SyntaxToken> tokens)
{ {
foreach (TToken token in tokens) foreach (var token in tokens)
{ {
Accept(token); Accept(token);
} }
} }
protected internal void Accept(TToken token) protected internal void Accept(SyntaxToken token)
{ {
if (token != null) if (token != null)
{ {
foreach (var error in token.Errors) foreach (var error in token.GetDiagnostics())
{ {
Context.ErrorSink.OnError(error); Context.ErrorSink.OnError(error);
} }
@ -372,11 +365,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
protected internal bool AcceptAll(params TTokenType[] types) protected internal bool AcceptAll(params SyntaxKind[] kinds)
{ {
foreach (TTokenType type in types) foreach (var kind in kinds)
{ {
if (CurrentToken == null || !TokenTypeEquals(CurrentToken.Type, type)) if (CurrentToken == null || !TokenKindEquals(CurrentToken.Kind, kind))
{ {
return false; return false;
} }
@ -442,7 +435,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected IDisposable PushSpanConfig(Action<SpanBuilder, Action<SpanBuilder>> newConfig) protected IDisposable PushSpanConfig(Action<SpanBuilder, Action<SpanBuilder>> newConfig)
{ {
Action<SpanBuilder> old = SpanConfig; var old = SpanConfig;
ConfigureSpan(newConfig); ConfigureSpan(newConfig);
return new DisposableAction(() => SpanConfig = old); return new DisposableAction(() => SpanConfig = old);
} }
@ -455,7 +448,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected void ConfigureSpan(Action<SpanBuilder, Action<SpanBuilder>> config) protected void ConfigureSpan(Action<SpanBuilder, Action<SpanBuilder>> config)
{ {
Action<SpanBuilder> prev = SpanConfig; var prev = SpanConfig;
if (config == null) if (config == null)
{ {
SpanConfig = null; SpanConfig = null;
@ -472,9 +465,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Expected(Language.GetKnownTokenType(type)); Expected(Language.GetKnownTokenType(type));
} }
protected internal void Expected(params TTokenType[] types) protected internal void Expected(params SyntaxKind[] types)
{ {
Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Type)); Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Kind));
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
@ -483,7 +476,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Optional(Language.GetKnownTokenType(type)); return Optional(Language.GetKnownTokenType(type));
} }
protected internal bool Optional(TTokenType type) protected internal bool Optional(SyntaxKind type)
{ {
if (At(type)) if (At(type))
{ {
@ -503,61 +496,61 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true; return true;
} }
protected internal void AcceptWhile(TTokenType type) protected internal void AcceptWhile(SyntaxKind type)
{ {
AcceptWhile(sym => TokenTypeEquals(type, sym.Type)); AcceptWhile(token => TokenKindEquals(type, token.Kind));
} }
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format // We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
protected internal void AcceptWhile(TTokenType type1, TTokenType type2) protected internal void AcceptWhile(SyntaxKind type1, SyntaxKind type2)
{ {
AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type)); AcceptWhile(token => TokenKindEquals(type1, token.Kind) || TokenKindEquals(type2, token.Kind));
} }
protected internal void AcceptWhile(TTokenType type1, TTokenType type2, TTokenType type3) protected internal void AcceptWhile(SyntaxKind type1, SyntaxKind type2, SyntaxKind type3)
{ {
AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type) || TokenTypeEquals(type3, sym.Type)); AcceptWhile(token => TokenKindEquals(type1, token.Kind) || TokenKindEquals(type2, token.Kind) || TokenKindEquals(type3, token.Kind));
} }
protected internal void AcceptWhile(params TTokenType[] types) protected internal void AcceptWhile(params SyntaxKind[] types)
{ {
AcceptWhile(sym => types.Any(expected => TokenTypeEquals(expected, sym.Type))); AcceptWhile(token => types.Any(expected => TokenKindEquals(expected, token.Kind)));
} }
protected internal void AcceptUntil(TTokenType type) protected internal void AcceptUntil(SyntaxKind type)
{ {
AcceptWhile(sym => !TokenTypeEquals(type, sym.Type)); AcceptWhile(token => !TokenKindEquals(type, token.Kind));
} }
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format // We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
protected internal void AcceptUntil(TTokenType type1, TTokenType type2) protected internal void AcceptUntil(SyntaxKind type1, SyntaxKind type2)
{ {
AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type)); AcceptWhile(token => !TokenKindEquals(type1, token.Kind) && !TokenKindEquals(type2, token.Kind));
} }
protected internal void AcceptUntil(TTokenType type1, TTokenType type2, TTokenType type3) protected internal void AcceptUntil(SyntaxKind type1, SyntaxKind type2, SyntaxKind type3)
{ {
AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type) && !TokenTypeEquals(type3, sym.Type)); AcceptWhile(token => !TokenKindEquals(type1, token.Kind) && !TokenKindEquals(type2, token.Kind) && !TokenKindEquals(type3, token.Kind));
} }
protected internal void AcceptUntil(params TTokenType[] types) protected internal void AcceptUntil(params SyntaxKind[] types)
{ {
AcceptWhile(sym => types.All(expected => !TokenTypeEquals(expected, sym.Type))); AcceptWhile(token => types.All(expected => !TokenKindEquals(expected, token.Kind)));
} }
protected internal void AcceptWhile(Func<TToken, bool> condition) protected internal void AcceptWhile(Func<SyntaxToken, bool> condition)
{ {
Accept(ReadWhileLazy(condition)); Accept(ReadWhileLazy(condition));
} }
protected internal IEnumerable<TToken> ReadWhile(Func<TToken, bool> condition) protected internal IEnumerable<SyntaxToken> ReadWhile(Func<SyntaxToken, bool> condition)
{ {
return ReadWhileLazy(condition).ToList(); return ReadWhileLazy(condition).ToList();
} }
protected TToken AcceptWhiteSpaceInLines() protected SyntaxToken AcceptWhiteSpaceInLines()
{ {
TToken lastWs = null; SyntaxToken lastWs = null;
while (Language.IsWhiteSpace(CurrentToken) || Language.IsNewLine(CurrentToken)) while (Language.IsWhiteSpace(CurrentToken) || Language.IsNewLine(CurrentToken))
{ {
// Capture the previous whitespace node // Capture the previous whitespace node
@ -591,7 +584,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Don't open this to sub classes because it's lazy but it looks eager. // Don't open this to sub classes because it's lazy but it looks eager.
// You have to advance the Enumerable to read the next characters. // You have to advance the Enumerable to read the next characters.
internal IEnumerable<TToken> ReadWhileLazy(Func<TToken, bool> condition) internal IEnumerable<SyntaxToken> ReadWhileLazy(Func<SyntaxToken, bool> condition)
{ {
while (EnsureCurrent() && condition(CurrentToken)) while (EnsureCurrent() && condition(CurrentToken))
{ {

12
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
View File

@ -1,12 +1,12 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class TokenizerView<TTokenizer, TToken, TTokenType> internal class TokenizerView<TTokenizer>
where TTokenType : struct where TTokenizer : Tokenizer
where TTokenizer : Tokenizer<TToken, TTokenType>
where TToken : TokenBase<TTokenType>
{ {
public TokenizerView(TTokenizer tokenizer) public TokenizerView(TTokenizer tokenizer)
{ {
@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public TTokenizer Tokenizer { get; private set; } public TTokenizer Tokenizer { get; private set; }
public bool EndOfFile { get; private set; } public bool EndOfFile { get; private set; }
public TToken Current { get; private set; } public SyntaxToken Current { get; private set; }
public ITextDocument Source public ITextDocument Source
{ {
@ -29,7 +29,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return !EndOfFile; return !EndOfFile;
} }
public void PutBack(TToken token) public void PutBack(SyntaxToken token)
{ {
Source.Position -= token.Content.Length; Source.Position -= token.Content.Length;
Current = null; Current = null;

64
src/Microsoft.AspNetCore.Razor.Language/Syntax/GreenNode.cs
View File

@ -377,6 +377,70 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
} }
#endregion #endregion
#region Equivalence
public virtual bool IsEquivalentTo(GreenNode other)
{
if (this == other)
{
return true;
}
if (other == null)
{
return false;
}
return EquivalentToInternal(this, other);
}
private static bool EquivalentToInternal(GreenNode node1, GreenNode node2)
{
if (node1.Kind != node2.Kind)
{
// A single-element list is usually represented as just a single node,
// but can be represented as a List node with one child. Move to that
// child if necessary.
if (node1.IsList && node1.SlotCount == 1)
{
node1 = node1.GetSlot(0);
}
if (node2.IsList && node2.SlotCount == 1)
{
node2 = node2.GetSlot(0);
}
if (node1.Kind != node2.Kind)
{
return false;
}
}
if (node1.FullWidth != node2.FullWidth)
{
return false;
}
var n = node1.SlotCount;
if (n != node2.SlotCount)
{
return false;
}
for (var i = 0; i < n; i++)
{
var node1Child = node1.GetSlot(i);
var node2Child = node2.GetSlot(i);
if (node1Child != null && node2Child != null && !node1Child.IsEquivalentTo(node2Child))
{
return false;
}
}
return true;
}
#endregion
#region Factories #region Factories
public virtual GreenNode CreateList(IEnumerable<GreenNode> nodes, bool alwaysCreateListNode = false) public virtual GreenNode CreateList(IEnumerable<GreenNode> nodes, bool alwaysCreateListNode = false)
{ {

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/HtmlTextTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class HtmlTextTokenSyntax : SyntaxToken
{
internal HtmlTextTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.HtmlTextTokenSyntax Green => (InternalSyntax.HtmlTextTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.HtmlTextTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as HtmlTextTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.HtmlTextTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as HtmlTextTokenSyntax;
}
}
}

53
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/HtmlTextTokenSyntax.cs
View File

@ -1,53 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class HtmlTextTokenSyntax : SyntaxToken
{
internal HtmlTextTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.HtmlTextLiteralToken, text, null, null, diagnostics, null)
{
}
internal HtmlTextTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.HtmlTextLiteralToken, text, leadingTrivia, trailingTrivia)
{
}
protected HtmlTextTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected HtmlTextTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position)
{
return new Syntax.HtmlTextTokenSyntax(this, parent, position);
}
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new HtmlTextTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

50
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/NewLineTokenSyntax.cs
View File

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class NewLineTokenSyntax : SyntaxToken
{
internal NewLineTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.NewLine, text, null, null, diagnostics, null)
{
}
internal NewLineTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.NewLine, text, leadingTrivia, trailingTrivia)
{
}
protected NewLineTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected NewLineTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.NewLineTokenSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new NewLineTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

45
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/PunctuationSyntax.cs
View File

@ -1,45 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class PunctuationSyntax : SyntaxToken
{
internal PunctuationSyntax(SyntaxKind kind, string name, RazorDiagnostic[] diagnostics)
: this(kind, name, null, null, diagnostics, null)
{
}
internal PunctuationSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: this(kind, name, leadingTrivia, trailingTrivia, null, null)
{
}
internal PunctuationSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.PunctuationSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new PunctuationSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new PunctuationSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new PunctuationSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new PunctuationSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

26
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxFactory.cs
View File

@ -1,6 +1,9 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Collections.Generic;
using System.Linq;
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{ {
internal static class SyntaxFactory internal static class SyntaxFactory
@ -10,29 +13,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return new HtmlTextSyntax(textTokens.Node); return new HtmlTextSyntax(textTokens.Node);
} }
internal static HtmlTextTokenSyntax HtmlTextToken(string text, params RazorDiagnostic[] diagnostics) internal static SyntaxToken Token(SyntaxKind kind, string content, IEnumerable<RazorDiagnostic> diagnostics)
{ {
return new HtmlTextTokenSyntax(text, diagnostics); return Token(kind, content, diagnostics.ToArray());
} }
internal static WhitespaceTokenSyntax WhitespaceToken(string text, params RazorDiagnostic[] diagnostics) internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
{ {
return new WhitespaceTokenSyntax(text, diagnostics); return new SyntaxToken(kind, content, diagnostics);
}
internal static NewLineTokenSyntax NewLineToken(string text, params RazorDiagnostic[] diagnostics)
{
return new NewLineTokenSyntax(text, diagnostics);
}
internal static PunctuationSyntax Punctuation(SyntaxKind syntaxKind, string text, params RazorDiagnostic[] diagnostics)
{
return new PunctuationSyntax(syntaxKind, text, diagnostics);
}
internal static UnknownTokenSyntax UnknownToken(string text, params RazorDiagnostic[] diagnostics)
{
return new UnknownTokenSyntax(text, diagnostics);
} }
} }
} }

103
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxToken.cs
View File

@ -4,32 +4,39 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using Microsoft.AspNetCore.Razor.Language.Legacy;
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{ {
internal abstract class SyntaxToken : GreenNode internal class SyntaxToken : GreenNode
{ {
internal SyntaxToken(SyntaxKind tokenKind, string text, GreenNode leadingTrivia, GreenNode trailingTrivia) internal SyntaxToken(SyntaxKind kind, string content, RazorDiagnostic[] diagnostics)
: base(tokenKind, text.Length) : base(kind, content.Length, diagnostics, annotations: null)
{ {
Text = text; Content = content;
}
internal SyntaxToken(SyntaxKind kind, string content, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, content.Length)
{
Content = content;
LeadingTrivia = leadingTrivia; LeadingTrivia = leadingTrivia;
AdjustFlagsAndWidth(leadingTrivia); AdjustFlagsAndWidth(leadingTrivia);
TrailingTrivia = trailingTrivia; TrailingTrivia = trailingTrivia;
AdjustFlagsAndWidth(trailingTrivia); AdjustFlagsAndWidth(trailingTrivia);
} }
internal SyntaxToken(SyntaxKind tokenKind, string text, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations) internal SyntaxToken(SyntaxKind kind, string content, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(tokenKind, text.Length, diagnostics, annotations) : base(kind, content.Length, diagnostics, annotations)
{ {
Text = text; Content = content;
LeadingTrivia = leadingTrivia; LeadingTrivia = leadingTrivia;
AdjustFlagsAndWidth(leadingTrivia); AdjustFlagsAndWidth(leadingTrivia);
TrailingTrivia = trailingTrivia; TrailingTrivia = trailingTrivia;
AdjustFlagsAndWidth(trailingTrivia); AdjustFlagsAndWidth(trailingTrivia);
} }
public string Text { get; } public string Content { get; }
public GreenNode LeadingTrivia { get; } public GreenNode LeadingTrivia { get; }
@ -37,7 +44,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
internal override bool IsToken => true; internal override bool IsToken => true;
public override int Width => Text.Length; public override int Width => Content.Length;
internal override SyntaxNode CreateRed(SyntaxNode parent, int position)
{
return new Syntax.SyntaxToken(this, parent, position);
}
protected override void WriteTokenTo(TextWriter writer, bool leading, bool trailing) protected override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{ {
@ -50,7 +62,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
} }
} }
writer.Write(Text); writer.Write(Content);
if (trailing) if (trailing)
{ {
@ -87,14 +99,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return TokenWithLeadingTrivia(trivia); return TokenWithLeadingTrivia(trivia);
} }
public abstract SyntaxToken TokenWithLeadingTrivia(GreenNode trivia); public virtual SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new SyntaxToken(Kind, Content, trivia, TrailingTrivia, GetDiagnostics(), GetAnnotations());
}
public sealed override GreenNode WithTrailingTrivia(GreenNode trivia) public sealed override GreenNode WithTrailingTrivia(GreenNode trivia)
{ {
return TokenWithTrailingTrivia(trivia); return TokenWithTrailingTrivia(trivia);
} }
public abstract SyntaxToken TokenWithTrailingTrivia(GreenNode trivia); public virtual SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new SyntaxToken(Kind, Content, LeadingTrivia, trivia, GetDiagnostics(), GetAnnotations());
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new SyntaxToken(Kind, Content, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new SyntaxToken(Kind, Content, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
protected override sealed int GetSlotCount() protected override sealed int GetSlotCount()
{ {
@ -103,7 +131,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
internal override sealed GreenNode GetSlot(int index) internal override sealed GreenNode GetSlot(int index)
{ {
throw new InvalidOperationException(); throw new InvalidOperationException("Tokens don't have slots.");
} }
internal override GreenNode Accept(SyntaxVisitor visitor) internal override GreenNode Accept(SyntaxVisitor visitor)
@ -111,9 +139,56 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return visitor.VisitSyntaxToken(this); return visitor.VisitSyntaxToken(this);
} }
public override bool IsEquivalentTo(GreenNode other)
{
if (!base.IsEquivalentTo(other))
{
return false;
}
var otherToken = (SyntaxToken)other;
if (Content != otherToken.Content)
{
return false;
}
var thisLeading = GetLeadingTrivia();
var otherLeading = otherToken.GetLeadingTrivia();
if (thisLeading != otherLeading)
{
if (thisLeading == null || otherLeading == null)
{
return false;
}
if (!thisLeading.IsEquivalentTo(otherLeading))
{
return false;
}
}
var thisTrailing = GetTrailingTrivia();
var otherTrailing = otherToken.GetTrailingTrivia();
if (thisTrailing != otherTrailing)
{
if (thisTrailing == null || otherTrailing == null)
{
return false;
}
if (!thisTrailing.IsEquivalentTo(otherTrailing))
{
return false;
}
}
return true;
}
public override string ToString() public override string ToString()
{ {
return Text; return Content;
} }
} }
} }

15
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxTrivia.cs
View File

@ -76,5 +76,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{ {
return new SyntaxTrivia(Kind, Text, GetDiagnostics(), annotations); return new SyntaxTrivia(Kind, Text, GetDiagnostics(), annotations);
} }
public override bool IsEquivalentTo(GreenNode other)
{
if (!base.IsEquivalentTo(other))
{
return false;
}
if (Text != ((SyntaxTrivia)other).Text)
{
return false;
}
return true;
}
} }
} }

50
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/UnknownTokenSyntax.cs
View File

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class UnknownTokenSyntax : SyntaxToken
{
internal UnknownTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.Unknown, text, null, null, diagnostics, null)
{
}
internal UnknownTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.Unknown, text, leadingTrivia, trailingTrivia)
{
}
protected UnknownTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected UnknownTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.UnknownTokenSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new UnknownTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

50
src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/WhitespaceTokenSyntax.cs
View File

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal class WhitespaceTokenSyntax : SyntaxToken
{
internal WhitespaceTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
: base(SyntaxKind.Whitespace, text, null, null, diagnostics, null)
{
}
internal WhitespaceTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(SyntaxKind.Whitespace, text, leadingTrivia, trailingTrivia)
{
}
protected WhitespaceTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
: base(kind, name, leadingTrivia, trailingTrivia)
{
}
protected WhitespaceTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
: base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
{
}
internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.WhitespaceTokenSyntax(this, parent, position);
public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
{
return new WhitespaceTokenSyntax(Kind, Text, trivia, TrailingTrivia);
}
public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
{
return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, trivia);
}
internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
{
return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
}
internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
{
return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/NewLineTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class NewLineTokenSyntax : SyntaxToken
{
internal NewLineTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.NewLineTokenSyntax Green => (InternalSyntax.NewLineTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.NewLineTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as NewLineTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.NewLineTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as NewLineTokenSyntax;
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/PunctuationSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class PunctuationSyntax : SyntaxToken
{
internal PunctuationSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.PunctuationSyntax Green => (InternalSyntax.PunctuationSyntax)base.Green;
public string Punctuation => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.PunctuationSyntax(Kind, Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as PunctuationSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.PunctuationSyntax(Kind, Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as PunctuationSyntax;
}
}
}

13
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxFactory.cs Normal file
View File

@ -0,0 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal static class SyntaxFactory
{
internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
{
return new SyntaxToken(InternalSyntax.SyntaxFactory.Token(kind, content), parent: null, position: 0);
}
}
}

85
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxKind.cs
View File

@ -5,28 +5,93 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
internal enum SyntaxKind : byte internal enum SyntaxKind : byte
{ {
#region Nodes
HtmlText,
HtmlDocument,
HtmlDeclaration,
#endregion
#region Tokens
// Common
Unknown, Unknown,
List, List,
Whitespace, Whitespace,
NewLine, NewLine,
Colon,
QuestionMark,
RightBracket,
LeftBracket,
Equals,
Transition,
// HTML // HTML
HtmlText, HtmlTextLiteral,
HtmlDocument,
HtmlDeclaration,
HtmlTextLiteralToken,
OpenAngle, OpenAngle,
Bang, Bang,
ForwardSlash, ForwardSlash,
QuestionMark,
DoubleHyphen, DoubleHyphen,
LeftBracket,
CloseAngle, CloseAngle,
RightBracket,
Equals,
DoubleQuote, DoubleQuote,
SingleQuote, SingleQuote,
Transition,
Colon, // CSharp literals
Identifier,
Keyword,
IntegerLiteral,
CSharpComment,
RealLiteral,
CharacterLiteral,
StringLiteral,
// CSharp operators
Arrow,
Minus,
Decrement,
MinusAssign,
NotEqual,
Not,
Modulo,
ModuloAssign,
AndAssign,
And,
DoubleAnd,
LeftParenthesis,
RightParenthesis,
Star,
MultiplyAssign,
Comma,
Dot,
Slash,
DivideAssign,
DoubleColon,
Semicolon,
NullCoalesce,
XorAssign,
Xor,
LeftBrace,
OrAssign,
DoubleOr,
Or,
RightBrace,
Tilde,
Plus,
PlusAssign,
Increment,
LessThan,
LessThanEqual,
LeftShift,
LeftShiftAssign,
Assign,
GreaterThan,
GreaterThanEqual,
RightShift,
RightShiftAssign,
Hash,
// Razor specific
RazorComment,
RazorCommentStar,
RazorCommentTransition,
#endregion
} }
} }

15
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxNode.cs
View File

@ -239,6 +239,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
return Green.GetAnnotations(); return Green.GetAnnotations();
} }
public bool IsEquivalentTo(SyntaxNode other)
{
if (this == other)
{
return true;
}
if (other == null)
{
return false;
}
return Green.IsEquivalentTo(other.Green);
}
public override string ToString() public override string ToString()
{ {
return Green.ToString(); return Green.ToString();

73
src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxToken.cs
View File

@ -3,29 +3,70 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Legacy;
namespace Microsoft.AspNetCore.Razor.Language.Syntax namespace Microsoft.AspNetCore.Razor.Language.Syntax
{ {
internal abstract class SyntaxToken : SyntaxNode internal class SyntaxToken : SyntaxNode
{ {
internal SyntaxToken(GreenNode green, SyntaxNode parent, int position) internal SyntaxToken(GreenNode green, SyntaxNode parent, int position)
: this(green, parent, null, position)
{
}
// Temporary plumbing
internal SyntaxToken(GreenNode green, SyntaxNode parent, Span parentSpan, int position)
: base(green, parent, position) : base(green, parent, position)
{ {
Debug.Assert(parent == null || !parent.Green.IsList, "list cannot be a parent");
Debug.Assert(green == null || green.IsToken, "green must be a token");
ParentSpan = parentSpan;
}
// Temporary plumbing
internal Span ParentSpan { get; }
// Temporary plumbing
internal SourceLocation Start
{
get
{
if (ParentSpan == null)
{
return SourceLocation.Undefined;
}
var tracker = new SourceLocationTracker(ParentSpan.Start);
for (var i = 0; i < ParentSpan.Tokens.Count; i++)
{
var token = ParentSpan.Tokens[i];
if (object.ReferenceEquals(this, token))
{
break;
}
tracker.UpdateLocation(token.Content);
}
return tracker.CurrentLocation;
}
} }
internal new InternalSyntax.SyntaxToken Green => (InternalSyntax.SyntaxToken)base.Green; internal new InternalSyntax.SyntaxToken Green => (InternalSyntax.SyntaxToken)base.Green;
public string Text => Green.Text; public string Content => Green.Content;
internal override sealed SyntaxNode GetCachedSlot(int index) internal override sealed SyntaxNode GetCachedSlot(int index)
{ {
throw new InvalidOperationException(); throw new InvalidOperationException("Tokens can't have slots.");
} }
internal override sealed SyntaxNode GetNodeSlot(int slot) internal override sealed SyntaxNode GetNodeSlot(int slot)
{ {
throw new InvalidOperationException(); throw new InvalidOperationException("Tokens can't have slots.");
} }
internal override SyntaxNode Accept(SyntaxVisitor visitor) internal override SyntaxNode Accept(SyntaxVisitor visitor)
@ -33,24 +74,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
return visitor.VisitSyntaxToken(this); return visitor.VisitSyntaxToken(this);
} }
internal abstract SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia); public SyntaxToken WithLeadingTrivia(SyntaxNode trivia)
{
return Green != null
? new SyntaxToken(Green.WithLeadingTrivia(trivia.Green), parent: null, position: 0)
: default(SyntaxToken);
}
internal abstract SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia); public SyntaxToken WithTrailingTrivia(SyntaxNode trivia)
{
public SyntaxToken WithLeadingTrivia(SyntaxNode trivia) => WithLeadingTriviaCore(trivia); return Green != null
? new SyntaxToken(Green.WithTrailingTrivia(trivia.Green), parent: null, position: 0)
public SyntaxToken WithTrailingTrivia(SyntaxNode trivia) => WithTrailingTriviaCore(trivia); : default(SyntaxToken);
}
public SyntaxToken WithLeadingTrivia(IEnumerable<SyntaxTrivia> trivia) public SyntaxToken WithLeadingTrivia(IEnumerable<SyntaxTrivia> trivia)
{ {
var greenList = trivia?.Select(t => t.Green); var greenList = trivia?.Select(t => t.Green);
return WithLeadingTriviaCore(Green.CreateList(greenList)?.CreateRed()); return WithLeadingTrivia(Green.CreateList(greenList)?.CreateRed());
} }
public SyntaxToken WithTrailingTrivia(IEnumerable<SyntaxTrivia> trivia) public SyntaxToken WithTrailingTrivia(IEnumerable<SyntaxTrivia> trivia)
{ {
var greenList = trivia?.Select(t => t.Green); var greenList = trivia?.Select(t => t.Green);
return WithTrailingTriviaCore(Green.CreateList(greenList)?.CreateRed()); return WithTrailingTrivia(Green.CreateList(greenList)?.CreateRed());
} }
public override SyntaxTriviaList GetLeadingTrivia() public override SyntaxTriviaList GetLeadingTrivia()
@ -85,7 +132,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
public override string ToString() public override string ToString()
{ {
return Text; return Content;
} }
} }
} }

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/UnknownTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class UnknownTokenSyntax : SyntaxToken
{
internal UnknownTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.UnknownTokenSyntax Green => (InternalSyntax.UnknownTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.UnknownTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as UnknownTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.UnknownTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as UnknownTokenSyntax;
}
}
}

27
src/Microsoft.AspNetCore.Razor.Language/Syntax/WhitespaceTokenSyntax.cs
View File

@ -1,27 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
internal class WhitespaceTokenSyntax : SyntaxToken
{
internal WhitespaceTokenSyntax(GreenNode green, SyntaxNode parent, int position)
: base(green, parent, position)
{
}
internal new InternalSyntax.WhitespaceTokenSyntax Green => (InternalSyntax.WhitespaceTokenSyntax)base.Green;
public string Value => Text;
internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.WhitespaceTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as WhitespaceTokenSyntax;
}
internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
{
return new InternalSyntax.WhitespaceTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as WhitespaceTokenSyntax;
}
}
}

12
src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorCompletionFactsService.cs
View File

@ -7,6 +7,7 @@ using System.ComponentModel.Composition;
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language; using Microsoft.AspNetCore.Razor.Language;
using Microsoft.AspNetCore.Razor.Language.Legacy; using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.VisualStudio.Editor.Razor namespace Microsoft.VisualStudio.Editor.Razor
{ {
@ -83,16 +84,11 @@ namespace Microsoft.VisualStudio.Editor.Razor
} }
// Internal for testing // Internal for testing
internal static bool IsDirectiveCompletableToken(IToken token) internal static bool IsDirectiveCompletableToken(SyntaxToken token)
{ {
if (!(token is CSharpToken csharpToken)) return token.Kind == SyntaxKind.Identifier ||
{
return false;
}
return csharpToken.Type == CSharpTokenType.Identifier ||
// Marker symbol // Marker symbol
csharpToken.Type == CSharpTokenType.Unknown; token.Kind == SyntaxKind.Unknown;
} }
} }
} }

3
src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
View File

@ -143,8 +143,7 @@ namespace Microsoft.VisualStudio.Editor.Razor
{ {
return currentChild is Span currentSpan && return currentChild is Span currentSpan &&
currentSpan.Tokens.Count == 1 && currentSpan.Tokens.Count == 1 &&
currentSpan.Tokens[0] is CSharpToken symbol && currentSpan.Tokens[0].Kind == SyntaxKind.LeftBrace;
symbol.Type == CSharpTokenType.LeftBrace;
} }
} }
} }

12
src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
View File

@ -206,17 +206,5 @@ namespace Microsoft.VisualStudio.Editor.Razor
return false; return false;
} }
private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.IToken symbol)
{
if (!(symbol is CSharpToken csharpSymbol))
{
return false;
}
return csharpSymbol.Type == CSharpTokenType.Identifier ||
// Marker symbol
csharpSymbol.Type == CSharpTokenType.Unknown;
}
} }
} }

5
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -13,7 +14,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Arrange // Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero); var spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text)); spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = spanBuilder.Build(); var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder() var blockBuilder = new BlockBuilder()
{ {
@ -29,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var parentBlock = blockBuilder.Build(); var parentBlock = blockBuilder.Build();
var originalBlockLength = parentBlock.Length; var originalBlockLength = parentBlock.Length;
spanBuilder = new SpanBuilder(SourceLocation.Zero); spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text)); spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
span.ReplaceWith(spanBuilder); span.ReplaceWith(spanBuilder);
// Wire up parents now so we can re-trigger ChildChanged to cause cache refresh. // Wire up parents now so we can re-trigger ChildChanged to cause cache refresh.

2
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
View File

@ -11,7 +11,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void GetSample_RightShiftAssign_ReturnsCorrectToken() public void GetSample_RightShiftAssign_ReturnsCorrectToken()
{ {
// Arrange & Act // Arrange & Act
var token = CSharpLanguageCharacteristics.Instance.GetSample(CSharpTokenType.RightShiftAssign); var token = CSharpLanguageCharacteristics.Instance.GetSample(SyntaxKind.RightShiftAssign);
// Assert // Assert
Assert.Equal(">>=", token); Assert.Equal(">>=", token);

45
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
View File

@ -1,22 +1,23 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
{ {
private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining; private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact] [Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment() public void Next_Ignores_Star_At_EOF_In_RazorComment()
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *", "@* Foo * Bar * Baz *",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken(" Foo * Bar * Baz *", CSharpTokenType.RazorComment)); SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz *"));
} }
[Fact] [Fact]
@ -24,11 +25,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *@", "@* Foo * Bar * Baz *@",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken(" Foo * Bar * Baz ", CSharpTokenType.RazorComment), SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz "),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken("@", CSharpTokenType.RazorCommentTransition)); SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
} }
[Fact] [Fact]
@ -36,59 +37,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo Bar Baz *@", "@* Foo Bar Baz *@",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken(" Foo Bar Baz ", CSharpTokenType.RazorComment), SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo Bar Baz "),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken("@", CSharpTokenType.RazorCommentTransition)); SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
} }
[Fact] [Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment() public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{ {
TestTokenizer("// Foo Bar Baz", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment)); TestTokenizer("// Foo Bar Baz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"));
} }
[Fact] [Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline() public void Single_Line_Comment_Is_Terminated_By_Newline()
{ {
TestTokenizer("// Foo Bar Baz\na", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment), IgnoreRemaining); TestTokenizer("// Foo Bar Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect() public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{ {
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpToken("// Foo/*Bar*/ Baz", CSharpTokenType.Comment), IgnoreRemaining); TestTokenizer("// Foo/*Bar*/ Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo/*Bar*/ Baz"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment() public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{ {
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment)); TestTokenizer("/* Foo\nBar\nBaz */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"));
} }
[Fact] [Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence() public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{ {
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz */a", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF() public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{ {
TestTokenizer("/* Foo\nBar\nBaz", new CSharpToken("/* Foo\nBar\nBaz", CSharpTokenType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence() public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{ {
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpToken("/* Foo/*\nBar\nBaz*/", CSharpTokenType.Comment), IgnoreRemaining); TestTokenizer("/* Foo/*\nBar\nBaz*/ */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo/*\nBar\nBaz*/"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence() public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{ {
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpToken("/* Foo\nBar\nBaz* */", CSharpTokenType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz* */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz* */"), IgnoreRemaining);
} }
} }
} }

27
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -10,73 +11,73 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact] [Fact]
public void Simple_Identifier_Is_Recognized() public void Simple_Identifier_Is_Recognized()
{ {
TestTokenizer("foo", new CSharpToken("foo", CSharpTokenType.Identifier)); TestTokenizer("foo", SyntaxFactory.Token(SyntaxKind.Identifier, "foo"));
} }
[Fact] [Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized() public void Identifier_Starting_With_Underscore_Is_Recognized()
{ {
TestTokenizer("_foo", new CSharpToken("_foo", CSharpTokenType.Identifier)); TestTokenizer("_foo", SyntaxFactory.Token(SyntaxKind.Identifier, "_foo"));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Digits() public void Identifier_Can_Contain_Digits()
{ {
TestTokenizer("foo4", new CSharpToken("foo4", CSharpTokenType.Identifier)); TestTokenizer("foo4", SyntaxFactory.Token(SyntaxKind.Identifier, "foo4"));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Titlecase_Letter() public void Identifier_Can_Start_With_Titlecase_Letter()
{ {
TestTokenizer("ῼfoo", new CSharpToken("ῼfoo", CSharpTokenType.Identifier)); TestTokenizer("ῼfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ῼfoo"));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Letter_Modifier() public void Identifier_Can_Start_With_Letter_Modifier()
{ {
TestTokenizer("ᵊfoo", new CSharpToken("ᵊfoo", CSharpTokenType.Identifier)); TestTokenizer("ᵊfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ᵊfoo"));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Other_Letter() public void Identifier_Can_Start_With_Other_Letter()
{ {
TestTokenizer("ƻfoo", new CSharpToken("ƻfoo", CSharpTokenType.Identifier)); TestTokenizer("ƻfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ƻfoo"));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Number_Letter() public void Identifier_Can_Start_With_Number_Letter()
{ {
TestTokenizer("Ⅽool", new CSharpToken("Ⅽool", CSharpTokenType.Identifier)); TestTokenizer("Ⅽool", SyntaxFactory.Token(SyntaxKind.Identifier, "Ⅽool"));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark() public void Identifier_Can_Contain_Non_Spacing_Mark()
{ {
TestTokenizer("foo\u0300", new CSharpToken("foo\u0300", CSharpTokenType.Identifier)); TestTokenizer("foo\u0300", SyntaxFactory.Token(SyntaxKind.Identifier, "foo\u0300"));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark() public void Identifier_Can_Contain_Spacing_Combining_Mark()
{ {
TestTokenizer("fooः", new CSharpToken("fooः", CSharpTokenType.Identifier)); TestTokenizer("fooः", SyntaxFactory.Token(SyntaxKind.Identifier, "fooः"));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Non_English_Digit() public void Identifier_Can_Contain_Non_English_Digit()
{ {
TestTokenizer("foo١", new CSharpToken("foo١", CSharpTokenType.Identifier)); TestTokenizer("foo١", SyntaxFactory.Token(SyntaxKind.Identifier, "foo١"));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Connector_Punctuation() public void Identifier_Can_Contain_Connector_Punctuation()
{ {
TestTokenizer("foo‿bar", new CSharpToken("foo‿bar", CSharpTokenType.Identifier)); TestTokenizer("foo‿bar", SyntaxFactory.Token(SyntaxKind.Identifier, "foo‿bar"));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Format_Character() public void Identifier_Can_Contain_Format_Character()
{ {
TestTokenizer("foo؃bar", new CSharpToken("foo؃bar", CSharpTokenType.Identifier)); TestTokenizer("foo؃bar", SyntaxFactory.Token(SyntaxKind.Identifier, "foo؃bar"));
} }
[Fact] [Fact]
@ -164,7 +165,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType) private void TestKeyword(string keyword, CSharpKeyword keywordType)
{ {
TestTokenizer(keyword, new CSharpToken(keyword, CSharpTokenType.Keyword) { Keyword = keywordType }); TestTokenizer(keyword, SyntaxFactory.Token(SyntaxKind.Keyword, keyword));
} }
} }
} }

161
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
View File

@ -2,286 +2,287 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System; using System;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase
{ {
private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining; private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact] [Fact]
public void Simple_Integer_Literal_Is_Recognized() public void Simple_Integer_Literal_Is_Recognized()
{ {
TestSingleToken("01189998819991197253", CSharpTokenType.IntegerLiteral); TestSingleToken("01189998819991197253", SyntaxKind.IntegerLiteral);
} }
[Fact] [Fact]
public void Integer_Type_Suffix_Is_Recognized() public void Integer_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("42U", CSharpTokenType.IntegerLiteral); TestSingleToken("42U", SyntaxKind.IntegerLiteral);
TestSingleToken("42u", CSharpTokenType.IntegerLiteral); TestSingleToken("42u", SyntaxKind.IntegerLiteral);
TestSingleToken("42L", CSharpTokenType.IntegerLiteral); TestSingleToken("42L", SyntaxKind.IntegerLiteral);
TestSingleToken("42l", CSharpTokenType.IntegerLiteral); TestSingleToken("42l", SyntaxKind.IntegerLiteral);
TestSingleToken("42UL", CSharpTokenType.IntegerLiteral); TestSingleToken("42UL", SyntaxKind.IntegerLiteral);
TestSingleToken("42Ul", CSharpTokenType.IntegerLiteral); TestSingleToken("42Ul", SyntaxKind.IntegerLiteral);
TestSingleToken("42uL", CSharpTokenType.IntegerLiteral); TestSingleToken("42uL", SyntaxKind.IntegerLiteral);
TestSingleToken("42ul", CSharpTokenType.IntegerLiteral); TestSingleToken("42ul", SyntaxKind.IntegerLiteral);
TestSingleToken("42LU", CSharpTokenType.IntegerLiteral); TestSingleToken("42LU", SyntaxKind.IntegerLiteral);
TestSingleToken("42Lu", CSharpTokenType.IntegerLiteral); TestSingleToken("42Lu", SyntaxKind.IntegerLiteral);
TestSingleToken("42lU", CSharpTokenType.IntegerLiteral); TestSingleToken("42lU", SyntaxKind.IntegerLiteral);
TestSingleToken("42lu", CSharpTokenType.IntegerLiteral); TestSingleToken("42lu", SyntaxKind.IntegerLiteral);
} }
[Fact] [Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix() public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{ {
TestTokenizer("42a", new CSharpToken("42", CSharpTokenType.IntegerLiteral), IgnoreRemaining); TestTokenizer("42a", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "42"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Simple_Hex_Literal_Is_Recognized() public void Simple_Hex_Literal_Is_Recognized()
{ {
TestSingleToken("0x0123456789ABCDEF", CSharpTokenType.IntegerLiteral); TestSingleToken("0x0123456789ABCDEF", SyntaxKind.IntegerLiteral);
} }
[Fact] [Fact]
public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal() public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal()
{ {
TestSingleToken("0xDEADBEEFU", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFU", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFu", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFu", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFL", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFL", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFl", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFl", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFUL", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFUL", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFUl", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFUl", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFuL", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFuL", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFul", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFul", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFLU", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFLU", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFLu", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFLu", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFlU", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFlU", SyntaxKind.IntegerLiteral);
TestSingleToken("0xDEADBEEFlu", CSharpTokenType.IntegerLiteral); TestSingleToken("0xDEADBEEFlu", SyntaxKind.IntegerLiteral);
} }
[Fact] [Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix() public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{ {
TestTokenizer("0xDEADBEEFz", new CSharpToken("0xDEADBEEF", CSharpTokenType.IntegerLiteral), IgnoreRemaining); TestTokenizer("0xDEADBEEFz", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "0xDEADBEEF"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal() public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{ {
TestTokenizer("3.a", new CSharpToken("3", CSharpTokenType.IntegerLiteral), IgnoreRemaining); TestTokenizer("3.a", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "3"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Simple_Real_Literal_Is_Recognized() public void Simple_Real_Literal_Is_Recognized()
{ {
TestTokenizer("3.14159", new CSharpToken("3.14159", CSharpTokenType.RealLiteral)); TestTokenizer("3.14159", SyntaxFactory.Token(SyntaxKind.RealLiteral, "3.14159"));
} }
[Fact] [Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized() public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{ {
TestTokenizer(".14159", new CSharpToken(".14159", CSharpTokenType.RealLiteral)); TestTokenizer(".14159", SyntaxFactory.Token(SyntaxKind.RealLiteral, ".14159"));
} }
[Fact] [Fact]
public void Integer_With_Real_Type_Suffix_Is_Recognized() public void Integer_With_Real_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("42F", CSharpTokenType.RealLiteral); TestSingleToken("42F", SyntaxKind.RealLiteral);
TestSingleToken("42f", CSharpTokenType.RealLiteral); TestSingleToken("42f", SyntaxKind.RealLiteral);
TestSingleToken("42D", CSharpTokenType.RealLiteral); TestSingleToken("42D", SyntaxKind.RealLiteral);
TestSingleToken("42d", CSharpTokenType.RealLiteral); TestSingleToken("42d", SyntaxKind.RealLiteral);
TestSingleToken("42M", CSharpTokenType.RealLiteral); TestSingleToken("42M", SyntaxKind.RealLiteral);
TestSingleToken("42m", CSharpTokenType.RealLiteral); TestSingleToken("42m", SyntaxKind.RealLiteral);
} }
[Fact] [Fact]
public void Integer_With_Exponent_Is_Recognized() public void Integer_With_Exponent_Is_Recognized()
{ {
TestSingleToken("1e10", CSharpTokenType.RealLiteral); TestSingleToken("1e10", SyntaxKind.RealLiteral);
TestSingleToken("1E10", CSharpTokenType.RealLiteral); TestSingleToken("1E10", SyntaxKind.RealLiteral);
TestSingleToken("1e+10", CSharpTokenType.RealLiteral); TestSingleToken("1e+10", SyntaxKind.RealLiteral);
TestSingleToken("1E+10", CSharpTokenType.RealLiteral); TestSingleToken("1E+10", SyntaxKind.RealLiteral);
TestSingleToken("1e-10", CSharpTokenType.RealLiteral); TestSingleToken("1e-10", SyntaxKind.RealLiteral);
TestSingleToken("1E-10", CSharpTokenType.RealLiteral); TestSingleToken("1E-10", SyntaxKind.RealLiteral);
} }
[Fact] [Fact]
public void Real_Number_With_Type_Suffix_Is_Recognized() public void Real_Number_With_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("3.14F", CSharpTokenType.RealLiteral); TestSingleToken("3.14F", SyntaxKind.RealLiteral);
TestSingleToken("3.14f", CSharpTokenType.RealLiteral); TestSingleToken("3.14f", SyntaxKind.RealLiteral);
TestSingleToken("3.14D", CSharpTokenType.RealLiteral); TestSingleToken("3.14D", SyntaxKind.RealLiteral);
TestSingleToken("3.14d", CSharpTokenType.RealLiteral); TestSingleToken("3.14d", SyntaxKind.RealLiteral);
TestSingleToken("3.14M", CSharpTokenType.RealLiteral); TestSingleToken("3.14M", SyntaxKind.RealLiteral);
TestSingleToken("3.14m", CSharpTokenType.RealLiteral); TestSingleToken("3.14m", SyntaxKind.RealLiteral);
} }
[Fact] [Fact]
public void Real_Number_With_Exponent_Is_Recognized() public void Real_Number_With_Exponent_Is_Recognized()
{ {
TestSingleToken("3.14E10", CSharpTokenType.RealLiteral); TestSingleToken("3.14E10", SyntaxKind.RealLiteral);
TestSingleToken("3.14e10", CSharpTokenType.RealLiteral); TestSingleToken("3.14e10", SyntaxKind.RealLiteral);
TestSingleToken("3.14E+10", CSharpTokenType.RealLiteral); TestSingleToken("3.14E+10", SyntaxKind.RealLiteral);
TestSingleToken("3.14e+10", CSharpTokenType.RealLiteral); TestSingleToken("3.14e+10", SyntaxKind.RealLiteral);
TestSingleToken("3.14E-10", CSharpTokenType.RealLiteral); TestSingleToken("3.14E-10", SyntaxKind.RealLiteral);
TestSingleToken("3.14e-10", CSharpTokenType.RealLiteral); TestSingleToken("3.14e-10", SyntaxKind.RealLiteral);
} }
[Fact] [Fact]
public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized() public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("3.14E+10F", CSharpTokenType.RealLiteral); TestSingleToken("3.14E+10F", SyntaxKind.RealLiteral);
} }
[Fact] [Fact]
public void Single_Character_Literal_Is_Recognized() public void Single_Character_Literal_Is_Recognized()
{ {
TestSingleToken("'f'", CSharpTokenType.CharacterLiteral); TestSingleToken("'f'", SyntaxKind.CharacterLiteral);
} }
[Fact] [Fact]
public void Multi_Character_Literal_Is_Recognized() public void Multi_Character_Literal_Is_Recognized()
{ {
TestSingleToken("'foo'", CSharpTokenType.CharacterLiteral); TestSingleToken("'foo'", SyntaxKind.CharacterLiteral);
} }
[Fact] [Fact]
public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated() public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated()
{ {
TestSingleToken("'foo bar", CSharpTokenType.CharacterLiteral); TestSingleToken("'foo bar", SyntaxKind.CharacterLiteral);
} }
[Fact] [Fact]
public void Character_Literal_Not_Terminated_By_Escaped_Quote() public void Character_Literal_Not_Terminated_By_Escaped_Quote()
{ {
TestSingleToken("'foo\\'bar'", CSharpTokenType.CharacterLiteral); TestSingleToken("'foo\\'bar'", SyntaxKind.CharacterLiteral);
} }
[Fact] [Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated() public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{ {
TestTokenizer("'foo\n", new CSharpToken("'foo", CSharpTokenType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\n", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("'foo\\\n", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\n", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer("'foo\\\nflarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\nflarg", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("'foo\\" + Environment.NewLine, new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\" + Environment.NewLine, SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining); TestTokenizer($"'foo\\{Environment.NewLine}flarg", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Allows_Escaped_Escape() public void Character_Literal_Allows_Escaped_Escape()
{ {
TestTokenizer("'foo\\\\'blah", new CSharpToken("'foo\\\\'", CSharpTokenType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\\'blah", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\\\'"), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Is_Recognized() public void String_Literal_Is_Recognized()
{ {
TestSingleToken("\"foo\"", CSharpTokenType.StringLiteral); TestSingleToken("\"foo\"", SyntaxKind.StringLiteral);
} }
[Fact] [Fact]
public void String_Literal_Is_Terminated_By_EOF_If_Unterminated() public void String_Literal_Is_Terminated_By_EOF_If_Unterminated()
{ {
TestSingleToken("\"foo bar", CSharpTokenType.StringLiteral); TestSingleToken("\"foo bar", SyntaxKind.StringLiteral);
} }
[Fact] [Fact]
public void String_Literal_Not_Terminated_By_Escaped_Quote() public void String_Literal_Not_Terminated_By_Escaped_Quote()
{ {
TestSingleToken("\"foo\\\"bar\"", CSharpTokenType.StringLiteral); TestSingleToken("\"foo\\\"bar\"", SyntaxKind.StringLiteral);
} }
[Fact] [Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated() public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{ {
TestTokenizer("\"foo\n", new CSharpToken("\"foo", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\n", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo"), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("\"foo\\\n", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\n", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer("\"foo\\\nflarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\nflarg", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\" + Environment.NewLine, SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer($"\"foo\\{Environment.NewLine}flarg", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Allows_Escaped_Escape() public void String_Literal_Allows_Escaped_Escape()
{ {
TestTokenizer("\"foo\\\\\"blah", new CSharpToken("\"foo\\\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\\\"blah", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\\\\""), IgnoreRemaining);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Can_Contain_Newlines() public void Verbatim_String_Literal_Can_Contain_Newlines()
{ {
TestSingleToken("@\"foo\nbar\nbaz\"", CSharpTokenType.StringLiteral); TestSingleToken("@\"foo\nbar\nbaz\"", SyntaxKind.StringLiteral);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote() public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote()
{ {
TestSingleToken("@\"foo\"\"bar\"", CSharpTokenType.StringLiteral); TestSingleToken("@\"foo\"\"bar\"", SyntaxKind.StringLiteral);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote() public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{ {
TestTokenizer("@\"foo\\\"bar\"", new CSharpToken("@\"foo\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining); TestTokenizer("@\"foo\\\"bar\"", SyntaxFactory.Token(SyntaxKind.StringLiteral, "@\"foo\\\""), IgnoreRemaining);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Is_Terminated_By_EOF() public void Verbatim_String_Literal_Is_Terminated_By_EOF()
{ {
TestSingleToken("@\"foo", CSharpTokenType.StringLiteral); TestSingleToken("@\"foo", SyntaxKind.StringLiteral);
} }
} }
} }

99
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -10,287 +11,287 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact] [Fact]
public void LeftBrace_Is_Recognized() public void LeftBrace_Is_Recognized()
{ {
TestSingleToken("{", CSharpTokenType.LeftBrace); TestSingleToken("{", SyntaxKind.LeftBrace);
} }
[Fact] [Fact]
public void Plus_Is_Recognized() public void Plus_Is_Recognized()
{ {
TestSingleToken("+", CSharpTokenType.Plus); TestSingleToken("+", SyntaxKind.Plus);
} }
[Fact] [Fact]
public void Assign_Is_Recognized() public void Assign_Is_Recognized()
{ {
TestSingleToken("=", CSharpTokenType.Assign); TestSingleToken("=", SyntaxKind.Assign);
} }
[Fact] [Fact]
public void Arrow_Is_Recognized() public void Arrow_Is_Recognized()
{ {
TestSingleToken("->", CSharpTokenType.Arrow); TestSingleToken("->", SyntaxKind.Arrow);
} }
[Fact] [Fact]
public void AndAssign_Is_Recognized() public void AndAssign_Is_Recognized()
{ {
TestSingleToken("&=", CSharpTokenType.AndAssign); TestSingleToken("&=", SyntaxKind.AndAssign);
} }
[Fact] [Fact]
public void RightBrace_Is_Recognized() public void RightBrace_Is_Recognized()
{ {
TestSingleToken("}", CSharpTokenType.RightBrace); TestSingleToken("}", SyntaxKind.RightBrace);
} }
[Fact] [Fact]
public void Minus_Is_Recognized() public void Minus_Is_Recognized()
{ {
TestSingleToken("-", CSharpTokenType.Minus); TestSingleToken("-", SyntaxKind.Minus);
} }
[Fact] [Fact]
public void LessThan_Is_Recognized() public void LessThan_Is_Recognized()
{ {
TestSingleToken("<", CSharpTokenType.LessThan); TestSingleToken("<", SyntaxKind.LessThan);
} }
[Fact] [Fact]
public void Equals_Is_Recognized() public void Equals_Is_Recognized()
{ {
TestSingleToken("==", CSharpTokenType.Equals); TestSingleToken("==", SyntaxKind.Equals);
} }
[Fact] [Fact]
public void OrAssign_Is_Recognized() public void OrAssign_Is_Recognized()
{ {
TestSingleToken("|=", CSharpTokenType.OrAssign); TestSingleToken("|=", SyntaxKind.OrAssign);
} }
[Fact] [Fact]
public void LeftBracket_Is_Recognized() public void LeftBracket_Is_Recognized()
{ {
TestSingleToken("[", CSharpTokenType.LeftBracket); TestSingleToken("[", SyntaxKind.LeftBracket);
} }
[Fact] [Fact]
public void Star_Is_Recognized() public void Star_Is_Recognized()
{ {
TestSingleToken("*", CSharpTokenType.Star); TestSingleToken("*", SyntaxKind.Star);
} }
[Fact] [Fact]
public void GreaterThan_Is_Recognized() public void GreaterThan_Is_Recognized()
{ {
TestSingleToken(">", CSharpTokenType.GreaterThan); TestSingleToken(">", SyntaxKind.GreaterThan);
} }
[Fact] [Fact]
public void NotEqual_Is_Recognized() public void NotEqual_Is_Recognized()
{ {
TestSingleToken("!=", CSharpTokenType.NotEqual); TestSingleToken("!=", SyntaxKind.NotEqual);
} }
[Fact] [Fact]
public void XorAssign_Is_Recognized() public void XorAssign_Is_Recognized()
{ {
TestSingleToken("^=", CSharpTokenType.XorAssign); TestSingleToken("^=", SyntaxKind.XorAssign);
} }
[Fact] [Fact]
public void RightBracket_Is_Recognized() public void RightBracket_Is_Recognized()
{ {
TestSingleToken("]", CSharpTokenType.RightBracket); TestSingleToken("]", SyntaxKind.RightBracket);
} }
[Fact] [Fact]
public void Slash_Is_Recognized() public void Slash_Is_Recognized()
{ {
TestSingleToken("/", CSharpTokenType.Slash); TestSingleToken("/", SyntaxKind.Slash);
} }
[Fact] [Fact]
public void QuestionMark_Is_Recognized() public void QuestionMark_Is_Recognized()
{ {
TestSingleToken("?", CSharpTokenType.QuestionMark); TestSingleToken("?", SyntaxKind.QuestionMark);
} }
[Fact] [Fact]
public void LessThanEqual_Is_Recognized() public void LessThanEqual_Is_Recognized()
{ {
TestSingleToken("<=", CSharpTokenType.LessThanEqual); TestSingleToken("<=", SyntaxKind.LessThanEqual);
} }
[Fact] [Fact]
public void LeftShift_Is_Not_Specially_Recognized() public void LeftShift_Is_Not_Specially_Recognized()
{ {
TestTokenizer("<<", TestTokenizer("<<",
new CSharpToken("<", CSharpTokenType.LessThan), SyntaxFactory.Token(SyntaxKind.LessThan, "<"),
new CSharpToken("<", CSharpTokenType.LessThan)); SyntaxFactory.Token(SyntaxKind.LessThan, "<"));
} }
[Fact] [Fact]
public void LeftParen_Is_Recognized() public void LeftParen_Is_Recognized()
{ {
TestSingleToken("(", CSharpTokenType.LeftParenthesis); TestSingleToken("(", SyntaxKind.LeftParenthesis);
} }
[Fact] [Fact]
public void Modulo_Is_Recognized() public void Modulo_Is_Recognized()
{ {
TestSingleToken("%", CSharpTokenType.Modulo); TestSingleToken("%", SyntaxKind.Modulo);
} }
[Fact] [Fact]
public void NullCoalesce_Is_Recognized() public void NullCoalesce_Is_Recognized()
{ {
TestSingleToken("??", CSharpTokenType.NullCoalesce); TestSingleToken("??", SyntaxKind.NullCoalesce);
} }
[Fact] [Fact]
public void GreaterThanEqual_Is_Recognized() public void GreaterThanEqual_Is_Recognized()
{ {
TestSingleToken(">=", CSharpTokenType.GreaterThanEqual); TestSingleToken(">=", SyntaxKind.GreaterThanEqual);
} }
[Fact] [Fact]
public void EqualGreaterThan_Is_Recognized() public void EqualGreaterThan_Is_Recognized()
{ {
TestSingleToken("=>", CSharpTokenType.GreaterThanEqual); TestSingleToken("=>", SyntaxKind.GreaterThanEqual);
} }
[Fact] [Fact]
public void RightParen_Is_Recognized() public void RightParen_Is_Recognized()
{ {
TestSingleToken(")", CSharpTokenType.RightParenthesis); TestSingleToken(")", SyntaxKind.RightParenthesis);
} }
[Fact] [Fact]
public void And_Is_Recognized() public void And_Is_Recognized()
{ {
TestSingleToken("&", CSharpTokenType.And); TestSingleToken("&", SyntaxKind.And);
} }
[Fact] [Fact]
public void DoubleColon_Is_Recognized() public void DoubleColon_Is_Recognized()
{ {
TestSingleToken("::", CSharpTokenType.DoubleColon); TestSingleToken("::", SyntaxKind.DoubleColon);
} }
[Fact] [Fact]
public void PlusAssign_Is_Recognized() public void PlusAssign_Is_Recognized()
{ {
TestSingleToken("+=", CSharpTokenType.PlusAssign); TestSingleToken("+=", SyntaxKind.PlusAssign);
} }
[Fact] [Fact]
public void Semicolon_Is_Recognized() public void Semicolon_Is_Recognized()
{ {
TestSingleToken(";", CSharpTokenType.Semicolon); TestSingleToken(";", SyntaxKind.Semicolon);
} }
[Fact] [Fact]
public void Tilde_Is_Recognized() public void Tilde_Is_Recognized()
{ {
TestSingleToken("~", CSharpTokenType.Tilde); TestSingleToken("~", SyntaxKind.Tilde);
} }
[Fact] [Fact]
public void DoubleOr_Is_Recognized() public void DoubleOr_Is_Recognized()
{ {
TestSingleToken("||", CSharpTokenType.DoubleOr); TestSingleToken("||", SyntaxKind.DoubleOr);
} }
[Fact] [Fact]
public void ModuloAssign_Is_Recognized() public void ModuloAssign_Is_Recognized()
{ {
TestSingleToken("%=", CSharpTokenType.ModuloAssign); TestSingleToken("%=", SyntaxKind.ModuloAssign);
} }
[Fact] [Fact]
public void Colon_Is_Recognized() public void Colon_Is_Recognized()
{ {
TestSingleToken(":", CSharpTokenType.Colon); TestSingleToken(":", SyntaxKind.Colon);
} }
[Fact] [Fact]
public void Not_Is_Recognized() public void Not_Is_Recognized()
{ {
TestSingleToken("!", CSharpTokenType.Not); TestSingleToken("!", SyntaxKind.Not);
} }
[Fact] [Fact]
public void DoubleAnd_Is_Recognized() public void DoubleAnd_Is_Recognized()
{ {
TestSingleToken("&&", CSharpTokenType.DoubleAnd); TestSingleToken("&&", SyntaxKind.DoubleAnd);
} }
[Fact] [Fact]
public void DivideAssign_Is_Recognized() public void DivideAssign_Is_Recognized()
{ {
TestSingleToken("/=", CSharpTokenType.DivideAssign); TestSingleToken("/=", SyntaxKind.DivideAssign);
} }
[Fact] [Fact]
public void Comma_Is_Recognized() public void Comma_Is_Recognized()
{ {
TestSingleToken(",", CSharpTokenType.Comma); TestSingleToken(",", SyntaxKind.Comma);
} }
[Fact] [Fact]
public void Xor_Is_Recognized() public void Xor_Is_Recognized()
{ {
TestSingleToken("^", CSharpTokenType.Xor); TestSingleToken("^", SyntaxKind.Xor);
} }
[Fact] [Fact]
public void Decrement_Is_Recognized() public void Decrement_Is_Recognized()
{ {
TestSingleToken("--", CSharpTokenType.Decrement); TestSingleToken("--", SyntaxKind.Decrement);
} }
[Fact] [Fact]
public void MultiplyAssign_Is_Recognized() public void MultiplyAssign_Is_Recognized()
{ {
TestSingleToken("*=", CSharpTokenType.MultiplyAssign); TestSingleToken("*=", SyntaxKind.MultiplyAssign);
} }
[Fact] [Fact]
public void Dot_Is_Recognized() public void Dot_Is_Recognized()
{ {
TestSingleToken(".", CSharpTokenType.Dot); TestSingleToken(".", SyntaxKind.Dot);
} }
[Fact] [Fact]
public void Or_Is_Recognized() public void Or_Is_Recognized()
{ {
TestSingleToken("|", CSharpTokenType.Or); TestSingleToken("|", SyntaxKind.Or);
} }
[Fact] [Fact]
public void Increment_Is_Recognized() public void Increment_Is_Recognized()
{ {
TestSingleToken("++", CSharpTokenType.Increment); TestSingleToken("++", SyntaxKind.Increment);
} }
[Fact] [Fact]
public void MinusAssign_Is_Recognized() public void MinusAssign_Is_Recognized()
{ {
TestSingleToken("-=", CSharpTokenType.MinusAssign); TestSingleToken("-=", SyntaxKind.MinusAssign);
} }
[Fact] [Fact]
public void RightShift_Is_Not_Specially_Recognized() public void RightShift_Is_Not_Specially_Recognized()
{ {
TestTokenizer(">>", TestTokenizer(">>",
new CSharpToken(">", CSharpTokenType.GreaterThan), SyntaxFactory.Token(SyntaxKind.GreaterThan, ">"),
new CSharpToken(">", CSharpTokenType.GreaterThan)); SyntaxFactory.Token(SyntaxKind.GreaterThan, ">"));
} }
[Fact] [Fact]
public void Hash_Is_Recognized() public void Hash_Is_Recognized()
{ {
TestSingleToken("#", CSharpTokenType.Hash); TestSingleToken("#", SyntaxKind.Hash);
} }
} }
} }

39
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
View File

@ -1,13 +1,14 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public class CSharpTokenizerTest : CSharpTokenizerTestBase public class CSharpTokenizerTest : CSharpTokenizerTestBase
{ {
private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining; private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact] [Fact]
public void Next_Returns_Null_When_EOF_Reached() public void Next_Returns_Null_When_EOF_Reached()
@ -20,8 +21,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\ra", "\r\ra",
new CSharpToken("\r", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
new CSharpToken("\r", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
IgnoreRemaining); IgnoreRemaining);
} }
@ -30,8 +31,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\n\na", "\n\na",
new CSharpToken("\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
new CSharpToken("\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
IgnoreRemaining); IgnoreRemaining);
} }
@ -41,8 +42,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// NEL: Unicode "Next Line" U+0085 // NEL: Unicode "Next Line" U+0085
TestTokenizer( TestTokenizer(
"\u0085\u0085a", "\u0085\u0085a",
new CSharpToken("\u0085", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\u0085"),
new CSharpToken("\u0085", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\u0085"),
IgnoreRemaining); IgnoreRemaining);
} }
@ -52,8 +53,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Line Separator" U+2028 // Unicode "Line Separator" U+2028
TestTokenizer( TestTokenizer(
"\u2028\u2028a", "\u2028\u2028a",
new CSharpToken("\u2028", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\u2028"),
new CSharpToken("\u2028", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\u2028"),
IgnoreRemaining); IgnoreRemaining);
} }
@ -63,8 +64,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Paragraph Separator" U+2029 // Unicode "Paragraph Separator" U+2029
TestTokenizer( TestTokenizer(
"\u2029\u2029a", "\u2029\u2029a",
new CSharpToken("\u2029", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\u2029"),
new CSharpToken("\u2029", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\u2029"),
IgnoreRemaining); IgnoreRemaining);
} }
@ -73,8 +74,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n\r\na", "\r\n\r\na",
new CSharpToken("\r\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
new CSharpToken("\r\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
IgnoreRemaining); IgnoreRemaining);
} }
@ -83,15 +84,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
" \f\t\u000B \n ", " \f\t\u000B \n ",
new CSharpToken(" \f\t\u000B ", CSharpTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " \f\t\u000B "),
new CSharpToken("\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
new CSharpToken(" ", CSharpTokenType.WhiteSpace)); SyntaxFactory.Token(SyntaxKind.Whitespace, " "));
} }
[Fact] [Fact]
public void Transition_Is_Recognized() public void Transition_Is_Recognized()
{ {
TestSingleToken("@", CSharpTokenType.Transition); TestSingleToken("@", SyntaxKind.Transition);
} }
[Fact] [Fact]
@ -99,8 +100,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@(", "@(",
new CSharpToken("@", CSharpTokenType.Transition), SyntaxFactory.Token(SyntaxKind.Transition, "@"),
new CSharpToken("(", CSharpTokenType.LeftParenthesis)); SyntaxFactory.Token(SyntaxKind.LeftParenthesis, "("));
} }
} }
} }

13
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
View File

@ -1,11 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public abstract class CSharpTokenizerTestBase : TokenizerTestBase public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{ {
private static CSharpToken _ignoreRemaining = new CSharpToken(string.Empty, CSharpTokenType.Unknown); private static SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
internal override object IgnoreRemaining internal override object IgnoreRemaining
{ {
@ -17,14 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source); return new CSharpTokenizer(source);
} }
internal void TestSingleToken(string text, CSharpTokenType expectedTokenType) internal void TestSingleToken(string text, SyntaxKind expectedTokenKind)
{ {
TestTokenizer(text, new CSharpToken(text, expectedTokenType)); TestTokenizer(text, SyntaxFactory.Token(expectedTokenKind, text));
}
internal void TestTokenizer(string input, params CSharpToken[] expectedTokens)
{
base.TestTokenizer<CSharpToken, CSharpTokenType>(input, expectedTokens);
} }
} }
} }

37
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -12,12 +13,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n @something \r\n @this is ignored", "\r\n @something \r\n @this is ignored",
new CSharpToken("\r\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
new CSharpToken(" ", CSharpTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new CSharpToken("@", CSharpTokenType.Transition), SyntaxFactory.Token(SyntaxKind.Transition, "@"),
new CSharpToken("something", CSharpTokenType.Identifier), SyntaxFactory.Token(SyntaxKind.Identifier, "something"),
new CSharpToken(" ", CSharpTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new CSharpToken("\r\n", CSharpTokenType.NewLine)); SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
} }
[Fact] [Fact]
@ -25,18 +26,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@*included*@\r\n @something \"value\"\r\n @this is ignored", "@*included*@\r\n @something \"value\"\r\n @this is ignored",
new CSharpToken("@", CSharpTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken("included", CSharpTokenType.RazorComment), SyntaxFactory.Token(SyntaxKind.RazorComment, "included"),
new CSharpToken("*", CSharpTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new CSharpToken("@", CSharpTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new CSharpToken("\r\n", CSharpTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
new CSharpToken(" ", CSharpTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new CSharpToken("@", CSharpTokenType.Transition), SyntaxFactory.Token(SyntaxKind.Transition, "@"),
new CSharpToken("something", CSharpTokenType.Identifier), SyntaxFactory.Token(SyntaxKind.Identifier, "something"),
new CSharpToken(" ", CSharpTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new CSharpToken("\"value\"", CSharpTokenType.StringLiteral), SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"value\""),
new CSharpToken("\r\n", CSharpTokenType.NewLine)); SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
} }
internal override object CreateTokenizer(ITextDocument source) internal override object CreateTokenizer(ITextDocument source)

25
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -12,9 +13,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n <div>Ignored</div>", "\r\n <div>Ignored</div>",
new HtmlToken("\r\n", HtmlTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
new HtmlToken(" ", HtmlTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new HtmlToken("<", HtmlTokenType.OpenAngle)); SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
} }
[Fact] [Fact]
@ -22,15 +23,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n @*included*@ <div>Ignored</div>", "\r\n @*included*@ <div>Ignored</div>",
new HtmlToken("\r\n", HtmlTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
new HtmlToken(" ", HtmlTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken("included", HtmlTokenType.RazorComment), SyntaxFactory.Token(SyntaxKind.RazorComment, "included"),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new HtmlToken(" ", HtmlTokenType.WhiteSpace), SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
new HtmlToken("<", HtmlTokenType.OpenAngle)); SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
} }
internal override object CreateTokenizer(ITextDocument source) internal override object CreateTokenizer(ITextDocument source)

48
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
View File

@ -2,23 +2,24 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Legacy; using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
{ {
public class HtmlMarkupParserTests public class HtmlMarkupParserTests
{ {
private static readonly HtmlToken doubleHyphenToken = new HtmlToken("--", HtmlTokenType.DoubleHyphen); private static readonly SyntaxToken doubleHyphenToken = SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--");
public static IEnumerable<object[]> NonDashTokens public static IEnumerable<object[]> NonDashTokens
{ {
get get
{ {
yield return new[] { new HtmlToken("--", HtmlTokenType.DoubleHyphen) }; yield return new[] { SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--") };
yield return new[] { new HtmlToken("asdf", HtmlTokenType.Text) }; yield return new[] { SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf") };
yield return new[] { new HtmlToken(">", HtmlTokenType.CloseAngle) }; yield return new[] { SyntaxFactory.Token(SyntaxKind.CloseAngle, ">") };
yield return new[] { new HtmlToken("<", HtmlTokenType.OpenAngle) }; yield return new[] { SyntaxFactory.Token(SyntaxKind.OpenAngle, "<") };
yield return new[] { new HtmlToken("!", HtmlTokenType.Bang) }; yield return new[] { SyntaxFactory.Token(SyntaxKind.Bang, "!") };
} }
} }
@ -27,7 +28,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsHyphen_ReturnsFalseForNonDashToken(object token) public void IsHyphen_ReturnsFalseForNonDashToken(object token)
{ {
// Arrange // Arrange
var convertedToken = (HtmlToken)token; var convertedToken = (SyntaxToken)token;
// Act & Assert // Act & Assert
Assert.False(HtmlMarkupParser.IsHyphen(convertedToken)); Assert.False(HtmlMarkupParser.IsHyphen(convertedToken));
@ -37,7 +38,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsHyphen_ReturnsTrueForADashToken() public void IsHyphen_ReturnsTrueForADashToken()
{ {
// Arrange // Arrange
var dashToken = new HtmlToken("-", HtmlTokenType.Text); var dashToken = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "-");
// Act & Assert // Act & Assert
Assert.True(HtmlMarkupParser.IsHyphen(dashToken)); Assert.True(HtmlMarkupParser.IsHyphen(dashToken));
@ -53,9 +54,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
var token = sut.AcceptAllButLastDoubleHyphens(); var token = sut.AcceptAllButLastDoubleHyphens();
// Assert // Assert
Assert.Equal(doubleHyphenToken, token); Assert.True(doubleHyphenToken.IsEquivalentTo(token));
Assert.True(sut.At(HtmlTokenType.CloseAngle)); Assert.True(sut.At(SyntaxKind.CloseAngle));
Assert.Equal(doubleHyphenToken, sut.PreviousToken); Assert.True(doubleHyphenToken.IsEquivalentTo(sut.PreviousToken));
} }
[Fact] [Fact]
@ -68,8 +69,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
var token = sut.AcceptAllButLastDoubleHyphens(); var token = sut.AcceptAllButLastDoubleHyphens();
// Assert // Assert
Assert.Equal(doubleHyphenToken, token); Assert.True(doubleHyphenToken.IsEquivalentTo(token));
Assert.True(sut.At(HtmlTokenType.CloseAngle)); Assert.True(sut.At(SyntaxKind.CloseAngle));
Assert.True(HtmlMarkupParser.IsHyphen(sut.PreviousToken)); Assert.True(HtmlMarkupParser.IsHyphen(sut.PreviousToken));
} }
@ -157,8 +158,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsFalseForAllowedContent() public void IsCommentContentEndingInvalid_ReturnsFalseForAllowedContent()
{ {
// Arrange // Arrange
var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text); var expectedToken1 = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "a");
var sequence = Enumerable.Range((int)'a', 26).Select(item => new HtmlToken(((char)item).ToString(), HtmlTokenType.Text)); var sequence = Enumerable.Range((int)'a', 26).Select(item => SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, ((char)item).ToString()));
// Act & Assert // Act & Assert
Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence)); Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -168,8 +169,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsTrueForDisallowedContent() public void IsCommentContentEndingInvalid_ReturnsTrueForDisallowedContent()
{ {
// Arrange // Arrange
var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text); var expectedToken1 = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "a");
var sequence = new[] { new HtmlToken("<", HtmlTokenType.OpenAngle), new HtmlToken("!", HtmlTokenType.Bang), new HtmlToken("-", HtmlTokenType.Text) }; var sequence = new[]
{
SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"),
SyntaxFactory.Token(SyntaxKind.Bang, "!"),
SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "-")
};
// Act & Assert // Act & Assert
Assert.True(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence)); Assert.True(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -179,8 +185,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsFalseForEmptyContent() public void IsCommentContentEndingInvalid_ReturnsFalseForEmptyContent()
{ {
// Arrange // Arrange
var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text); var expectedToken1 = SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "a");
var sequence = Array.Empty<HtmlToken>(); var sequence = Array.Empty<SyntaxToken>();
// Act & Assert // Act & Assert
Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence)); Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -188,7 +194,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
private class TestHtmlMarkupParser : HtmlMarkupParser private class TestHtmlMarkupParser : HtmlMarkupParser
{ {
public new HtmlToken PreviousToken public new SyntaxToken PreviousToken
{ {
get => base.PreviousToken; get => base.PreviousToken;
} }
@ -203,7 +209,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
this.EnsureCurrent(); this.EnsureCurrent();
} }
public new HtmlToken AcceptAllButLastDoubleHyphens() public new SyntaxToken AcceptAllButLastDoubleHyphens()
{ {
return base.AcceptAllButLastDoubleHyphens(); return base.AcceptAllButLastDoubleHyphens();
} }

67
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlTokenizerTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -17,113 +18,113 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void Text_Is_Recognized() public void Text_Is_Recognized()
{ {
TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384", TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384",
new HtmlToken("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlTokenType.Text)); SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "foo-9309&smlkmb;::-3029022,.sdkq92384"));
} }
[Fact] [Fact]
public void Whitespace_Is_Recognized() public void Whitespace_Is_Recognized()
{ {
TestTokenizer(" \t\f ", TestTokenizer(" \t\f ",
new HtmlToken(" \t\f ", HtmlTokenType.WhiteSpace)); SyntaxFactory.Token(SyntaxKind.Whitespace, " \t\f "));
} }
[Fact] [Fact]
public void Newline_Is_Recognized() public void Newline_Is_Recognized()
{ {
TestTokenizer("\n\r\r\n", TestTokenizer("\n\r\r\n",
new HtmlToken("\n", HtmlTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
new HtmlToken("\r", HtmlTokenType.NewLine), SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
new HtmlToken("\r\n", HtmlTokenType.NewLine)); SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
} }
[Fact] [Fact]
public void Transition_Is_Not_Recognized_Mid_Text_If_Surrounded_By_Alphanumeric_Characters() public void Transition_Is_Not_Recognized_Mid_Text_If_Surrounded_By_Alphanumeric_Characters()
{ {
TestSingleToken("foo@bar", HtmlTokenType.Text); TestSingleToken("foo@bar", SyntaxKind.HtmlTextLiteral);
} }
[Fact] [Fact]
public void OpenAngle_Is_Recognized() public void OpenAngle_Is_Recognized()
{ {
TestSingleToken("<", HtmlTokenType.OpenAngle); TestSingleToken("<", SyntaxKind.OpenAngle);
} }
[Fact] [Fact]
public void Bang_Is_Recognized() public void Bang_Is_Recognized()
{ {
TestSingleToken("!", HtmlTokenType.Bang); TestSingleToken("!", SyntaxKind.Bang);
} }
[Fact] [Fact]
public void Solidus_Is_Recognized() public void Solidus_Is_Recognized()
{ {
TestSingleToken("/", HtmlTokenType.ForwardSlash); TestSingleToken("/", SyntaxKind.ForwardSlash);
} }
[Fact] [Fact]
public void QuestionMark_Is_Recognized() public void QuestionMark_Is_Recognized()
{ {
TestSingleToken("?", HtmlTokenType.QuestionMark); TestSingleToken("?", SyntaxKind.QuestionMark);
} }
[Fact] [Fact]
public void LeftBracket_Is_Recognized() public void LeftBracket_Is_Recognized()
{ {
TestSingleToken("[", HtmlTokenType.LeftBracket); TestSingleToken("[", SyntaxKind.LeftBracket);
} }
[Fact] [Fact]
public void CloseAngle_Is_Recognized() public void CloseAngle_Is_Recognized()
{ {
TestSingleToken(">", HtmlTokenType.CloseAngle); TestSingleToken(">", SyntaxKind.CloseAngle);
} }
[Fact] [Fact]
public void RightBracket_Is_Recognized() public void RightBracket_Is_Recognized()
{ {
TestSingleToken("]", HtmlTokenType.RightBracket); TestSingleToken("]", SyntaxKind.RightBracket);
} }
[Fact] [Fact]
public void Equals_Is_Recognized() public void Equals_Is_Recognized()
{ {
TestSingleToken("=", HtmlTokenType.Equals); TestSingleToken("=", SyntaxKind.Equals);
} }
[Fact] [Fact]
public void DoubleQuote_Is_Recognized() public void DoubleQuote_Is_Recognized()
{ {
TestSingleToken("\"", HtmlTokenType.DoubleQuote); TestSingleToken("\"", SyntaxKind.DoubleQuote);
} }
[Fact] [Fact]
public void SingleQuote_Is_Recognized() public void SingleQuote_Is_Recognized()
{ {
TestSingleToken("'", HtmlTokenType.SingleQuote); TestSingleToken("'", SyntaxKind.SingleQuote);
} }
[Fact] [Fact]
public void Transition_Is_Recognized() public void Transition_Is_Recognized()
{ {
TestSingleToken("@", HtmlTokenType.Transition); TestSingleToken("@", SyntaxKind.Transition);
} }
[Fact] [Fact]
public void DoubleHyphen_Is_Recognized() public void DoubleHyphen_Is_Recognized()
{ {
TestSingleToken("--", HtmlTokenType.DoubleHyphen); TestSingleToken("--", SyntaxKind.DoubleHyphen);
} }
[Fact] [Fact]
public void SingleHyphen_Is_Not_Recognized() public void SingleHyphen_Is_Not_Recognized()
{ {
TestSingleToken("-", HtmlTokenType.Text); TestSingleToken("-", SyntaxKind.HtmlTextLiteral);
} }
[Fact] [Fact]
public void SingleHyphen_Mid_Text_Is_Not_Recognized_As_Separate_Token() public void SingleHyphen_Mid_Text_Is_Not_Recognized_As_Separate_Token()
{ {
TestSingleToken("foo-bar", HtmlTokenType.Text); TestSingleToken("foo-bar", SyntaxKind.HtmlTextLiteral);
} }
[Fact] [Fact]
@ -131,9 +132,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *", "@* Foo * Bar * Baz *",
new HtmlToken("@", HtmlTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken(" Foo * Bar * Baz *", HtmlTokenType.RazorComment)); SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz *"));
} }
[Fact] [Fact]
@ -141,11 +142,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *@", "@* Foo * Bar * Baz *@",
new HtmlToken("@", HtmlTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken(" Foo * Bar * Baz ", HtmlTokenType.RazorComment), SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz "),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition)); SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
} }
[Fact] [Fact]
@ -153,11 +154,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo Bar Baz *@", "@* Foo Bar Baz *@",
new HtmlToken("@", HtmlTokenType.RazorCommentTransition), SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken(" Foo Bar Baz ", HtmlTokenType.RazorComment), SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo Bar Baz "),
new HtmlToken("*", HtmlTokenType.RazorCommentStar), SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
new HtmlToken("@", HtmlTokenType.RazorCommentTransition)); SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
} }
} }
} }

13
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlTokenizerTestBase.cs
View File

@ -1,11 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public abstract class HtmlTokenizerTestBase : TokenizerTestBase public abstract class HtmlTokenizerTestBase : TokenizerTestBase
{ {
private static HtmlToken _ignoreRemaining = new HtmlToken(string.Empty, HtmlTokenType.Unknown); private static SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
internal override object IgnoreRemaining internal override object IgnoreRemaining
{ {
@ -17,14 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source); return new HtmlTokenizer(source);
} }
internal void TestSingleToken(string text, HtmlTokenType expectedTokenType) internal void TestSingleToken(string text, SyntaxKind expectedTokenKind)
{ {
TestTokenizer(text, new HtmlToken(text, expectedTokenType)); TestTokenizer(text, SyntaxFactory.Token(expectedTokenKind, text));
}
internal void TestTokenizer(string input, params HtmlToken[] expectedTokens)
{
base.TestTokenizer<HtmlToken, HtmlTokenType>(input, expectedTokens);
} }
} }
} }

24
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ImplicitExpressionEditHandlerTest.cs
View File

@ -3,6 +3,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Syntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -159,7 +160,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_SingleLeftParenthesis_CountsCorrectly() public void TryUpdateBalanceCount_SingleLeftParenthesis_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpToken("(", CSharpTokenType.LeftParenthesis); var token = SyntaxFactory.Token(SyntaxKind.LeftParenthesis, "(");
var count = 0; var count = 0;
// Act // Act
@ -174,7 +175,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_SingleRightParenthesis_CountsCorrectly() public void TryUpdateBalanceCount_SingleRightParenthesis_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpToken(")", CSharpTokenType.RightParenthesis); var token = SyntaxFactory.Token(SyntaxKind.RightParenthesis, ")");
var count = 2; var count = 2;
// Act // Act
@ -189,7 +190,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_IncompleteStringLiteral_CountsCorrectly() public void TryUpdateBalanceCount_IncompleteStringLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpToken("\"((", CSharpTokenType.StringLiteral); var token = SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"((");
var count = 2; var count = 2;
// Act // Act
@ -204,7 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_IncompleteCharacterLiteral_CountsCorrectly() public void TryUpdateBalanceCount_IncompleteCharacterLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpToken("'((", CSharpTokenType.CharacterLiteral); var token = SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'((");
var count = 2; var count = 2;
// Act // Act
@ -219,7 +220,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_CompleteStringLiteral_CountsCorrectly() public void TryUpdateBalanceCount_CompleteStringLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpToken("\"((\"", CSharpTokenType.StringLiteral); var token = SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"((\"");
var count = 2; var count = 2;
// Act // Act
@ -234,7 +235,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_CompleteCharacterLiteral_CountsCorrectly() public void TryUpdateBalanceCount_CompleteCharacterLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpToken("'('", CSharpTokenType.CharacterLiteral); var token = SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'('");
var count = 2; var count = 2;
// Act // Act
@ -249,7 +250,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesis_ReturnsFalse() public void TryUpdateBalanceCount_InvalidParenthesis_ReturnsFalse()
{ {
// Arrange // Arrange
var token = new CSharpToken(")", CSharpTokenType.RightParenthesis); var token = SyntaxFactory.Token(SyntaxKind.RightParenthesis, ")");
var count = 0; var count = 0;
// Act // Act
@ -264,7 +265,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesisStringLiteral_ReturnsFalse() public void TryUpdateBalanceCount_InvalidParenthesisStringLiteral_ReturnsFalse()
{ {
// Arrange // Arrange
var token = new CSharpToken("\")", CSharpTokenType.StringLiteral); var token = SyntaxFactory.Token(SyntaxKind.StringLiteral, "\")");
var count = 0; var count = 0;
// Act // Act
@ -279,7 +280,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesisCharacterLiteral_ReturnsFalse() public void TryUpdateBalanceCount_InvalidParenthesisCharacterLiteral_ReturnsFalse()
{ {
// Arrange // Arrange
var token = new CSharpToken("')", CSharpTokenType.CharacterLiteral); var token = SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "')");
var count = 0; var count = 0;
// Act // Act
@ -453,11 +454,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return span; return span;
} }
private static IReadOnlyList<CSharpToken> GetTokens(SourceLocation start, string content) private static IReadOnlyList<SyntaxToken> GetTokens(SourceLocation start, string content)
{ {
var parent = GetSpan(start, content); var parent = GetSpan(start, content);
var tokens = parent.Tokens.Cast<CSharpToken>().ToArray(); return parent.Tokens;
return tokens;
} }
} }
} }

11
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/SpanTest.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -12,10 +13,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Arrange // Arrange
var builder = new SpanBuilder(SourceLocation.Zero); var builder = new SpanBuilder(SourceLocation.Zero);
builder.Accept(new HtmlToken("hello", HtmlTokenType.Text)); builder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = builder.Build(); var span = builder.Build();
var newBuilder = new SpanBuilder(SourceLocation.Zero); var newBuilder = new SpanBuilder(SourceLocation.Zero);
newBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text)); newBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
var originalLength = span.Length; var originalLength = span.Length;
// Act // Act
@ -33,7 +34,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Arrange // Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero); var spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text)); spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = spanBuilder.Build(); var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder() var blockBuilder = new BlockBuilder()
{ {
@ -44,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.Parent = block; span.Parent = block;
var originalBlockLength = block.Length; var originalBlockLength = block.Length;
var newSpanBuilder = new SpanBuilder(SourceLocation.Zero); var newSpanBuilder = new SpanBuilder(SourceLocation.Zero);
newSpanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text)); newSpanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
// Act // Act
span.ReplaceWith(newSpanBuilder); span.ReplaceWith(newSpanBuilder);
@ -64,7 +65,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind = SpanKindInternal.Transition, Kind = SpanKindInternal.Transition,
ChunkGenerator = new ExpressionChunkGenerator(), ChunkGenerator = new ExpressionChunkGenerator(),
}; };
spanBuilder.Accept(new CSharpToken("@", CSharpTokenType.Transition)); spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.Transition, "@"));
var span = spanBuilder.Build(); var span = spanBuilder.Build();
// Act // Act

50
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/TokenizerLookaheadTest.cs
View File

@ -5,6 +5,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -64,7 +65,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Act // Act
var i = 3; var i = 3;
IEnumerable<HtmlToken> previousTokens = null; IEnumerable<SyntaxToken> previousTokens = null;
var tokenFound = tokenizer.LookaheadUntil((s, p) => var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{ {
previousTokens = p; previousTokens = p;
@ -77,9 +78,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// For the very first element, there will be no previous items, so null is expected // For the very first element, there will be no previous items, so null is expected
var orderIndex = 0; var orderIndex = 0;
Assert.Null(previousTokens.ElementAt(orderIndex++)); Assert.Null(previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), previousTokens.ElementAt(orderIndex++)); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf"), previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), previousTokens.ElementAt(orderIndex++)); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--"), previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlToken("fvd", HtmlTokenType.Text), previousTokens.ElementAt(orderIndex++)); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "fvd"), previousTokens.ElementAt(orderIndex++));
} }
[Fact] [Fact]
@ -89,7 +90,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokenizer = CreateContentTokenizer("asdf--fvd"); var tokenizer = CreateContentTokenizer("asdf--fvd");
// Act // Act
var tokens = new Stack<HtmlToken>(); var tokens = new Stack<SyntaxToken>();
var tokenFound = tokenizer.LookaheadUntil((s, p) => var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{ {
tokens.Push(s); tokens.Push(s);
@ -99,9 +100,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Assert // Assert
Assert.False(tokenFound); Assert.False(tokenFound);
Assert.Equal(3, tokens.Count); Assert.Equal(3, tokens.Count);
Assert.Equal(new HtmlToken("fvd", HtmlTokenType.Text), tokens.Pop()); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "fvd"), tokens.Pop());
Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), tokens.Pop()); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--"), tokens.Pop());
Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), tokens.Pop()); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf"), tokens.Pop());
} }
[Fact] [Fact]
@ -111,18 +112,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokenizer = CreateContentTokenizer("asdf--fvd"); var tokenizer = CreateContentTokenizer("asdf--fvd");
// Act // Act
var tokens = new Stack<HtmlToken>(); var tokens = new Stack<SyntaxToken>();
var tokenFound = tokenizer.LookaheadUntil((s, p) => var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{ {
tokens.Push(s); tokens.Push(s);
return s.Type == HtmlTokenType.DoubleHyphen; return s.Kind == SyntaxKind.DoubleHyphen;
}); });
// Assert // Assert
Assert.True(tokenFound); Assert.True(tokenFound);
Assert.Equal(2, tokens.Count); Assert.Equal(2, tokens.Count);
Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), tokens.Pop()); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--"), tokens.Pop());
Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), tokens.Pop()); AssertTokenEqual(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "asdf"), tokens.Pop());
} }
private static TestTokenizerBackedParser CreateContentTokenizer(string content) private static TestTokenizerBackedParser CreateContentTokenizer(string content)
@ -135,7 +136,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return tokenizer; return tokenizer;
} }
private class ExposedTokenizer : Tokenizer<CSharpToken, CSharpTokenType> private static void AssertTokenEqual(SyntaxToken expected, SyntaxToken actual)
{
Assert.True(expected.IsEquivalentTo(actual), "Tokens not equal.");
}
private class ExposedTokenizer : Tokenizer
{ {
public ExposedTokenizer(string input) public ExposedTokenizer(string input)
: base(new SeekableTextReader(input, filePath: null)) : base(new SeekableTextReader(input, filePath: null))
@ -150,7 +156,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public override CSharpTokenType RazorCommentStarType public override SyntaxKind RazorCommentStarKind
{ {
get get
{ {
@ -158,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public override CSharpTokenType RazorCommentTransitionType public override SyntaxKind RazorCommentTransitionKind
{ {
get get
{ {
@ -166,7 +172,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public override CSharpTokenType RazorCommentType public override SyntaxKind RazorCommentKind
{ {
get get
{ {
@ -182,9 +188,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
protected override CSharpToken CreateToken( protected override SyntaxToken CreateToken(
string content, string content,
CSharpTokenType type, SyntaxKind type,
IReadOnlyList<RazorDiagnostic> errors) IReadOnlyList<RazorDiagnostic> errors)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
@ -196,9 +202,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
private class TestTokenizerBackedParser : TokenizerBackedParser<HtmlTokenizer, HtmlToken, HtmlTokenType> private class TestTokenizerBackedParser : TokenizerBackedParser<HtmlTokenizer>
{ {
internal TestTokenizerBackedParser(LanguageCharacteristics<HtmlTokenizer, HtmlToken, HtmlTokenType> language, ParserContext context) : base(language, context) internal TestTokenizerBackedParser(LanguageCharacteristics<HtmlTokenizer> language, ParserContext context) : base(language, context)
{ {
} }
@ -207,12 +213,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
throw new NotImplementedException(); throw new NotImplementedException();
} }
protected override bool TokenTypeEquals(HtmlTokenType x, HtmlTokenType y) protected override bool TokenKindEquals(SyntaxKind x, SyntaxKind y)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
internal new bool LookaheadUntil(Func<HtmlToken, IEnumerable<HtmlToken>, bool> condition) internal new bool LookaheadUntil(Func<SyntaxToken, IEnumerable<SyntaxToken>, bool> condition)
{ {
return base.LookaheadUntil(condition); return base.LookaheadUntil(condition);
} }

13
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/TokenizerTestBase.cs
View File

@ -4,6 +4,7 @@
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using System.Text; using System.Text;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
@ -13,18 +14,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal abstract object IgnoreRemaining { get; } internal abstract object IgnoreRemaining { get; }
internal abstract object CreateTokenizer(ITextDocument source); internal abstract object CreateTokenizer(ITextDocument source);
internal void TestTokenizer<TSymbol, TSymbolType>(string input, params TSymbol[] expectedSymbols) internal void TestTokenizer(string input, params SyntaxToken[] expectedSymbols)
where TSymbolType : struct
where TSymbol : TokenBase<TSymbolType>
{ {
// Arrange // Arrange
var success = true; var success = true;
var output = new StringBuilder(); var output = new StringBuilder();
using (var source = new SeekableTextReader(input, filePath: null)) using (var source = new SeekableTextReader(input, filePath: null))
{ {
var tokenizer = (Tokenizer<TSymbol, TSymbolType>)CreateTokenizer(source); var tokenizer = (Tokenizer)CreateTokenizer(source);
var counter = 0; var counter = 0;
TSymbol current = null; SyntaxToken current = null;
while ((current = tokenizer.NextToken()) != null) while ((current = tokenizer.NextToken()) != null)
{ {
if (counter >= expectedSymbols.Length) if (counter >= expectedSymbols.Length)
@ -34,11 +33,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining)) else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{ {
output.AppendLine(string.Format("P: Ignored {0}", current)); output.AppendLine(string.Format("P: Ignored |{0}|", current));
} }
else else
{ {
if (!Equals(expectedSymbols[counter], current)) if (!expectedSymbols[counter].IsEquivalentTo(current))
{ {
output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current)); output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
success = false; success = false;

21
test/Microsoft.AspNetCore.Razor.Language.Test/SourceChangeTest.cs
View File

@ -3,6 +3,7 @@
using System; using System;
using Microsoft.AspNetCore.Razor.Language.Legacy; using Microsoft.AspNetCore.Razor.Language.Legacy;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit; using Xunit;
namespace Microsoft.AspNetCore.Razor.Language namespace Microsoft.AspNetCore.Razor.Language
@ -104,8 +105,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(0, 0, 0)); var builder = new SpanBuilder(new SourceLocation(0, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(0, 0, 0), "Hello, ")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(7, 0, 7), "World")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder); var span = new Span(builder);
@ -123,8 +124,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder); var span = new Span(builder);
@ -142,8 +143,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder); var span = new Span(builder);
@ -161,8 +162,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder); var span = new Span(builder);
@ -180,8 +181,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "Hello, "));
builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World")); builder.Accept(SyntaxFactory.Token(SyntaxKind.Unknown, "World"));
var span = new Span(builder); var span = new Span(builder);

8
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/FunctionsDirectiveAutoCompleteAtEOF.stree.txt
View File

@ -1,9 +1,9 @@
Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 11 - (0:0,0) Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 11 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1 MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[functions]; SyntaxKind.Identifier;[functions];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:1 Code span - Gen<Stmt> - [] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:1
CSharpTokenType.Unknown;[]; SyntaxKind.Unknown;[];

10
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/FunctionsDirectiveAutoCompleteAtStartOfFile.stree.txt
View File

@ -1,10 +1,10 @@
Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 16 - (0:0,0) Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 16 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1 MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[functions]; SyntaxKind.Identifier;[functions];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [LFfoo] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:2 Code span - Gen<Stmt> - [LFfoo] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:2
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];

14
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/SectionDirectiveAutoCompleteAtEOF.stree.txt
View File

@ -1,16 +1,16 @@
Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 17 - (0:0,0) Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 17 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1 MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[section]; SyntaxKind.Identifier;[section];
Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1 Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1 Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1
CSharpTokenType.Identifier;[Header]; SyntaxKind.Identifier;[Header];
Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1 Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Markup block - Gen<None> - 0 - (17:0,17) Markup block - Gen<None> - 0 - (17:0,17)
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1 Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
HtmlTokenType.Unknown;[]; SyntaxKind.Unknown;[];

30
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/SectionDirectiveAutoCompleteAtStartOfFile.stree.txt
View File

@ -1,29 +1,29 @@
Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 29 - (0:0,0) Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 29 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1 MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.Identifier;[section]; SyntaxKind.Identifier;[section];
Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1 Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1 Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1
CSharpTokenType.Identifier;[Header]; SyntaxKind.Identifier;[Header];
Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1 Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Markup block - Gen<None> - 12 - (17:0,17) Markup block - Gen<None> - 12 - (17:0,17)
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1 Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Tag block - Gen<None> - 3 - (19:1,0) Tag block - Gen<None> - 3 - (19:1,0)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:Any - (19:1,0) - Tokens:3 Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:Any - (19:1,0) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[p]; SyntaxKind.HtmlTextLiteral;[p];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
SyntaxKind.HtmlText - [Foo] - [22..25) - FullWidth: 3 - Slots: 1 SyntaxKind.HtmlText - [Foo] - [22..25) - FullWidth: 3 - Slots: 1
SyntaxKind.HtmlTextLiteralToken;[Foo]; SyntaxKind.HtmlTextLiteral;[Foo];
Tag block - Gen<None> - 4 - (25:1,6) Tag block - Gen<None> - 4 - (25:1,6)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:Any - (25:1,6) - Tokens:4 Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:Any - (25:1,6) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[p]; SyntaxKind.HtmlTextLiteral;[p];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];

6
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/VerbatimBlockAutoCompleteAtEOF.stree.txt
View File

@ -1,7 +1,7 @@
Statement block - Gen<None> - 2 - (0:0,0) Statement block - Gen<None> - 2 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1 MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1 Code span - Gen<Stmt> - [] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1
CSharpTokenType.Unknown;[]; SyntaxKind.Unknown;[];

22
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/VerbatimBlockAutoCompleteAtStartOfFile.stree.txt
View File

@ -1,21 +1,21 @@
Statement block - Gen<None> - 11 - (0:0,0) Statement block - Gen<None> - 11 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1 MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [LF] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1 Code span - Gen<Stmt> - [LF] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Markup block - Gen<None> - 7 - (4:1,0) Markup block - Gen<None> - 7 - (4:1,0)
Tag block - Gen<None> - 3 - (4:1,0) Tag block - Gen<None> - 3 - (4:1,0)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (4:1,0) - Tokens:3 Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (4:1,0) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[p]; SyntaxKind.HtmlTextLiteral;[p];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Tag block - Gen<None> - 4 - (7:1,3) Tag block - Gen<None> - 4 - (7:1,3)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (7:1,3) - Tokens:4 Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (7:1,3) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[p]; SyntaxKind.HtmlTextLiteral;[p];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Code span - Gen<Stmt> - [] - SpanEditHandler;Accepts:Any - (11:1,7) - Tokens:1 Code span - Gen<Stmt> - [] - SpanEditHandler;Accepts:Any - (11:1,7) - Tokens:1
CSharpTokenType.Unknown;[]; SyntaxKind.Unknown;[];

116
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/AcceptsElseIfWithNoCondition.stree.txt
View File

@ -1,60 +1,60 @@
Statement block - Gen<None> - 106 - (0:0,0) Statement block - Gen<None> - 106 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if { foo(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:58 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if { foo(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:58
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"foo } bar"]; SyntaxKind.StringLiteral;[@"foo } bar"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[else]; SyntaxKind.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

24
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/AcceptsTrailingDotIntoImplicitExpressionWhenEmbeddedInCode.stree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 17 - (0:0,0) Statement block - Gen<None> - 17 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7 Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Expression block - Gen<Expr> - 5 - (10:0,10) Expression block - Gen<Expr> - 5 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [foo.] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:2 Code span - Gen<Expr> - [foo.] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:2
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (15:0,15) - Tokens:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (15:0,15) - Tokens:2
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

16
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/AllowsEmptyBlockStatement.stree.txt
View File

@ -1,10 +1,10 @@
Statement block - Gen<None> - 13 - (0:0,0) Statement block - Gen<None> - 13 - (0:0,0)
Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8 Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[false]; SyntaxKind.Keyword;[false];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/BalancingBracketsIgnoresStringLiteralCharactersAndBrackets.stree.txt
View File

@ -1,19 +1,19 @@
Statement block - Gen<None> - 47 - (0:0,0) Statement block - Gen<None> - 47 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) {LF // bar } " baz 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:17 Code span - Gen<Stmt> - [if(foo) {LF // bar } " baz 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:17
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Comment;[// bar } " baz ']; SyntaxKind.CSharpComment;[// bar } " baz '];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[zoop]; SyntaxKind.Identifier;[zoop];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

38
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/BalancingBracketsIgnoresStringLiteralCharactersAndBracketsInsideBlockComments.stree.txt
View File

@ -1,21 +1,21 @@
Statement block - Gen<None> - 54 - (0:0,0) Statement block - Gen<None> - 54 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) {LF /* bar } " */ ' baz } 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:19 Code span - Gen<Stmt> - [if(foo) {LF /* bar } " */ ' baz } 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:19
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Comment;[/* bar } " */]; SyntaxKind.CSharpComment;[/* bar } " */];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.CharacterLiteral;[' baz } ']; SyntaxKind.CharacterLiteral;[' baz } '];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[zoop]; SyntaxKind.Identifier;[zoop];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

8
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CapturesNewlineAfterUsing.stree.txt
View File

@ -1,6 +1,6 @@
Directive block - Gen<None> - 11 - (0:0,0) Directive block - Gen<None> - 11 - (0:0,0)
Code span - Gen<Import: Foo;> - [using FooLF] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:4 Code span - Gen<Import: Foo;> - [using FooLF] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:4
CSharpTokenType.Keyword;[using]; SyntaxKind.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];

48
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlock.stree.txt
View File

@ -1,26 +1,26 @@
Statement block - Gen<None> - 40 - (0:0,0) Statement block - Gen<None> - 40 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar);] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:24 Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar);] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:24
CSharpTokenType.Keyword;[do]; SyntaxKind.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[while]; SyntaxKind.Keyword;[while];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.NotEqual;[!=]; SyntaxKind.NotEqual;[!=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];

46
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingSemicolon.stree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 39 - (0:0,0) Statement block - Gen<None> - 39 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar)] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:23 Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar)] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:23
CSharpTokenType.Keyword;[do]; SyntaxKind.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[while]; SyntaxKind.Keyword;[while];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.NotEqual;[!=]; SyntaxKind.NotEqual;[!=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];

28
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingWhileClauseEntirely.stree.txt
View File

@ -1,16 +1,16 @@
Statement block - Gen<None> - 21 - (0:0,0) Statement block - Gen<None> - 21 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:14 Code span - Gen<Stmt> - [do { var foo = bar; }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:14
CSharpTokenType.Keyword;[do]; SyntaxKind.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

32
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingWhileCondition.stree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 27 - (0:0,0) Statement block - Gen<None> - 27 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:16 Code span - Gen<Stmt> - [do { var foo = bar; } while] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:16
CSharpTokenType.Keyword;[do]; SyntaxKind.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[while]; SyntaxKind.Keyword;[while];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesDoWhileBlockMissingWhileConditionWithSemicolon.stree.txt
View File

@ -1,19 +1,19 @@
Statement block - Gen<None> - 28 - (0:0,0) Statement block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while;] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:17 Code span - Gen<Stmt> - [do { var foo = bar; } while;] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:17
CSharpTokenType.Keyword;[do]; SyntaxKind.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[while]; SyntaxKind.Keyword;[while];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];

76
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/CorrectlyParsesMarkupInDoWhileBlock.stree.txt
View File

@ -1,50 +1,50 @@
Statement block - Gen<None> - 58 - (0:0,0) Statement block - Gen<None> - 58 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [do { var foo = bar;] - SpanEditHandler;Accepts:Any - (1:0,1) - Tokens:12 Code span - Gen<Stmt> - [do { var foo = bar;] - SpanEditHandler;Accepts:Any - (1:0,1) - Tokens:12
CSharpTokenType.Keyword;[do]; SyntaxKind.Keyword;[do];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
Markup block - Gen<None> - 12 - (20:0,20) Markup block - Gen<None> - 12 - (20:0,20)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (20:0,20) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (20:0,20) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 3 - (21:0,21) Tag block - Gen<None> - 3 - (21:0,21)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:3 Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[p]; SyntaxKind.HtmlTextLiteral;[p];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1 Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1
HtmlTokenType.Text;[Foo]; SyntaxKind.HtmlTextLiteral;[Foo];
Tag block - Gen<None> - 4 - (27:0,27) Tag block - Gen<None> - 4 - (27:0,27)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (27:0,27) - Tokens:4 Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (27:0,27) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[p]; SyntaxKind.HtmlTextLiteral;[p];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (31:0,31) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (31:0,31) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Code span - Gen<Stmt> - [foo++; } while (foo<bar>);] - SpanEditHandler;Accepts:None - (32:0,32) - Tokens:15 Code span - Gen<Stmt> - [foo++; } while (foo<bar>);] - SpanEditHandler;Accepts:None - (32:0,32) - Tokens:15
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.Increment;[++]; SyntaxKind.Increment;[++];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[while]; SyntaxKind.Keyword;[while];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.GreaterThan;[>]; SyntaxKind.GreaterThan;[>];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];

70
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/DoesNotAllowMultipleFinallyBlocks.stree.txt
View File

@ -1,37 +1,37 @@
Statement block - Gen<None> - 55 - (0:0,0) Statement block - Gen<None> - 55 - (0:0,0)
Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:35 Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:35
CSharpTokenType.Keyword;[try]; SyntaxKind.Keyword;[try];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[finally]; SyntaxKind.Keyword;[finally];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

30
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/DoesNotParseOnSwitchCharacterNotFollowedByOpenAngleOrColon.stree.txt
View File

@ -1,17 +1,17 @@
Statement block - Gen<None> - 30 - (0:0,0) Statement block - Gen<None> - 30 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { @"Foo".ToString(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15 Code span - Gen<Stmt> - [if(foo) { @"Foo".ToString(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;[@"Foo"]; SyntaxKind.StringLiteral;[@"Foo"];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[ToString]; SyntaxKind.Identifier;[ToString];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

6
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/DoesntCaptureWhitespaceAfterUsing.stree.txt
View File

@ -1,5 +1,5 @@
Directive block - Gen<None> - 9 - (0:0,0) Directive block - Gen<None> - 9 - (0:0,0)
Code span - Gen<Import: Foo;> - [using Foo] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:3 Code span - Gen<Import: Foo;> - [using Foo] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:3
CSharpTokenType.Keyword;[using]; SyntaxKind.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];

22
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/HasErrorsIfNamespaceAliasMissingSemicolon.stree.txt
View File

@ -1,13 +1,13 @@
Directive block - Gen<None> - 29 - (0:0,0) Directive block - Gen<None> - 29 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz = FooBarBaz;> - [using Foo.Bar.Baz = FooBarBaz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:11 Code span - Gen<Import: Foo.Bar.Baz = FooBarBaz;> - [using Foo.Bar.Baz = FooBarBaz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:11
CSharpTokenType.Keyword;[using]; SyntaxKind.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Baz]; SyntaxKind.Identifier;[Baz];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[FooBarBaz]; SyntaxKind.Identifier;[FooBarBaz];

14
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/HasErrorsIfNamespaceImportMissingSemicolon.stree.txt
View File

@ -1,9 +1,9 @@
Directive block - Gen<None> - 17 - (0:0,0) Directive block - Gen<None> - 17 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:7 Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[using]; SyntaxKind.Keyword;[using];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Baz]; SyntaxKind.Identifier;[Baz];

38
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithAtDoesntCauseError.stree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 28 - (0:0,0) Statement block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8 Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[true]; SyntaxKind.Keyword;[true];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Statement block - Gen<None> - 14 - (12:0,12) Statement block - Gen<None> - 14 - (12:0,12)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:8 Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:8
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[false]; SyntaxKind.Keyword;[false];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (26:0,26) - Tokens:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (26:0,26) - Tokens:2
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

74
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithCSharpAt.stree.txt
View File

@ -1,41 +1,41 @@
Statement block - Gen<None> - 49 - (0:0,0) Statement block - Gen<None> - 49 - (0:0,0)
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1 MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Code span - Gen<Stmt> - [ if (true) { var val = @x; if (val != 3) { } } ] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[<null>];AtEOL - (1:0,1) - Tokens:35 Code span - Gen<Stmt> - [ if (true) { var val = @x; if (val != 3) { } } ] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[<null>];AtEOL - (1:0,1) - Tokens:35
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[true]; SyntaxKind.Keyword;[true];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[val]; SyntaxKind.Identifier;[val];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
CSharpTokenType.Identifier;[x]; SyntaxKind.Identifier;[x];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[val]; SyntaxKind.Identifier;[val];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.NotEqual;[!=]; SyntaxKind.NotEqual;[!=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[3]; SyntaxKind.IntegerLiteral;[3];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
MetaCode span - Gen<None> - [}] - SpanEditHandler;Accepts:None - (48:0,48) - Tokens:1 MetaCode span - Gen<None> - [}] - SpanEditHandler;Accepts:None - (48:0,48) - Tokens:1
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

62
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithMarkupSetsDotAsMarkup.stree.txt
View File

@ -1,50 +1,50 @@
Statement block - Gen<None> - 51 - (0:0,0) Statement block - Gen<None> - 51 - (0:0,0)
Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8 Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[true]; SyntaxKind.Keyword;[true];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Statement block - Gen<None> - 37 - (12:0,12) Statement block - Gen<None> - 37 - (12:0,12)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [if(false) {] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:6 Code span - Gen<Stmt> - [if(false) {] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:6
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[false]; SyntaxKind.Keyword;[false];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
Markup block - Gen<None> - 24 - (24:0,24) Markup block - Gen<None> - 24 - (24:0,24)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 5 - (25:0,25) Tag block - Gen<None> - 5 - (25:0,25)
Markup span - Gen<Markup> - [<div>] - SpanEditHandler;Accepts:None - (25:0,25) - Tokens:3 Markup span - Gen<Markup> - [<div>] - SpanEditHandler;Accepts:None - (25:0,25) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[div]; SyntaxKind.HtmlTextLiteral;[div];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (30:0,30) - Tokens:1 Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (30:0,30) - Tokens:1
HtmlTokenType.Unknown;[]; SyntaxKind.Unknown;[];
Expression block - Gen<Expr> - 10 - (30:0,30) Expression block - Gen<Expr> - 10 - (30:0,30)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (30:0,30) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (30:0,30) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [something] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (31:0,31) - Tokens:1 Code span - Gen<Expr> - [something] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (31:0,31) - Tokens:1
CSharpTokenType.Identifier;[something]; SyntaxKind.Identifier;[something];
Markup span - Gen<Markup> - [.] - SpanEditHandler;Accepts:Any - (40:0,40) - Tokens:1 Markup span - Gen<Markup> - [.] - SpanEditHandler;Accepts:Any - (40:0,40) - Tokens:1
HtmlTokenType.Text;[.]; SyntaxKind.HtmlTextLiteral;[.];
Tag block - Gen<None> - 6 - (41:0,41) Tag block - Gen<None> - 6 - (41:0,41)
Markup span - Gen<Markup> - [</div>] - SpanEditHandler;Accepts:None - (41:0,41) - Tokens:4 Markup span - Gen<Markup> - [</div>] - SpanEditHandler;Accepts:None - (41:0,41) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[div]; SyntaxKind.HtmlTextLiteral;[div];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (47:0,47) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (47:0,47) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Code span - Gen<Stmt> - [}] - SpanEditHandler;Accepts:Any - (48:0,48) - Tokens:1 Code span - Gen<Stmt> - [}] - SpanEditHandler;Accepts:Any - (48:0,48) - Tokens:1
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (49:0,49) - Tokens:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (49:0,49) - Tokens:2
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

248
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsersCanNestRecursively.stree.txt
View File

@ -1,179 +1,179 @@
Statement block - Gen<None> - 351 - (0:0,0) Statement block - Gen<None> - 351 - (0:0,0)
Code span - Gen<Stmt> - [foreach(var c in db.Categories) {LF] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15 Code span - Gen<Stmt> - [foreach(var c in db.Categories) {LF] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15
CSharpTokenType.Keyword;[foreach]; SyntaxKind.Keyword;[foreach];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[c]; SyntaxKind.Identifier;[c];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[in]; SyntaxKind.Keyword;[in];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[db]; SyntaxKind.Identifier;[db];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Categories]; SyntaxKind.Identifier;[Categories];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Markup block - Gen<None> - 307 - (35:1,0) Markup block - Gen<None> - 307 - (35:1,0)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (35:1,0) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (35:1,0) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 5 - (47:1,12) Tag block - Gen<None> - 5 - (47:1,12)
Markup span - Gen<Markup> - [<div>] - SpanEditHandler;Accepts:None - (47:1,12) - Tokens:3 Markup span - Gen<Markup> - [<div>] - SpanEditHandler;Accepts:None - (47:1,12) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[div]; SyntaxKind.HtmlTextLiteral;[div];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (52:1,17) - Tokens:2 Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (52:1,17) - Tokens:2
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 4 - (70:2,16) Tag block - Gen<None> - 4 - (70:2,16)
Markup span - Gen<Markup> - [<h1>] - SpanEditHandler;Accepts:None - (70:2,16) - Tokens:3 Markup span - Gen<Markup> - [<h1>] - SpanEditHandler;Accepts:None - (70:2,16) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[h1]; SyntaxKind.HtmlTextLiteral;[h1];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (74:2,20) - Tokens:1 Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (74:2,20) - Tokens:1
HtmlTokenType.Unknown;[]; SyntaxKind.Unknown;[];
Expression block - Gen<Expr> - 7 - (74:2,20) Expression block - Gen<Expr> - 7 - (74:2,20)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (74:2,20) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (74:2,20) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [c.Name] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (75:2,21) - Tokens:3 Code span - Gen<Expr> - [c.Name] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (75:2,21) - Tokens:3
CSharpTokenType.Identifier;[c]; SyntaxKind.Identifier;[c];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Name]; SyntaxKind.Identifier;[Name];
Tag block - Gen<None> - 5 - (81:2,27) Tag block - Gen<None> - 5 - (81:2,27)
Markup span - Gen<Markup> - [</h1>] - SpanEditHandler;Accepts:None - (81:2,27) - Tokens:4 Markup span - Gen<Markup> - [</h1>] - SpanEditHandler;Accepts:None - (81:2,27) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[h1]; SyntaxKind.HtmlTextLiteral;[h1];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (86:2,32) - Tokens:2 Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (86:2,32) - Tokens:2
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 4 - (104:3,16) Tag block - Gen<None> - 4 - (104:3,16)
Markup span - Gen<Markup> - [<ul>] - SpanEditHandler;Accepts:None - (104:3,16) - Tokens:3 Markup span - Gen<Markup> - [<ul>] - SpanEditHandler;Accepts:None - (104:3,16) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[ul]; SyntaxKind.HtmlTextLiteral;[ul];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (108:3,20) - Tokens:1 Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (108:3,20) - Tokens:1
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Statement block - Gen<None> - 189 - (110:4,0) Statement block - Gen<None> - 189 - (110:4,0)
Code span - Gen<Stmt> - [ ] - SpanEditHandler;Accepts:Any - (110:4,0) - Tokens:1 Code span - Gen<Stmt> - [ ] - SpanEditHandler;Accepts:Any - (110:4,0) - Tokens:1
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (130:4,20) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (130:4,20) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Stmt> - [foreach(var p in c.Products) {LF] - SpanEditHandler;Accepts:Any - (131:4,21) - Tokens:15 Code span - Gen<Stmt> - [foreach(var p in c.Products) {LF] - SpanEditHandler;Accepts:Any - (131:4,21) - Tokens:15
CSharpTokenType.Keyword;[foreach]; SyntaxKind.Keyword;[foreach];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[var]; SyntaxKind.Identifier;[var];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[p]; SyntaxKind.Identifier;[p];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[in]; SyntaxKind.Keyword;[in];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[c]; SyntaxKind.Identifier;[c];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Products]; SyntaxKind.Identifier;[Products];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Markup block - Gen<None> - 113 - (163:5,0) Markup block - Gen<None> - 113 - (163:5,0)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (163:5,0) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (163:5,0) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 4 - (187:5,24) Tag block - Gen<None> - 4 - (187:5,24)
Markup span - Gen<Markup> - [<li>] - SpanEditHandler;Accepts:None - (187:5,24) - Tokens:3 Markup span - Gen<Markup> - [<li>] - SpanEditHandler;Accepts:None - (187:5,24) - Tokens:3
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[li]; SyntaxKind.HtmlTextLiteral;[li];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Tag block - Gen<None> - 67 - (191:5,28) Tag block - Gen<None> - 67 - (191:5,28)
Markup span - Gen<Markup> - [<a] - SpanEditHandler;Accepts:Any - (191:5,28) - Tokens:2 Markup span - Gen<Markup> - [<a] - SpanEditHandler;Accepts:Any - (191:5,28) - Tokens:2
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.Text;[a]; SyntaxKind.HtmlTextLiteral;[a];
Markup block - Gen<Attr:href, href="@(193:5,30),"@(256:5,93)> - 64 - (193:5,30) Markup block - Gen<Attr:href, href="@(193:5,30),"@(256:5,93)> - 64 - (193:5,30)
Markup span - Gen<None> - [ href="] - SpanEditHandler;Accepts:Any - (193:5,30) - Tokens:4 Markup span - Gen<None> - [ href="] - SpanEditHandler;Accepts:Any - (193:5,30) - Tokens:4
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
HtmlTokenType.Text;[href]; SyntaxKind.HtmlTextLiteral;[href];
HtmlTokenType.Equals;[=]; SyntaxKind.Equals;[=];
HtmlTokenType.DoubleQuote;["]; SyntaxKind.DoubleQuote;["];
Markup block - Gen<DynAttr:@(200:5,37)> - 56 - (200:5,37) Markup block - Gen<DynAttr:@(200:5,37)> - 56 - (200:5,37)
Expression block - Gen<Expr> - 56 - (200:5,37) Expression block - Gen<Expr> - 56 - (200:5,37)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (200:5,37) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (200:5,37) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [Html.ActionUrl("Products", "Detail", new { id = p.Id })] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (201:5,38) - Tokens:24 Code span - Gen<Expr> - [Html.ActionUrl("Products", "Detail", new { id = p.Id })] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (201:5,38) - Tokens:24
CSharpTokenType.Identifier;[Html]; SyntaxKind.Identifier;[Html];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[ActionUrl]; SyntaxKind.Identifier;[ActionUrl];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;["Products"]; SyntaxKind.StringLiteral;["Products"];
CSharpTokenType.Comma;[,]; SyntaxKind.Comma;[,];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["Detail"]; SyntaxKind.StringLiteral;["Detail"];
CSharpTokenType.Comma;[,]; SyntaxKind.Comma;[,];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[id]; SyntaxKind.Identifier;[id];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[p]; SyntaxKind.Identifier;[p];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Id]; SyntaxKind.Identifier;[Id];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
Markup span - Gen<None> - ["] - SpanEditHandler;Accepts:Any - (256:5,93) - Tokens:1 Markup span - Gen<None> - ["] - SpanEditHandler;Accepts:Any - (256:5,93) - Tokens:1
HtmlTokenType.DoubleQuote;["]; SyntaxKind.DoubleQuote;["];
Markup span - Gen<Markup> - [>] - SpanEditHandler;Accepts:None - (257:5,94) - Tokens:1 Markup span - Gen<Markup> - [>] - SpanEditHandler;Accepts:None - (257:5,94) - Tokens:1
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (258:5,95) - Tokens:1 Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (258:5,95) - Tokens:1
HtmlTokenType.Unknown;[]; SyntaxKind.Unknown;[];
Expression block - Gen<Expr> - 7 - (258:5,95) Expression block - Gen<Expr> - 7 - (258:5,95)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (258:5,95) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (258:5,95) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [p.Name] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (259:5,96) - Tokens:3 Code span - Gen<Expr> - [p.Name] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[RTD];K14 - (259:5,96) - Tokens:3
CSharpTokenType.Identifier;[p]; SyntaxKind.Identifier;[p];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[Name]; SyntaxKind.Identifier;[Name];
Tag block - Gen<None> - 4 - (265:5,102) Tag block - Gen<None> - 4 - (265:5,102)
Markup span - Gen<Markup> - [</a>] - SpanEditHandler;Accepts:None - (265:5,102) - Tokens:4 Markup span - Gen<Markup> - [</a>] - SpanEditHandler;Accepts:None - (265:5,102) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[a]; SyntaxKind.HtmlTextLiteral;[a];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Tag block - Gen<None> - 5 - (269:5,106) Tag block - Gen<None> - 5 - (269:5,106)
Markup span - Gen<Markup> - [</li>] - SpanEditHandler;Accepts:None - (269:5,106) - Tokens:4 Markup span - Gen<Markup> - [</li>] - SpanEditHandler;Accepts:None - (269:5,106) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[li]; SyntaxKind.HtmlTextLiteral;[li];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:None - (274:5,111) - Tokens:1 Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:None - (274:5,111) - Tokens:1
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Code span - Gen<Stmt> - [ }LF] - SpanEditHandler;Accepts:None - (276:6,0) - Tokens:3 Code span - Gen<Stmt> - [ }LF] - SpanEditHandler;Accepts:None - (276:6,0) - Tokens:3
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (299:7,0) - Tokens:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (299:7,0) - Tokens:1
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 5 - (315:7,16) Tag block - Gen<None> - 5 - (315:7,16)
Markup span - Gen<Markup> - [</ul>] - SpanEditHandler;Accepts:None - (315:7,16) - Tokens:4 Markup span - Gen<Markup> - [</ul>] - SpanEditHandler;Accepts:None - (315:7,16) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[ul]; SyntaxKind.HtmlTextLiteral;[ul];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (320:7,21) - Tokens:2 Markup span - Gen<Markup> - [LF ] - SpanEditHandler;Accepts:Any - (320:7,21) - Tokens:2
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
HtmlTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Tag block - Gen<None> - 6 - (334:8,12) Tag block - Gen<None> - 6 - (334:8,12)
Markup span - Gen<Markup> - [</div>] - SpanEditHandler;Accepts:None - (334:8,12) - Tokens:4 Markup span - Gen<Markup> - [</div>] - SpanEditHandler;Accepts:None - (334:8,12) - Tokens:4
HtmlTokenType.OpenAngle;[<]; SyntaxKind.OpenAngle;[<];
HtmlTokenType.ForwardSlash;[/]; SyntaxKind.ForwardSlash;[/];
HtmlTokenType.Text;[div]; SyntaxKind.HtmlTextLiteral;[div];
HtmlTokenType.CloseAngle;[>]; SyntaxKind.CloseAngle;[>];
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:None - (340:8,18) - Tokens:1 Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:None - (340:8,18) - Tokens:1
HtmlTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:None - (342:9,0) - Tokens:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:None - (342:9,0) - Tokens:2
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

186
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesElseIfBranchesOfIfStatement.stree.txt
View File

@ -1,95 +1,95 @@
Statement block - Gen<None> - 180 - (0:0,0) Statement block - Gen<None> - 180 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:93 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:93
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"foo } bar"]; SyntaxKind.StringLiteral;[@"foo } bar"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[else]; SyntaxKind.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"]; SyntaxKind.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

36
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesExpressionOnSwitchCharacterFollowedByIdentifierStart.stree.txt
View File

@ -1,24 +1,24 @@
Statement block - Gen<None> - 25 - (0:0,0) Statement block - Gen<None> - 25 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7 Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Expression block - Gen<Expr> - 13 - (10:0,10) Expression block - Gen<Expr> - 13 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
Code span - Gen<Expr> - [foo[4].bar()] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:8 Code span - Gen<Expr> - [foo[4].bar()] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:8
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.LeftBracket;[[]; SyntaxKind.LeftBracket;[[];
CSharpTokenType.IntegerLiteral;[4]; SyntaxKind.IntegerLiteral;[4];
CSharpTokenType.RightBracket;[]]; SyntaxKind.RightBracket;[]];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (23:0,23) - Tokens:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (23:0,23) - Tokens:2
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesExpressionOnSwitchCharacterFollowedByOpenParen.stree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 24 - (0:0,0) Statement block - Gen<None> - 24 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7 Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
Expression block - Gen<Expr> - 12 - (10:0,10) Expression block - Gen<Expr> - 12 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpTokenType.Transition;[@]; SyntaxKind.Transition;[@];
MetaCode span - Gen<None> - [(] - SpanEditHandler;Accepts:None - (11:0,11) - Tokens:1 MetaCode span - Gen<None> - [(] - SpanEditHandler;Accepts:None - (11:0,11) - Tokens:1
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
Code span - Gen<Expr> - [foo + bar] - SpanEditHandler;Accepts:Any - (12:0,12) - Tokens:5 Code span - Gen<Expr> - [foo + bar] - SpanEditHandler;Accepts:Any - (12:0,12) - Tokens:5
CSharpTokenType.Identifier;[foo]; SyntaxKind.Identifier;[foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Plus;[+]; SyntaxKind.Plus;[+];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[bar]; SyntaxKind.Identifier;[bar];
MetaCode span - Gen<None> - [)] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:1 MetaCode span - Gen<None> - [)] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:1
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (22:0,22) - Tokens:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (22:0,22) - Tokens:2
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

474
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParsesMultipleElseIfBranchesOfIfStatement.stree.txt
View File

@ -1,239 +1,239 @@
Statement block - Gen<None> - 459 - (0:0,0) Statement block - Gen<None> - 459 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:237 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:237
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"foo } bar"]; SyntaxKind.StringLiteral;[@"foo } bar"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[else]; SyntaxKind.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"]; SyntaxKind.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[else]; SyntaxKind.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"]; SyntaxKind.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[else]; SyntaxKind.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"]; SyntaxKind.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[else]; SyntaxKind.Keyword;[else];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[if]; SyntaxKind.Keyword;[if];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.Keyword;[int]; SyntaxKind.Keyword;[int];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[0]; SyntaxKind.IntegerLiteral;[0];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[i]; SyntaxKind.Identifier;[i];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LessThan;[<]; SyntaxKind.LessThan;[<];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.IntegerLiteral;[10]; SyntaxKind.IntegerLiteral;[10];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Keyword;[new]; SyntaxKind.Keyword;[new];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Foo]; SyntaxKind.Identifier;[Foo];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Bar]; SyntaxKind.Identifier;[Bar];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Assign;[=]; SyntaxKind.Assign;[=];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.StringLiteral;["baz"]; SyntaxKind.StringLiteral;["baz"];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.LeftBrace;[{]; SyntaxKind.LeftBrace;[{];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.WhiteSpace;[ ]; SyntaxKind.Whitespace;[ ];
CSharpTokenType.Identifier;[Debug]; SyntaxKind.Identifier;[Debug];
CSharpTokenType.Dot;[.]; SyntaxKind.Dot;[.];
CSharpTokenType.Identifier;[WriteLine]; SyntaxKind.Identifier;[WriteLine];
CSharpTokenType.LeftParenthesis;[(]; SyntaxKind.LeftParenthesis;[(];
CSharpTokenType.StringLiteral;[@"bar } baz"]; SyntaxKind.StringLiteral;[@"bar } baz"];
CSharpTokenType.RightParenthesis;[)]; SyntaxKind.RightParenthesis;[)];
CSharpTokenType.Semicolon;[;]; SyntaxKind.Semicolon;[;];
CSharpTokenType.NewLine;[LF]; SyntaxKind.NewLine;[LF];
CSharpTokenType.RightBrace;[}]; SyntaxKind.RightBrace;[}];

Some files were not shown because too many files have changed in this diff Show More