huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Renamed symbol to token everywhere 

- Updated Resources
- Updated test code
- Regenerated baselines
- Added exclusions for apicheck (See https://github.com/aspnet/BuildTools/issues/704)
This commit is contained in:
Ajay Bhargav Baaskaran 2018-07-06 14:24:15 -07:00
parent ed7c4e8f5c
commit ce4780a830
989 changed files with 22531 additions and 22501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs
View File

@ -593,14 +593,14 @@ namespace Microsoft.AspNetCore.Razor.Language
public override void VisitMarkupSpan(MarkupChunkGenerator chunkGenerator, Span span) public override void VisitMarkupSpan(MarkupChunkGenerator chunkGenerator, Span span)
{ {
if (span.Symbols.Count == 1) if (span.Tokens.Count == 1)
{ {
var symbol = span.Symbols[0] as HtmlSymbol; var token = span.Tokens[0] as HtmlToken;
if (symbol != null && if (token != null &&
symbol.Type == HtmlSymbolType.Unknown && token.Type == HtmlTokenType.Unknown &&
symbol.Content.Length == 0) token.Content.Length == 0)
{ {
// We don't want to create IR nodes for marker symbols. // We don't want to create IR nodes for marker tokens.
return; return;
} }
} }

2
src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs
View File

@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
internal class DirectiveTokenEditHandler : SpanEditHandler internal class DirectiveTokenEditHandler : SpanEditHandler
{ {
public DirectiveTokenEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer) : base(tokenizer) public DirectiveTokenEditHandler(Func<string, IEnumerable<IToken>> tokenizer) : base(tokenizer)
{ {
} }

16
src/Microsoft.AspNetCore.Razor.Language/Extensions/DesignTimeDirectiveTargetExtension.cs
View File

@ -59,8 +59,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
if (string.IsNullOrEmpty(node.Content)) if (string.IsNullOrEmpty(node.Content))
{ {
// This is most likely a marker symbol. // This is most likely a marker token.
WriteMarkerSymbol(context, node); WriteMarkerToken(context, node);
break; break;
} }
@ -80,8 +80,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
if (string.IsNullOrEmpty(node.Content)) if (string.IsNullOrEmpty(node.Content))
{ {
// This is most likely a marker symbol. // This is most likely a marker token.
WriteMarkerSymbol(context, node); WriteMarkerToken(context, node);
break; break;
} }
@ -102,8 +102,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
if (string.IsNullOrEmpty(node.Content)) if (string.IsNullOrEmpty(node.Content))
{ {
// This is most likely a marker symbol. // This is most likely a marker token.
WriteMarkerSymbol(context, node); WriteMarkerToken(context, node);
break; break;
} }
@ -155,9 +155,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
context.CodeWriter.WriteLine("))();"); context.CodeWriter.WriteLine("))();");
} }
private void WriteMarkerSymbol(CodeRenderingContext context, DirectiveTokenIntermediateNode node) private void WriteMarkerToken(CodeRenderingContext context, DirectiveTokenIntermediateNode node)
{ {
// We want to map marker symbols to a location in the generated document // We want to map marker tokens to a location in the generated document
// that will provide CSharp intellisense. // that will provide CSharp intellisense.
context.AddSourceMappingFor(node); context.AddSourceMappingFor(node);
context.CodeWriter.Write(" "); context.CodeWriter.Write(" ");

6
src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs
View File

@ -11,18 +11,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
private static readonly int TypeHashCode = typeof(AutoCompleteEditHandler).GetHashCode(); private static readonly int TypeHashCode = typeof(AutoCompleteEditHandler).GetHashCode();
public AutoCompleteEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer) public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer)
: base(tokenizer) : base(tokenizer)
{ {
} }
public AutoCompleteEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer, bool autoCompleteAtEndOfSpan) public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer, bool autoCompleteAtEndOfSpan)
: this(tokenizer) : this(tokenizer)
{ {
AutoCompleteAtEndOfSpan = autoCompleteAtEndOfSpan; AutoCompleteAtEndOfSpan = autoCompleteAtEndOfSpan;
} }
public AutoCompleteEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer, AcceptedCharactersInternal accepted) public AutoCompleteEditHandler(Func<string, IEnumerable<IToken>> tokenizer, AcceptedCharactersInternal accepted)
: base(tokenizer, accepted) : base(tokenizer, accepted)
{ {
} }

438
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
View File

File diff suppressed because it is too large Load Diff

224
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs
View File

@ -6,61 +6,61 @@ using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer, CSharpSymbol, CSharpSymbolType> internal class CSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer, CSharpToken, CSharpTokenType>
{ {
private static readonly CSharpLanguageCharacteristics _instance = new CSharpLanguageCharacteristics(); private static readonly CSharpLanguageCharacteristics _instance = new CSharpLanguageCharacteristics();
private static Dictionary<CSharpSymbolType, string> _symbolSamples = new Dictionary<CSharpSymbolType, string>() private static Dictionary<CSharpTokenType, string> _tokenSamples = new Dictionary<CSharpTokenType, string>()
{ {
{ CSharpSymbolType.Arrow, "->" }, { CSharpTokenType.Arrow, "->" },
{ CSharpSymbolType.Minus, "-" }, { CSharpTokenType.Minus, "-" },
{ CSharpSymbolType.Decrement, "--" }, { CSharpTokenType.Decrement, "--" },
{ CSharpSymbolType.MinusAssign, "-=" }, { CSharpTokenType.MinusAssign, "-=" },
{ CSharpSymbolType.NotEqual, "!=" }, { CSharpTokenType.NotEqual, "!=" },
{ CSharpSymbolType.Not, "!" }, { CSharpTokenType.Not, "!" },
{ CSharpSymbolType.Modulo, "%" }, { CSharpTokenType.Modulo, "%" },
{ CSharpSymbolType.ModuloAssign, "%=" }, { CSharpTokenType.ModuloAssign, "%=" },
{ CSharpSymbolType.AndAssign, "&=" }, { CSharpTokenType.AndAssign, "&=" },
{ CSharpSymbolType.And, "&" }, { CSharpTokenType.And, "&" },
{ CSharpSymbolType.DoubleAnd, "&&" }, { CSharpTokenType.DoubleAnd, "&&" },
{ CSharpSymbolType.LeftParenthesis, "(" }, { CSharpTokenType.LeftParenthesis, "(" },
{ CSharpSymbolType.RightParenthesis, ")" }, { CSharpTokenType.RightParenthesis, ")" },
{ CSharpSymbolType.Star, "*" }, { CSharpTokenType.Star, "*" },
{ CSharpSymbolType.MultiplyAssign, "*=" }, { CSharpTokenType.MultiplyAssign, "*=" },
{ CSharpSymbolType.Comma, "," }, { CSharpTokenType.Comma, "," },
{ CSharpSymbolType.Dot, "." }, { CSharpTokenType.Dot, "." },
{ CSharpSymbolType.Slash, "/" }, { CSharpTokenType.Slash, "/" },
{ CSharpSymbolType.DivideAssign, "/=" }, { CSharpTokenType.DivideAssign, "/=" },
{ CSharpSymbolType.DoubleColon, "::" }, { CSharpTokenType.DoubleColon, "::" },
{ CSharpSymbolType.Colon, ":" }, { CSharpTokenType.Colon, ":" },
{ CSharpSymbolType.Semicolon, ";" }, { CSharpTokenType.Semicolon, ";" },
{ CSharpSymbolType.QuestionMark, "?" }, { CSharpTokenType.QuestionMark, "?" },
{ CSharpSymbolType.NullCoalesce, "??" }, { CSharpTokenType.NullCoalesce, "??" },
{ CSharpSymbolType.RightBracket, "]" }, { CSharpTokenType.RightBracket, "]" },
{ CSharpSymbolType.LeftBracket, "[" }, { CSharpTokenType.LeftBracket, "[" },
{ CSharpSymbolType.XorAssign, "^=" }, { CSharpTokenType.XorAssign, "^=" },
{ CSharpSymbolType.Xor, "^" }, { CSharpTokenType.Xor, "^" },
{ CSharpSymbolType.LeftBrace, "{" }, { CSharpTokenType.LeftBrace, "{" },
{ CSharpSymbolType.OrAssign, "|=" }, { CSharpTokenType.OrAssign, "|=" },
{ CSharpSymbolType.DoubleOr, "||" }, { CSharpTokenType.DoubleOr, "||" },
{ CSharpSymbolType.Or, "|" }, { CSharpTokenType.Or, "|" },
{ CSharpSymbolType.RightBrace, "}" }, { CSharpTokenType.RightBrace, "}" },
{ CSharpSymbolType.Tilde, "~" }, { CSharpTokenType.Tilde, "~" },
{ CSharpSymbolType.Plus, "+" }, { CSharpTokenType.Plus, "+" },
{ CSharpSymbolType.PlusAssign, "+=" }, { CSharpTokenType.PlusAssign, "+=" },
{ CSharpSymbolType.Increment, "++" }, { CSharpTokenType.Increment, "++" },
{ CSharpSymbolType.LessThan, "<" }, { CSharpTokenType.LessThan, "<" },
{ CSharpSymbolType.LessThanEqual, "<=" }, { CSharpTokenType.LessThanEqual, "<=" },
{ CSharpSymbolType.LeftShift, "<<" }, { CSharpTokenType.LeftShift, "<<" },
{ CSharpSymbolType.LeftShiftAssign, "<<=" }, { CSharpTokenType.LeftShiftAssign, "<<=" },
{ CSharpSymbolType.Assign, "=" }, { CSharpTokenType.Assign, "=" },
{ CSharpSymbolType.Equals, "==" }, { CSharpTokenType.Equals, "==" },
{ CSharpSymbolType.GreaterThan, ">" }, { CSharpTokenType.GreaterThan, ">" },
{ CSharpSymbolType.GreaterThanEqual, ">=" }, { CSharpTokenType.GreaterThanEqual, ">=" },
{ CSharpSymbolType.RightShift, ">>" }, { CSharpTokenType.RightShift, ">>" },
{ CSharpSymbolType.RightShiftAssign, ">>=" }, { CSharpTokenType.RightShiftAssign, ">>=" },
{ CSharpSymbolType.Hash, "#" }, { CSharpTokenType.Hash, "#" },
{ CSharpSymbolType.Transition, "@" }, { CSharpTokenType.Transition, "@" },
}; };
protected CSharpLanguageCharacteristics() protected CSharpLanguageCharacteristics()
@ -74,96 +74,96 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source); return new CSharpTokenizer(source);
} }
protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList<RazorDiagnostic> errors) protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new CSharpSymbol(content, type, errors); return new CSharpToken(content, type, errors);
} }
public override string GetSample(CSharpSymbolType type) public override string GetSample(CSharpTokenType type)
{ {
string sample; string sample;
if (!_symbolSamples.TryGetValue(type, out sample)) if (!_tokenSamples.TryGetValue(type, out sample))
{ {
switch (type) switch (type)
{ {
case CSharpSymbolType.Identifier: case CSharpTokenType.Identifier:
return Resources.CSharpSymbol_Identifier; return Resources.CSharpToken_Identifier;
case CSharpSymbolType.Keyword: case CSharpTokenType.Keyword:
return Resources.CSharpSymbol_Keyword; return Resources.CSharpToken_Keyword;
case CSharpSymbolType.IntegerLiteral: case CSharpTokenType.IntegerLiteral:
return Resources.CSharpSymbol_IntegerLiteral; return Resources.CSharpToken_IntegerLiteral;
case CSharpSymbolType.NewLine: case CSharpTokenType.NewLine:
return Resources.CSharpSymbol_Newline; return Resources.CSharpToken_Newline;
case CSharpSymbolType.WhiteSpace: case CSharpTokenType.WhiteSpace:
return Resources.CSharpSymbol_Whitespace; return Resources.CSharpToken_Whitespace;
case CSharpSymbolType.Comment: case CSharpTokenType.Comment:
return Resources.CSharpSymbol_Comment; return Resources.CSharpToken_Comment;
case CSharpSymbolType.RealLiteral: case CSharpTokenType.RealLiteral:
return Resources.CSharpSymbol_RealLiteral; return Resources.CSharpToken_RealLiteral;
case CSharpSymbolType.CharacterLiteral: case CSharpTokenType.CharacterLiteral:
return Resources.CSharpSymbol_CharacterLiteral; return Resources.CSharpToken_CharacterLiteral;
case CSharpSymbolType.StringLiteral: case CSharpTokenType.StringLiteral:
return Resources.CSharpSymbol_StringLiteral; return Resources.CSharpToken_StringLiteral;
default: default:
return Resources.Symbol_Unknown; return Resources.Token_Unknown;
} }
} }
return sample; return sample;
} }
public override CSharpSymbol CreateMarkerSymbol() public override CSharpToken CreateMarkerToken()
{ {
return new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown); return new CSharpToken(string.Empty, CSharpTokenType.Unknown);
} }
public override CSharpSymbolType GetKnownSymbolType(KnownSymbolType type) public override CSharpTokenType GetKnownTokenType(KnownTokenType type)
{ {
switch (type) switch (type)
{ {
case KnownSymbolType.Identifier: case KnownTokenType.Identifier:
return CSharpSymbolType.Identifier; return CSharpTokenType.Identifier;
case KnownSymbolType.Keyword: case KnownTokenType.Keyword:
return CSharpSymbolType.Keyword; return CSharpTokenType.Keyword;
case KnownSymbolType.NewLine: case KnownTokenType.NewLine:
return CSharpSymbolType.NewLine; return CSharpTokenType.NewLine;
case KnownSymbolType.WhiteSpace: case KnownTokenType.WhiteSpace:
return CSharpSymbolType.WhiteSpace; return CSharpTokenType.WhiteSpace;
case KnownSymbolType.Transition: case KnownTokenType.Transition:
return CSharpSymbolType.Transition; return CSharpTokenType.Transition;
case KnownSymbolType.CommentStart: case KnownTokenType.CommentStart:
return CSharpSymbolType.RazorCommentTransition; return CSharpTokenType.RazorCommentTransition;
case KnownSymbolType.CommentStar: case KnownTokenType.CommentStar:
return CSharpSymbolType.RazorCommentStar; return CSharpTokenType.RazorCommentStar;
case KnownSymbolType.CommentBody: case KnownTokenType.CommentBody:
return CSharpSymbolType.RazorComment; return CSharpTokenType.RazorComment;
default: default:
return CSharpSymbolType.Unknown; return CSharpTokenType.Unknown;
} }
} }
public override CSharpSymbolType FlipBracket(CSharpSymbolType bracket) public override CSharpTokenType FlipBracket(CSharpTokenType bracket)
{ {
switch (bracket) switch (bracket)
{ {
case CSharpSymbolType.LeftBrace: case CSharpTokenType.LeftBrace:
return CSharpSymbolType.RightBrace; return CSharpTokenType.RightBrace;
case CSharpSymbolType.LeftBracket: case CSharpTokenType.LeftBracket:
return CSharpSymbolType.RightBracket; return CSharpTokenType.RightBracket;
case CSharpSymbolType.LeftParenthesis: case CSharpTokenType.LeftParenthesis:
return CSharpSymbolType.RightParenthesis; return CSharpTokenType.RightParenthesis;
case CSharpSymbolType.LessThan: case CSharpTokenType.LessThan:
return CSharpSymbolType.GreaterThan; return CSharpTokenType.GreaterThan;
case CSharpSymbolType.RightBrace: case CSharpTokenType.RightBrace:
return CSharpSymbolType.LeftBrace; return CSharpTokenType.LeftBrace;
case CSharpSymbolType.RightBracket: case CSharpTokenType.RightBracket:
return CSharpSymbolType.LeftBracket; return CSharpTokenType.LeftBracket;
case CSharpSymbolType.RightParenthesis: case CSharpTokenType.RightParenthesis:
return CSharpSymbolType.LeftParenthesis; return CSharpTokenType.LeftParenthesis;
case CSharpSymbolType.GreaterThan: case CSharpTokenType.GreaterThan:
return CSharpSymbolType.LessThan; return CSharpTokenType.LessThan;
default: default:
Debug.Fail("FlipBracket must be called with a bracket character"); Debug.Fail("FlipBracket must be called with a bracket character");
return CSharpSymbolType.Unknown; return CSharpTokenType.Unknown;
} }
} }

12
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbol.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpToken.cs
View File

@ -6,11 +6,11 @@ using System.Collections.Generic;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CSharpSymbol : SymbolBase<CSharpSymbolType> internal class CSharpToken : TokenBase<CSharpTokenType>
{ {
public CSharpSymbol( public CSharpToken(
string content, string content,
CSharpSymbolType type) CSharpTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray) : base(content, type, RazorDiagnostic.EmptyArray)
{ {
if (content == null) if (content == null)
@ -19,9 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public CSharpSymbol( public CSharpToken(
string content, string content,
CSharpSymbolType type, CSharpTokenType type,
IReadOnlyList<RazorDiagnostic> errors) IReadOnlyList<RazorDiagnostic> errors)
: base(content, type, errors) : base(content, type, errors)
{ {
@ -35,7 +35,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var other = obj as CSharpSymbol; var other = obj as CSharpToken;
return base.Equals(other) && return base.Equals(other) &&
other.Keyword == Keyword; other.Keyword == Keyword;
} }

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbolType.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenType.cs
View File

@ -3,7 +3,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal enum CSharpSymbolType internal enum CSharpTokenType
{ {
Unknown, Unknown,
Identifier, Identifier,

276
src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs
View File

@ -8,9 +8,9 @@ using System.Globalization;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CSharpTokenizer : Tokenizer<CSharpSymbol, CSharpSymbolType> internal class CSharpTokenizer : Tokenizer<CSharpToken, CSharpTokenType>
{ {
private Dictionary<char, Func<CSharpSymbolType>> _operatorHandlers; private Dictionary<char, Func<CSharpTokenType>> _operatorHandlers;
private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal) private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal)
{ {
@ -100,31 +100,31 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
base.CurrentState = StartState; base.CurrentState = StartState;
_operatorHandlers = new Dictionary<char, Func<CSharpSymbolType>>() _operatorHandlers = new Dictionary<char, Func<CSharpTokenType>>()
{ {
{ '-', MinusOperator }, { '-', MinusOperator },
{ '<', LessThanOperator }, { '<', LessThanOperator },
{ '>', GreaterThanOperator }, { '>', GreaterThanOperator },
{ '&', CreateTwoCharOperatorHandler(CSharpSymbolType.And, '=', CSharpSymbolType.AndAssign, '&', CSharpSymbolType.DoubleAnd) }, { '&', CreateTwoCharOperatorHandler(CSharpTokenType.And, '=', CSharpTokenType.AndAssign, '&', CSharpTokenType.DoubleAnd) },
{ '|', CreateTwoCharOperatorHandler(CSharpSymbolType.Or, '=', CSharpSymbolType.OrAssign, '|', CSharpSymbolType.DoubleOr) }, { '|', CreateTwoCharOperatorHandler(CSharpTokenType.Or, '=', CSharpTokenType.OrAssign, '|', CSharpTokenType.DoubleOr) },
{ '+', CreateTwoCharOperatorHandler(CSharpSymbolType.Plus, '=', CSharpSymbolType.PlusAssign, '+', CSharpSymbolType.Increment) }, { '+', CreateTwoCharOperatorHandler(CSharpTokenType.Plus, '=', CSharpTokenType.PlusAssign, '+', CSharpTokenType.Increment) },
{ '=', CreateTwoCharOperatorHandler(CSharpSymbolType.Assign, '=', CSharpSymbolType.Equals, '>', CSharpSymbolType.GreaterThanEqual) }, { '=', CreateTwoCharOperatorHandler(CSharpTokenType.Assign, '=', CSharpTokenType.Equals, '>', CSharpTokenType.GreaterThanEqual) },
{ '!', CreateTwoCharOperatorHandler(CSharpSymbolType.Not, '=', CSharpSymbolType.NotEqual) }, { '!', CreateTwoCharOperatorHandler(CSharpTokenType.Not, '=', CSharpTokenType.NotEqual) },
{ '%', CreateTwoCharOperatorHandler(CSharpSymbolType.Modulo, '=', CSharpSymbolType.ModuloAssign) }, { '%', CreateTwoCharOperatorHandler(CSharpTokenType.Modulo, '=', CSharpTokenType.ModuloAssign) },
{ '*', CreateTwoCharOperatorHandler(CSharpSymbolType.Star, '=', CSharpSymbolType.MultiplyAssign) }, { '*', CreateTwoCharOperatorHandler(CSharpTokenType.Star, '=', CSharpTokenType.MultiplyAssign) },
{ ':', CreateTwoCharOperatorHandler(CSharpSymbolType.Colon, ':', CSharpSymbolType.DoubleColon) }, { ':', CreateTwoCharOperatorHandler(CSharpTokenType.Colon, ':', CSharpTokenType.DoubleColon) },
{ '?', CreateTwoCharOperatorHandler(CSharpSymbolType.QuestionMark, '?', CSharpSymbolType.NullCoalesce) }, { '?', CreateTwoCharOperatorHandler(CSharpTokenType.QuestionMark, '?', CSharpTokenType.NullCoalesce) },
{ '^', CreateTwoCharOperatorHandler(CSharpSymbolType.Xor, '=', CSharpSymbolType.XorAssign) }, { '^', CreateTwoCharOperatorHandler(CSharpTokenType.Xor, '=', CSharpTokenType.XorAssign) },
{ '(', () => CSharpSymbolType.LeftParenthesis }, { '(', () => CSharpTokenType.LeftParenthesis },
{ ')', () => CSharpSymbolType.RightParenthesis }, { ')', () => CSharpTokenType.RightParenthesis },
{ '{', () => CSharpSymbolType.LeftBrace }, { '{', () => CSharpTokenType.LeftBrace },
{ '}', () => CSharpSymbolType.RightBrace }, { '}', () => CSharpTokenType.RightBrace },
{ '[', () => CSharpSymbolType.LeftBracket }, { '[', () => CSharpTokenType.LeftBracket },
{ ']', () => CSharpSymbolType.RightBracket }, { ']', () => CSharpTokenType.RightBracket },
{ ',', () => CSharpSymbolType.Comma }, { ',', () => CSharpTokenType.Comma },
{ ';', () => CSharpSymbolType.Semicolon }, { ';', () => CSharpTokenType.Semicolon },
{ '~', () => CSharpSymbolType.Tilde }, { '~', () => CSharpTokenType.Tilde },
{ '#', () => CSharpSymbolType.Hash } { '#', () => CSharpTokenType.Hash }
}; };
} }
@ -132,11 +132,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState; private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState;
public override CSharpSymbolType RazorCommentType => CSharpSymbolType.RazorComment; public override CSharpTokenType RazorCommentType => CSharpTokenType.RazorComment;
public override CSharpSymbolType RazorCommentTransitionType => CSharpSymbolType.RazorCommentTransition; public override CSharpTokenType RazorCommentTransitionType => CSharpTokenType.RazorCommentTransition;
public override CSharpSymbolType RazorCommentStarType => CSharpSymbolType.RazorCommentStar; public override CSharpTokenType RazorCommentStarType => CSharpTokenType.RazorCommentStar;
protected override StateResult Dispatch() protected override StateResult Dispatch()
{ {
@ -160,8 +160,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return RazorCommentBody(); return RazorCommentBody();
case CSharpTokenizerState.StarAfterRazorCommentBody: case CSharpTokenizerState.StarAfterRazorCommentBody:
return StarAfterRazorCommentBody(); return StarAfterRazorCommentBody();
case CSharpTokenizerState.AtSymbolAfterRazorCommentBody: case CSharpTokenizerState.AtTokenAfterRazorCommentBody:
return AtSymbolAfterRazorCommentBody(); return AtTokenAfterRazorCommentBody();
default: default:
Debug.Fail("Invalid TokenizerState"); Debug.Fail("Invalid TokenizerState");
return default(StateResult); return default(StateResult);
@ -169,15 +169,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Optimize memory allocation by returning constants for the most frequent cases // Optimize memory allocation by returning constants for the most frequent cases
protected override string GetSymbolContent(CSharpSymbolType type) protected override string GetTokenContent(CSharpTokenType type)
{ {
var symbolLength = Buffer.Length; var tokenLength = Buffer.Length;
if (symbolLength == 1) if (tokenLength == 1)
{ {
switch (type) switch (type)
{ {
case CSharpSymbolType.IntegerLiteral: case CSharpTokenType.IntegerLiteral:
switch (Buffer[0]) switch (Buffer[0])
{ {
case '0': case '0':
@ -202,13 +202,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "9"; return "9";
} }
break; break;
case CSharpSymbolType.NewLine: case CSharpTokenType.NewLine:
if (Buffer[0] == '\n') if (Buffer[0] == '\n')
{ {
return "\n"; return "\n";
} }
break; break;
case CSharpSymbolType.WhiteSpace: case CSharpTokenType.WhiteSpace:
if (Buffer[0] == ' ') if (Buffer[0] == ' ')
{ {
return " "; return " ";
@ -218,134 +218,134 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t"; return "\t";
} }
break; break;
case CSharpSymbolType.Minus: case CSharpTokenType.Minus:
return "-"; return "-";
case CSharpSymbolType.Not: case CSharpTokenType.Not:
return "!"; return "!";
case CSharpSymbolType.Modulo: case CSharpTokenType.Modulo:
return "%"; return "%";
case CSharpSymbolType.And: case CSharpTokenType.And:
return "&"; return "&";
case CSharpSymbolType.LeftParenthesis: case CSharpTokenType.LeftParenthesis:
return "("; return "(";
case CSharpSymbolType.RightParenthesis: case CSharpTokenType.RightParenthesis:
return ")"; return ")";
case CSharpSymbolType.Star: case CSharpTokenType.Star:
return "*"; return "*";
case CSharpSymbolType.Comma: case CSharpTokenType.Comma:
return ","; return ",";
case CSharpSymbolType.Dot: case CSharpTokenType.Dot:
return "."; return ".";
case CSharpSymbolType.Slash: case CSharpTokenType.Slash:
return "/"; return "/";
case CSharpSymbolType.Colon: case CSharpTokenType.Colon:
return ":"; return ":";
case CSharpSymbolType.Semicolon: case CSharpTokenType.Semicolon:
return ";"; return ";";
case CSharpSymbolType.QuestionMark: case CSharpTokenType.QuestionMark:
return "?"; return "?";
case CSharpSymbolType.RightBracket: case CSharpTokenType.RightBracket:
return "]"; return "]";
case CSharpSymbolType.LeftBracket: case CSharpTokenType.LeftBracket:
return "["; return "[";
case CSharpSymbolType.Xor: case CSharpTokenType.Xor:
return "^"; return "^";
case CSharpSymbolType.LeftBrace: case CSharpTokenType.LeftBrace:
return "{"; return "{";
case CSharpSymbolType.Or: case CSharpTokenType.Or:
return "|"; return "|";
case CSharpSymbolType.RightBrace: case CSharpTokenType.RightBrace:
return "}"; return "}";
case CSharpSymbolType.Tilde: case CSharpTokenType.Tilde:
return "~"; return "~";
case CSharpSymbolType.Plus: case CSharpTokenType.Plus:
return "+"; return "+";
case CSharpSymbolType.LessThan: case CSharpTokenType.LessThan:
return "<"; return "<";
case CSharpSymbolType.Assign: case CSharpTokenType.Assign:
return "="; return "=";
case CSharpSymbolType.GreaterThan: case CSharpTokenType.GreaterThan:
return ">"; return ">";
case CSharpSymbolType.Hash: case CSharpTokenType.Hash:
return "#"; return "#";
case CSharpSymbolType.Transition: case CSharpTokenType.Transition:
return "@"; return "@";
} }
} }
else if (symbolLength == 2) else if (tokenLength == 2)
{ {
switch (type) switch (type)
{ {
case CSharpSymbolType.NewLine: case CSharpTokenType.NewLine:
return "\r\n"; return "\r\n";
case CSharpSymbolType.Arrow: case CSharpTokenType.Arrow:
return "->"; return "->";
case CSharpSymbolType.Decrement: case CSharpTokenType.Decrement:
return "--"; return "--";
case CSharpSymbolType.MinusAssign: case CSharpTokenType.MinusAssign:
return "-="; return "-=";
case CSharpSymbolType.NotEqual: case CSharpTokenType.NotEqual:
return "!="; return "!=";
case CSharpSymbolType.ModuloAssign: case CSharpTokenType.ModuloAssign:
return "%="; return "%=";
case CSharpSymbolType.AndAssign: case CSharpTokenType.AndAssign:
return "&="; return "&=";
case CSharpSymbolType.DoubleAnd: case CSharpTokenType.DoubleAnd:
return "&&"; return "&&";
case CSharpSymbolType.MultiplyAssign: case CSharpTokenType.MultiplyAssign:
return "*="; return "*=";
case CSharpSymbolType.DivideAssign: case CSharpTokenType.DivideAssign:
return "/="; return "/=";
case CSharpSymbolType.DoubleColon: case CSharpTokenType.DoubleColon:
return "::"; return "::";
case CSharpSymbolType.NullCoalesce: case CSharpTokenType.NullCoalesce:
return "??"; return "??";
case CSharpSymbolType.XorAssign: case CSharpTokenType.XorAssign:
return "^="; return "^=";
case CSharpSymbolType.OrAssign: case CSharpTokenType.OrAssign:
return "|="; return "|=";
case CSharpSymbolType.DoubleOr: case CSharpTokenType.DoubleOr:
return "||"; return "||";
case CSharpSymbolType.PlusAssign: case CSharpTokenType.PlusAssign:
return "+="; return "+=";
case CSharpSymbolType.Increment: case CSharpTokenType.Increment:
return "++"; return "++";
case CSharpSymbolType.LessThanEqual: case CSharpTokenType.LessThanEqual:
return "<="; return "<=";
case CSharpSymbolType.LeftShift: case CSharpTokenType.LeftShift:
return "<<"; return "<<";
case CSharpSymbolType.Equals: case CSharpTokenType.Equals:
return "=="; return "==";
case CSharpSymbolType.GreaterThanEqual: case CSharpTokenType.GreaterThanEqual:
if (Buffer[0] == '=') if (Buffer[0] == '=')
{ {
return "=>"; return "=>";
} }
return ">="; return ">=";
case CSharpSymbolType.RightShift: case CSharpTokenType.RightShift:
return ">>"; return ">>";
} }
} }
else if (symbolLength == 3) else if (tokenLength == 3)
{ {
switch (type) switch (type)
{ {
case CSharpSymbolType.LeftShiftAssign: case CSharpTokenType.LeftShiftAssign:
return "<<="; return "<<=";
case CSharpSymbolType.RightShiftAssign: case CSharpTokenType.RightShiftAssign:
return ">>="; return ">>=";
} }
} }
return base.GetSymbolContent(type); return base.GetTokenContent(type);
} }
protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList<RazorDiagnostic> errors) protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new CSharpSymbol(content, type, errors); return new CSharpToken(content, type, errors);
} }
private StateResult Data() private StateResult Data()
@ -359,13 +359,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); TakeCurrent();
} }
return Stay(EndSymbol(CSharpSymbolType.NewLine)); return Stay(EndToken(CSharpTokenType.NewLine));
} }
else if (ParserHelpers.IsWhitespace(CurrentCharacter)) else if (ParserHelpers.IsWhitespace(CurrentCharacter))
{ {
// CSharp Spec §2.3.3 // CSharp Spec §2.3.3
TakeUntil(c => !ParserHelpers.IsWhitespace(c)); TakeUntil(c => !ParserHelpers.IsWhitespace(c));
return Stay(EndSymbol(CSharpSymbolType.WhiteSpace)); return Stay(EndToken(CSharpTokenType.WhiteSpace));
} }
else if (IsIdentifierStart(CurrentCharacter)) else if (IsIdentifierStart(CurrentCharacter))
{ {
@ -378,7 +378,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
switch (CurrentCharacter) switch (CurrentCharacter)
{ {
case '@': case '@':
return AtSymbol(); return AtToken();
case '\'': case '\'':
TakeCurrent(); TakeCurrent();
return Transition(CSharpTokenizerState.QuotedCharacterLiteral); return Transition(CSharpTokenizerState.QuotedCharacterLiteral);
@ -390,7 +390,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return RealLiteral(); return RealLiteral();
} }
return Stay(Single(CSharpSymbolType.Dot)); return Stay(Single(CSharpTokenType.Dot));
case '/': case '/':
TakeCurrent(); TakeCurrent();
if (CurrentCharacter == '/') if (CurrentCharacter == '/')
@ -406,18 +406,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else if (CurrentCharacter == '=') else if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return Stay(EndSymbol(CSharpSymbolType.DivideAssign)); return Stay(EndToken(CSharpTokenType.DivideAssign));
} }
else else
{ {
return Stay(EndSymbol(CSharpSymbolType.Slash)); return Stay(EndToken(CSharpTokenType.Slash));
} }
default: default:
return Stay(EndSymbol(Operator())); return Stay(EndToken(Operator()));
} }
} }
private StateResult AtSymbol() private StateResult AtToken()
{ {
TakeCurrent(); TakeCurrent();
if (CurrentCharacter == '"') if (CurrentCharacter == '"')
@ -429,78 +429,78 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return Transition( return Transition(
CSharpTokenizerState.AfterRazorCommentTransition, CSharpTokenizerState.AfterRazorCommentTransition,
EndSymbol(CSharpSymbolType.RazorCommentTransition)); EndToken(CSharpTokenType.RazorCommentTransition));
} }
else if (CurrentCharacter == '@') else if (CurrentCharacter == '@')
{ {
// Could be escaped comment transition // Could be escaped comment transition
return Transition( return Transition(
CSharpTokenizerState.EscapedRazorCommentTransition, CSharpTokenizerState.EscapedRazorCommentTransition,
EndSymbol(CSharpSymbolType.Transition)); EndToken(CSharpTokenType.Transition));
} }
return Stay(EndSymbol(CSharpSymbolType.Transition)); return Stay(EndToken(CSharpTokenType.Transition));
} }
private StateResult EscapedRazorCommentTransition() private StateResult EscapedRazorCommentTransition()
{ {
TakeCurrent(); TakeCurrent();
return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Transition)); return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Transition));
} }
private CSharpSymbolType Operator() private CSharpTokenType Operator()
{ {
var first = CurrentCharacter; var first = CurrentCharacter;
TakeCurrent(); TakeCurrent();
Func<CSharpSymbolType> handler; Func<CSharpTokenType> handler;
if (_operatorHandlers.TryGetValue(first, out handler)) if (_operatorHandlers.TryGetValue(first, out handler))
{ {
return handler(); return handler();
} }
return CSharpSymbolType.Unknown; return CSharpTokenType.Unknown;
} }
private CSharpSymbolType LessThanOperator() private CSharpTokenType LessThanOperator()
{ {
if (CurrentCharacter == '=') if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return CSharpSymbolType.LessThanEqual; return CSharpTokenType.LessThanEqual;
} }
return CSharpSymbolType.LessThan; return CSharpTokenType.LessThan;
} }
private CSharpSymbolType GreaterThanOperator() private CSharpTokenType GreaterThanOperator()
{ {
if (CurrentCharacter == '=') if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return CSharpSymbolType.GreaterThanEqual; return CSharpTokenType.GreaterThanEqual;
} }
return CSharpSymbolType.GreaterThan; return CSharpTokenType.GreaterThan;
} }
private CSharpSymbolType MinusOperator() private CSharpTokenType MinusOperator()
{ {
if (CurrentCharacter == '>') if (CurrentCharacter == '>')
{ {
TakeCurrent(); TakeCurrent();
return CSharpSymbolType.Arrow; return CSharpTokenType.Arrow;
} }
else if (CurrentCharacter == '-') else if (CurrentCharacter == '-')
{ {
TakeCurrent(); TakeCurrent();
return CSharpSymbolType.Decrement; return CSharpTokenType.Decrement;
} }
else if (CurrentCharacter == '=') else if (CurrentCharacter == '=')
{ {
TakeCurrent(); TakeCurrent();
return CSharpSymbolType.MinusAssign; return CSharpTokenType.MinusAssign;
} }
return CSharpSymbolType.Minus; return CSharpTokenType.Minus;
} }
private Func<CSharpSymbolType> CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char second, CSharpSymbolType typeIfBoth) private Func<CSharpTokenType> CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char second, CSharpTokenType typeIfBoth)
{ {
return () => return () =>
{ {
@ -513,7 +513,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}; };
} }
private Func<CSharpSymbolType> CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char option1, CSharpSymbolType typeIfOption1, char option2, CSharpSymbolType typeIfOption2) private Func<CSharpTokenType> CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char option1, CSharpTokenType typeIfOption1, char option2, CSharpTokenType typeIfOption2)
{ {
return () => return () =>
{ {
@ -550,14 +550,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral( RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */))); new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
} }
return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.StringLiteral)); return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.StringLiteral));
} }
private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', CSharpSymbolType.CharacterLiteral); private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', CSharpTokenType.CharacterLiteral);
private StateResult QuotedStringLiteral() => QuotedLiteral('\"', CSharpSymbolType.StringLiteral); private StateResult QuotedStringLiteral() => QuotedLiteral('\"', CSharpTokenType.StringLiteral);
private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType) private StateResult QuotedLiteral(char quote, CSharpTokenType literalType)
{ {
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c)); TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\') if (CurrentCharacter == '\\')
@ -581,7 +581,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); // No-op if at EOF TakeCurrent(); // No-op if at EOF
} }
return Transition(CSharpTokenizerState.Data, EndSymbol(literalType)); return Transition(CSharpTokenizerState.Data, EndToken(literalType));
} }
// CSharp Spec §2.3.2 // CSharp Spec §2.3.2
@ -594,7 +594,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated( RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */))); new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Comment)); return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment));
} }
if (CurrentCharacter == '*') if (CurrentCharacter == '*')
{ {
@ -602,7 +602,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (CurrentCharacter == '/') if (CurrentCharacter == '/')
{ {
TakeCurrent(); TakeCurrent();
return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Comment)); return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment));
} }
} }
return Stay(); return Stay();
@ -612,7 +612,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult SingleLineComment() private StateResult SingleLineComment()
{ {
TakeUntil(c => ParserHelpers.IsNewLine(c)); TakeUntil(c => ParserHelpers.IsNewLine(c));
return Stay(EndSymbol(CSharpSymbolType.Comment)); return Stay(EndToken(CSharpTokenType.Comment));
} }
// CSharp Spec §2.4.4 // CSharp Spec §2.4.4
@ -632,7 +632,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeUntil(c => !IsHexDigit(c)); TakeUntil(c => !IsHexDigit(c));
TakeIntegerSuffix(); TakeIntegerSuffix();
return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral)); return Stay(EndToken(CSharpTokenType.IntegerLiteral));
} }
private StateResult DecimalLiteral() private StateResult DecimalLiteral()
@ -650,7 +650,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else else
{ {
TakeIntegerSuffix(); TakeIntegerSuffix();
return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral)); return Stay(EndToken(CSharpTokenType.IntegerLiteral));
} }
} }
@ -669,7 +669,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); TakeCurrent();
} }
return Stay(EndSymbol(CSharpSymbolType.RealLiteral)); return Stay(EndToken(CSharpTokenType.RealLiteral));
} }
// CSharp Spec §2.4.4.3 // CSharp Spec §2.4.4.3
@ -708,27 +708,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Debug.Assert(IsIdentifierStart(CurrentCharacter)); Debug.Assert(IsIdentifierStart(CurrentCharacter));
TakeCurrent(); TakeCurrent();
TakeUntil(c => !IsIdentifierPart(c)); TakeUntil(c => !IsIdentifierPart(c));
CSharpSymbol symbol = null; CSharpToken token = null;
if (HaveContent) if (HaveContent)
{ {
CSharpKeyword keyword; CSharpKeyword keyword;
var type = CSharpSymbolType.Identifier; var type = CSharpTokenType.Identifier;
var symbolContent = Buffer.ToString(); var tokenContent = Buffer.ToString();
if (_keywords.TryGetValue(symbolContent, out keyword)) if (_keywords.TryGetValue(tokenContent, out keyword))
{ {
type = CSharpSymbolType.Keyword; type = CSharpTokenType.Keyword;
} }
symbol = new CSharpSymbol(symbolContent, type) token = new CSharpToken(tokenContent, type)
{ {
Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null, Keyword = type == CSharpTokenType.Keyword ? (CSharpKeyword?)keyword : null,
}; };
Buffer.Clear(); Buffer.Clear();
CurrentErrors.Clear(); CurrentErrors.Clear();
} }
return Stay(symbol); return Stay(token);
} }
private StateResult Transition(CSharpTokenizerState state) private StateResult Transition(CSharpTokenizerState state)
@ -736,7 +736,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null); return Transition((int)state, result: null);
} }
private StateResult Transition(CSharpTokenizerState state, CSharpSymbol result) private StateResult Transition(CSharpTokenizerState state, CSharpToken result)
{ {
return Transition((int)state, result); return Transition((int)state, result);
} }
@ -793,7 +793,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition, EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition,
RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody, RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody,
StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody, StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody,
AtSymbolAfterRazorCommentBody = RazorCommentTokenizerState.AtSymbolAfterRazorCommentBody, AtTokenAfterRazorCommentBody = RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
} }
} }
} }

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs
View File

@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class CodeBlockEditHandler : SpanEditHandler internal class CodeBlockEditHandler : SpanEditHandler
{ {
public CodeBlockEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer) : base(tokenizer) public CodeBlockEditHandler(Func<string, IEnumerable<IToken>> tokenizer) : base(tokenizer)
{ {
} }

26
src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs
View File

@ -21,7 +21,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
_visitedFirstTokenStart = true; _visitedFirstTokenStart = true;
} }
else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpSymbolType.NewLine) else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpTokenType.NewLine)
{ {
_visitedFirstTokenLineEnd = true; _visitedFirstTokenLineEnd = true;
} }
@ -29,34 +29,34 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result; return result;
} }
public override CSharpSymbol NextSymbol() public override CSharpToken NextToken()
{ {
// Post-Condition: Buffer should be empty at the start of Next() // Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
StartSymbol(); StartToken();
if (EndOfFile || (_visitedFirstTokenStart && _visitedFirstTokenLineEnd)) if (EndOfFile || (_visitedFirstTokenStart && _visitedFirstTokenLineEnd))
{ {
return null; return null;
} }
var symbol = Turn(); var token = Turn();
// Post-Condition: Buffer should be empty at the end of Next() // Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
return symbol; return token;
} }
private bool IsValidTokenType(CSharpSymbolType type) private bool IsValidTokenType(CSharpTokenType type)
{ {
return type != CSharpSymbolType.WhiteSpace && return type != CSharpTokenType.WhiteSpace &&
type != CSharpSymbolType.NewLine && type != CSharpTokenType.NewLine &&
type != CSharpSymbolType.Comment && type != CSharpTokenType.Comment &&
type != CSharpSymbolType.RazorComment && type != CSharpTokenType.RazorComment &&
type != CSharpSymbolType.RazorCommentStar && type != CSharpTokenType.RazorCommentStar &&
type != CSharpSymbolType.RazorCommentTransition && type != CSharpTokenType.RazorCommentTransition &&
type != CSharpSymbolType.Transition; type != CSharpTokenType.Transition;
} }
} }
} }

22
src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs
View File

@ -24,33 +24,33 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result; return result;
} }
public override HtmlSymbol NextSymbol() public override HtmlToken NextToken()
{ {
// Post-Condition: Buffer should be empty at the start of Next() // Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
StartSymbol(); StartToken();
if (EndOfFile || _visitedFirstTokenStart) if (EndOfFile || _visitedFirstTokenStart)
{ {
return null; return null;
} }
var symbol = Turn(); var token = Turn();
// Post-Condition: Buffer should be empty at the end of Next() // Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
return symbol; return token;
} }
private bool IsValidTokenType(HtmlSymbolType type) private bool IsValidTokenType(HtmlTokenType type)
{ {
return type != HtmlSymbolType.WhiteSpace && return type != HtmlTokenType.WhiteSpace &&
type != HtmlSymbolType.NewLine && type != HtmlTokenType.NewLine &&
type != HtmlSymbolType.RazorComment && type != HtmlTokenType.RazorComment &&
type != HtmlSymbolType.RazorCommentStar && type != HtmlTokenType.RazorCommentStar &&
type != HtmlSymbolType.RazorCommentTransition && type != HtmlTokenType.RazorCommentTransition &&
type != HtmlSymbolType.Transition; type != HtmlTokenType.Transition;
} }
} }
} }

116
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
View File

@ -6,7 +6,7 @@ using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class HtmlLanguageCharacteristics : LanguageCharacteristics<HtmlTokenizer, HtmlSymbol, HtmlSymbolType> internal class HtmlLanguageCharacteristics : LanguageCharacteristics<HtmlTokenizer, HtmlToken, HtmlTokenType>
{ {
private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics(); private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics();
@ -19,50 +19,50 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _instance; } get { return _instance; }
} }
public override string GetSample(HtmlSymbolType type) public override string GetSample(HtmlTokenType type)
{ {
switch (type) switch (type)
{ {
case HtmlSymbolType.Text: case HtmlTokenType.Text:
return Resources.HtmlSymbol_Text; return Resources.HtmlToken_Text;
case HtmlSymbolType.WhiteSpace: case HtmlTokenType.WhiteSpace:
return Resources.HtmlSymbol_WhiteSpace; return Resources.HtmlToken_WhiteSpace;
case HtmlSymbolType.NewLine: case HtmlTokenType.NewLine:
return Resources.HtmlSymbol_NewLine; return Resources.HtmlToken_NewLine;
case HtmlSymbolType.OpenAngle: case HtmlTokenType.OpenAngle:
return "<"; return "<";
case HtmlSymbolType.Bang: case HtmlTokenType.Bang:
return "!"; return "!";
case HtmlSymbolType.ForwardSlash: case HtmlTokenType.ForwardSlash:
return "/"; return "/";
case HtmlSymbolType.QuestionMark: case HtmlTokenType.QuestionMark:
return "?"; return "?";
case HtmlSymbolType.DoubleHyphen: case HtmlTokenType.DoubleHyphen:
return "--"; return "--";
case HtmlSymbolType.LeftBracket: case HtmlTokenType.LeftBracket:
return "["; return "[";
case HtmlSymbolType.CloseAngle: case HtmlTokenType.CloseAngle:
return ">"; return ">";
case HtmlSymbolType.RightBracket: case HtmlTokenType.RightBracket:
return "]"; return "]";
case HtmlSymbolType.Equals: case HtmlTokenType.Equals:
return "="; return "=";
case HtmlSymbolType.DoubleQuote: case HtmlTokenType.DoubleQuote:
return "\""; return "\"";
case HtmlSymbolType.SingleQuote: case HtmlTokenType.SingleQuote:
return "'"; return "'";
case HtmlSymbolType.Transition: case HtmlTokenType.Transition:
return "@"; return "@";
case HtmlSymbolType.Colon: case HtmlTokenType.Colon:
return ":"; return ":";
case HtmlSymbolType.RazorComment: case HtmlTokenType.RazorComment:
return Resources.HtmlSymbol_RazorComment; return Resources.HtmlToken_RazorComment;
case HtmlSymbolType.RazorCommentStar: case HtmlTokenType.RazorCommentStar:
return "*"; return "*";
case HtmlSymbolType.RazorCommentTransition: case HtmlTokenType.RazorCommentTransition:
return "@"; return "@";
default: default:
return Resources.Symbol_Unknown; return Resources.Token_Unknown;
} }
} }
@ -71,57 +71,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source); return new HtmlTokenizer(source);
} }
public override HtmlSymbolType FlipBracket(HtmlSymbolType bracket) public override HtmlTokenType FlipBracket(HtmlTokenType bracket)
{ {
switch (bracket) switch (bracket)
{ {
case HtmlSymbolType.LeftBracket: case HtmlTokenType.LeftBracket:
return HtmlSymbolType.RightBracket; return HtmlTokenType.RightBracket;
case HtmlSymbolType.OpenAngle: case HtmlTokenType.OpenAngle:
return HtmlSymbolType.CloseAngle; return HtmlTokenType.CloseAngle;
case HtmlSymbolType.RightBracket: case HtmlTokenType.RightBracket:
return HtmlSymbolType.LeftBracket; return HtmlTokenType.LeftBracket;
case HtmlSymbolType.CloseAngle: case HtmlTokenType.CloseAngle:
return HtmlSymbolType.OpenAngle; return HtmlTokenType.OpenAngle;
default: default:
Debug.Fail("FlipBracket must be called with a bracket character"); Debug.Fail("FlipBracket must be called with a bracket character");
return HtmlSymbolType.Unknown; return HtmlTokenType.Unknown;
} }
} }
public override HtmlSymbol CreateMarkerSymbol() public override HtmlToken CreateMarkerToken()
{ {
return new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown); return new HtmlToken(string.Empty, HtmlTokenType.Unknown);
} }
public override HtmlSymbolType GetKnownSymbolType(KnownSymbolType type) public override HtmlTokenType GetKnownTokenType(KnownTokenType type)
{ {
switch (type) switch (type)
{ {
case KnownSymbolType.CommentStart: case KnownTokenType.CommentStart:
return HtmlSymbolType.RazorCommentTransition; return HtmlTokenType.RazorCommentTransition;
case KnownSymbolType.CommentStar: case KnownTokenType.CommentStar:
return HtmlSymbolType.RazorCommentStar; return HtmlTokenType.RazorCommentStar;
case KnownSymbolType.CommentBody: case KnownTokenType.CommentBody:
return HtmlSymbolType.RazorComment; return HtmlTokenType.RazorComment;
case KnownSymbolType.Identifier: case KnownTokenType.Identifier:
return HtmlSymbolType.Text; return HtmlTokenType.Text;
case KnownSymbolType.Keyword: case KnownTokenType.Keyword:
return HtmlSymbolType.Text; return HtmlTokenType.Text;
case KnownSymbolType.NewLine: case KnownTokenType.NewLine:
return HtmlSymbolType.NewLine; return HtmlTokenType.NewLine;
case KnownSymbolType.Transition: case KnownTokenType.Transition:
return HtmlSymbolType.Transition; return HtmlTokenType.Transition;
case KnownSymbolType.WhiteSpace: case KnownTokenType.WhiteSpace:
return HtmlSymbolType.WhiteSpace; return HtmlTokenType.WhiteSpace;
default: default:
return HtmlSymbolType.Unknown; return HtmlTokenType.Unknown;
} }
} }
protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList<RazorDiagnostic> errors) protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new HtmlSymbol(content, type, errors); return new HtmlToken(content, type, errors);
} }
} }
} }

574
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
View File

File diff suppressed because it is too large Load Diff

10
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbol.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlToken.cs
View File

@ -6,11 +6,11 @@ using System.Collections.Generic;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class HtmlSymbol : SymbolBase<HtmlSymbolType> internal class HtmlToken : TokenBase<HtmlTokenType>
{ {
internal static readonly HtmlSymbol Hyphen = new HtmlSymbol("-", HtmlSymbolType.Text); internal static readonly HtmlToken Hyphen = new HtmlToken("-", HtmlTokenType.Text);
public HtmlSymbol(string content, HtmlSymbolType type) public HtmlToken(string content, HtmlTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray) : base(content, type, RazorDiagnostic.EmptyArray)
{ {
if (content == null) if (content == null)
@ -19,9 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public HtmlSymbol( public HtmlToken(
string content, string content,
HtmlSymbolType type, HtmlTokenType type,
IReadOnlyList<RazorDiagnostic> errors) IReadOnlyList<RazorDiagnostic> errors)
: base(content, type, errors) : base(content, type, errors)
{ {

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbolType.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenType.cs
View File

@ -6,7 +6,7 @@ using System;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
[Flags] [Flags]
internal enum HtmlSymbolType internal enum HtmlTokenType
{ {
Unknown, Unknown,
Text, // Text which isn't one of the below Text, // Text which isn't one of the below

116
src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs
View File

@ -7,7 +7,7 @@ using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization // Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization
internal class HtmlTokenizer : Tokenizer<HtmlSymbol, HtmlSymbolType> internal class HtmlTokenizer : Tokenizer<HtmlToken, HtmlTokenType>
{ {
private const char TransitionChar = '@'; private const char TransitionChar = '@';
@ -21,24 +21,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new HtmlTokenizerState? CurrentState => (HtmlTokenizerState?)base.CurrentState; private new HtmlTokenizerState? CurrentState => (HtmlTokenizerState?)base.CurrentState;
public override HtmlSymbolType RazorCommentType public override HtmlTokenType RazorCommentType
{ {
get { return HtmlSymbolType.RazorComment; } get { return HtmlTokenType.RazorComment; }
} }
public override HtmlSymbolType RazorCommentTransitionType public override HtmlTokenType RazorCommentTransitionType
{ {
get { return HtmlSymbolType.RazorCommentTransition; } get { return HtmlTokenType.RazorCommentTransition; }
} }
public override HtmlSymbolType RazorCommentStarType public override HtmlTokenType RazorCommentStarType
{ {
get { return HtmlSymbolType.RazorCommentStar; } get { return HtmlTokenType.RazorCommentStar; }
} }
protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList<RazorDiagnostic> errors) protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList<RazorDiagnostic> errors)
{ {
return new HtmlSymbol(content, type, errors); return new HtmlToken(content, type, errors);
} }
protected override StateResult Dispatch() protected override StateResult Dispatch()
@ -57,8 +57,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return RazorCommentBody(); return RazorCommentBody();
case HtmlTokenizerState.StarAfterRazorCommentBody: case HtmlTokenizerState.StarAfterRazorCommentBody:
return StarAfterRazorCommentBody(); return StarAfterRazorCommentBody();
case HtmlTokenizerState.AtSymbolAfterRazorCommentBody: case HtmlTokenizerState.AtTokenAfterRazorCommentBody:
return AtSymbolAfterRazorCommentBody(); return AtTokenAfterRazorCommentBody();
default: default:
Debug.Fail("Invalid TokenizerState"); Debug.Fail("Invalid TokenizerState");
return default(StateResult); return default(StateResult);
@ -66,35 +66,35 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Optimize memory allocation by returning constants for the most frequent cases // Optimize memory allocation by returning constants for the most frequent cases
protected override string GetSymbolContent(HtmlSymbolType type) protected override string GetTokenContent(HtmlTokenType type)
{ {
var symbolLength = Buffer.Length; var tokenLength = Buffer.Length;
if (symbolLength == 1) if (tokenLength == 1)
{ {
switch (type) switch (type)
{ {
case HtmlSymbolType.OpenAngle: case HtmlTokenType.OpenAngle:
return "<"; return "<";
case HtmlSymbolType.Bang: case HtmlTokenType.Bang:
return "!"; return "!";
case HtmlSymbolType.ForwardSlash: case HtmlTokenType.ForwardSlash:
return "/"; return "/";
case HtmlSymbolType.QuestionMark: case HtmlTokenType.QuestionMark:
return "?"; return "?";
case HtmlSymbolType.LeftBracket: case HtmlTokenType.LeftBracket:
return "["; return "[";
case HtmlSymbolType.CloseAngle: case HtmlTokenType.CloseAngle:
return ">"; return ">";
case HtmlSymbolType.RightBracket: case HtmlTokenType.RightBracket:
return "]"; return "]";
case HtmlSymbolType.Equals: case HtmlTokenType.Equals:
return "="; return "=";
case HtmlSymbolType.DoubleQuote: case HtmlTokenType.DoubleQuote:
return "\""; return "\"";
case HtmlSymbolType.SingleQuote: case HtmlTokenType.SingleQuote:
return "'"; return "'";
case HtmlSymbolType.WhiteSpace: case HtmlTokenType.WhiteSpace:
if (Buffer[0] == ' ') if (Buffer[0] == ' ')
{ {
return " "; return " ";
@ -104,7 +104,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t"; return "\t";
} }
break; break;
case HtmlSymbolType.NewLine: case HtmlTokenType.NewLine:
if (Buffer[0] == '\n') if (Buffer[0] == '\n')
{ {
return "\n"; return "\n";
@ -113,12 +113,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
if (symbolLength == 2 && type == HtmlSymbolType.NewLine) if (tokenLength == 2 && type == HtmlTokenType.NewLine)
{ {
return "\r\n"; return "\r\n";
} }
return base.GetSymbolContent(type); return base.GetTokenContent(type);
} }
// http://dev.w3.org/html5/spec/Overview.html#data-state // http://dev.w3.org/html5/spec/Overview.html#data-state
@ -139,21 +139,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return Transition( return Transition(
HtmlTokenizerState.AfterRazorCommentTransition, HtmlTokenizerState.AfterRazorCommentTransition,
EndSymbol(HtmlSymbolType.RazorCommentTransition)); EndToken(HtmlTokenType.RazorCommentTransition));
} }
else if (CurrentCharacter == '@') else if (CurrentCharacter == '@')
{ {
// Could be escaped comment transition // Could be escaped comment transition
return Transition( return Transition(
HtmlTokenizerState.EscapedRazorCommentTransition, HtmlTokenizerState.EscapedRazorCommentTransition,
EndSymbol(HtmlSymbolType.Transition)); EndToken(HtmlTokenType.Transition));
} }
return Stay(EndSymbol(HtmlSymbolType.Transition)); return Stay(EndToken(HtmlTokenType.Transition));
} }
else if (AtSymbol()) else if (AtToken())
{ {
return Stay(Symbol()); return Stay(Token());
} }
else else
{ {
@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult EscapedRazorCommentTransition() private StateResult EscapedRazorCommentTransition()
{ {
TakeCurrent(); TakeCurrent();
return Transition(HtmlTokenizerState.Data, EndSymbol(HtmlSymbolType.Transition)); return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Transition));
} }
private StateResult Text() private StateResult Text()
@ -172,7 +172,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var prev = '\0'; var prev = '\0';
while (!EndOfFile && while (!EndOfFile &&
!(ParserHelpers.IsWhitespace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) && !(ParserHelpers.IsWhitespace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) &&
!AtSymbol()) !AtToken())
{ {
prev = CurrentCharacter; prev = CurrentCharacter;
TakeCurrent(); TakeCurrent();
@ -190,56 +190,56 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Output the Text token and return to the Data state to tokenize the next character (if there is one) // Output the Text token and return to the Data state to tokenize the next character (if there is one)
return Transition(HtmlTokenizerState.Data, EndSymbol(HtmlSymbolType.Text)); return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Text));
} }
private HtmlSymbol Symbol() private HtmlToken Token()
{ {
Debug.Assert(AtSymbol()); Debug.Assert(AtToken());
var sym = CurrentCharacter; var sym = CurrentCharacter;
TakeCurrent(); TakeCurrent();
switch (sym) switch (sym)
{ {
case '<': case '<':
return EndSymbol(HtmlSymbolType.OpenAngle); return EndToken(HtmlTokenType.OpenAngle);
case '!': case '!':
return EndSymbol(HtmlSymbolType.Bang); return EndToken(HtmlTokenType.Bang);
case '/': case '/':
return EndSymbol(HtmlSymbolType.ForwardSlash); return EndToken(HtmlTokenType.ForwardSlash);
case '?': case '?':
return EndSymbol(HtmlSymbolType.QuestionMark); return EndToken(HtmlTokenType.QuestionMark);
case '[': case '[':
return EndSymbol(HtmlSymbolType.LeftBracket); return EndToken(HtmlTokenType.LeftBracket);
case '>': case '>':
return EndSymbol(HtmlSymbolType.CloseAngle); return EndToken(HtmlTokenType.CloseAngle);
case ']': case ']':
return EndSymbol(HtmlSymbolType.RightBracket); return EndToken(HtmlTokenType.RightBracket);
case '=': case '=':
return EndSymbol(HtmlSymbolType.Equals); return EndToken(HtmlTokenType.Equals);
case '"': case '"':
return EndSymbol(HtmlSymbolType.DoubleQuote); return EndToken(HtmlTokenType.DoubleQuote);
case '\'': case '\'':
return EndSymbol(HtmlSymbolType.SingleQuote); return EndToken(HtmlTokenType.SingleQuote);
case '-': case '-':
Debug.Assert(CurrentCharacter == '-'); Debug.Assert(CurrentCharacter == '-');
TakeCurrent(); TakeCurrent();
return EndSymbol(HtmlSymbolType.DoubleHyphen); return EndToken(HtmlTokenType.DoubleHyphen);
default: default:
Debug.Fail("Unexpected symbol!"); Debug.Fail("Unexpected token!");
return EndSymbol(HtmlSymbolType.Unknown); return EndToken(HtmlTokenType.Unknown);
} }
} }
private HtmlSymbol Whitespace() private HtmlToken Whitespace()
{ {
while (ParserHelpers.IsWhitespace(CurrentCharacter)) while (ParserHelpers.IsWhitespace(CurrentCharacter))
{ {
TakeCurrent(); TakeCurrent();
} }
return EndSymbol(HtmlSymbolType.WhiteSpace); return EndToken(HtmlTokenType.WhiteSpace);
} }
private HtmlSymbol Newline() private HtmlToken Newline()
{ {
Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter)); Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter));
// CSharp Spec §2.3.1 // CSharp Spec §2.3.1
@ -249,10 +249,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TakeCurrent(); TakeCurrent();
} }
return EndSymbol(HtmlSymbolType.NewLine); return EndToken(HtmlTokenType.NewLine);
} }
private bool AtSymbol() private bool AtToken()
{ {
return CurrentCharacter == '<' || return CurrentCharacter == '<' ||
CurrentCharacter == '<' || CurrentCharacter == '<' ||
@ -274,7 +274,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null); return Transition((int)state, result: null);
} }
private StateResult Transition(HtmlTokenizerState state, HtmlSymbol result) private StateResult Transition(HtmlTokenizerState state, HtmlToken result)
{ {
return Transition((int)state, result); return Transition((int)state, result);
} }
@ -289,7 +289,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition, EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition,
RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody, RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody,
StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody, StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody,
AtSymbolAfterRazorCommentBody = RazorCommentTokenizerState.AtSymbolAfterRazorCommentBody, AtTokenAfterRazorCommentBody = RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
} }
} }
} }

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/ISymbol.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/IToken.cs
View File

@ -3,7 +3,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal interface ISymbol internal interface IToken
{ {
Span Parent { get; set; } Span Parent { get; set; }

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs
View File

@ -5,6 +5,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal interface ITokenizer internal interface ITokenizer
{ {
ISymbol NextSymbol(); IToken NextToken();
} }
} }

52
src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs
View File

@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private readonly ISet<string> _keywords; private readonly ISet<string> _keywords;
private readonly IReadOnlyCollection<string> _readOnlyKeywords; private readonly IReadOnlyCollection<string> _readOnlyKeywords;
public ImplicitExpressionEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer, ISet<string> keywords, bool acceptTrailingDot) public ImplicitExpressionEditHandler(Func<string, IEnumerable<IToken>> tokenizer, ISet<string> keywords, bool acceptTrailingDot)
: base(tokenizer) : base(tokenizer)
{ {
_keywords = keywords ?? new HashSet<string>(); _keywords = keywords ?? new HashSet<string>();
@ -170,44 +170,44 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false; return false;
} }
for (var i = 0; i < target.Symbols.Count; i++) for (var i = 0; i < target.Tokens.Count; i++)
{ {
var symbol = target.Symbols[i] as CSharpSymbol; var token = target.Tokens[i] as CSharpToken;
if (symbol == null) if (token == null)
{ {
break; break;
} }
var symbolStartIndex = symbol.Start.AbsoluteIndex; var tokenStartIndex = token.Start.AbsoluteIndex;
var symbolEndIndex = symbolStartIndex + symbol.Content.Length; var tokenEndIndex = tokenStartIndex + token.Content.Length;
// We're looking for the first symbol that contains the SourceChange. // We're looking for the first token that contains the SourceChange.
if (symbolEndIndex > change.Span.AbsoluteIndex) if (tokenEndIndex > change.Span.AbsoluteIndex)
{ {
if (symbolEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && symbol.Type == CSharpSymbolType.Identifier) if (tokenEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && token.Type == CSharpTokenType.Identifier)
{ {
// The symbol we're changing happens to be an identifier. Need to check if its transformed state is also one. // The token we're changing happens to be an identifier. Need to check if its transformed state is also one.
// We do this transformation logic to capture the case that the new text change happens to not be an identifier; // We do this transformation logic to capture the case that the new text change happens to not be an identifier;
// i.e. "5". Alone, it's numeric, within an identifier it's classified as identifier. // i.e. "5". Alone, it's numeric, within an identifier it's classified as identifier.
var transformedContent = change.GetEditedContent(symbol.Content, change.Span.AbsoluteIndex - symbolStartIndex); var transformedContent = change.GetEditedContent(token.Content, change.Span.AbsoluteIndex - tokenStartIndex);
var newSymbols = Tokenizer(transformedContent); var newTokens = Tokenizer(transformedContent);
if (newSymbols.Count() != 1) if (newTokens.Count() != 1)
{ {
// The transformed content resulted in more than one symbol; we can only replace a single identifier with // The transformed content resulted in more than one token; we can only replace a single identifier with
// another single identifier. // another single identifier.
break; break;
} }
var newSymbol = (CSharpSymbol)newSymbols.First(); var newToken = (CSharpToken)newTokens.First();
if (newSymbol.Type == CSharpSymbolType.Identifier) if (newToken.Type == CSharpTokenType.Identifier)
{ {
return true; return true;
} }
} }
// Change is touching a non-identifier symbol or spans multiple symbols. // Change is touching a non-identifier token or spans multiple tokens.
break; break;
} }
@ -241,7 +241,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var changeStart = change.Span.AbsoluteIndex; var changeStart = change.Span.AbsoluteIndex;
var changeLength = change.Span.Length; var changeLength = change.Span.Length;
var changeEnd = changeStart + changeLength; var changeEnd = changeStart + changeLength;
var tokens = target.Symbols.Cast<CSharpSymbol>().ToArray(); var tokens = target.Tokens.Cast<CSharpToken>().ToArray();
if (!IsInsideParenthesis(changeStart, tokens) || !IsInsideParenthesis(changeEnd, tokens)) if (!IsInsideParenthesis(changeStart, tokens) || !IsInsideParenthesis(changeEnd, tokens))
{ {
// Either the start or end of the delete does not fall inside of parenthesis, unacceptable inner deletion. // Either the start or end of the delete does not fall inside of parenthesis, unacceptable inner deletion.
@ -274,7 +274,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false; return false;
} }
var tokens = target.Symbols.Cast<CSharpSymbol>().ToArray(); var tokens = target.Tokens.Cast<CSharpToken>().ToArray();
if (IsInsideParenthesis(change.Span.AbsoluteIndex, tokens)) if (IsInsideParenthesis(change.Span.AbsoluteIndex, tokens))
{ {
return true; return true;
@ -284,7 +284,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Internal for testing // Internal for testing
internal static bool IsInsideParenthesis(int position, IReadOnlyList<CSharpSymbol> tokens) internal static bool IsInsideParenthesis(int position, IReadOnlyList<CSharpToken> tokens)
{ {
var balanceCount = 0; var balanceCount = 0;
var foundInsertionPoint = false; var foundInsertionPoint = false;
@ -322,7 +322,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Internal for testing // Internal for testing
internal static bool ContainsPosition(int position, CSharpSymbol currentToken) internal static bool ContainsPosition(int position, CSharpToken currentToken)
{ {
var tokenStart = currentToken.Start.AbsoluteIndex; var tokenStart = currentToken.Start.AbsoluteIndex;
if (tokenStart == position) if (tokenStart == position)
@ -342,14 +342,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Internal for testing // Internal for testing
internal static bool TryUpdateBalanceCount(CSharpSymbol token, ref int count) internal static bool TryUpdateBalanceCount(CSharpToken token, ref int count)
{ {
var updatedCount = count; var updatedCount = count;
if (token.Type == CSharpSymbolType.LeftParenthesis) if (token.Type == CSharpTokenType.LeftParenthesis)
{ {
updatedCount++; updatedCount++;
} }
else if (token.Type == CSharpSymbolType.RightParenthesis) else if (token.Type == CSharpTokenType.RightParenthesis)
{ {
if (updatedCount == 0) if (updatedCount == 0)
{ {
@ -358,7 +358,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
updatedCount--; updatedCount--;
} }
else if (token.Type == CSharpSymbolType.StringLiteral) else if (token.Type == CSharpTokenType.StringLiteral)
{ {
var content = token.Content; var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '"') if (content.Length > 0 && content[content.Length - 1] != '"')
@ -370,7 +370,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
} }
else if (token.Type == CSharpSymbolType.CharacterLiteral) else if (token.Type == CSharpTokenType.CharacterLiteral)
{ {
var content = token.Content; var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '\'') if (content.Length > 0 && content[content.Length - 1] != '\'')

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownSymbolType.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownTokenType.cs
View File

@ -3,7 +3,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal enum KnownSymbolType internal enum KnownTokenType
{ {
WhiteSpace, WhiteSpace,
NewLine, NewLine,

82
src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs
View File

@ -6,104 +6,104 @@ using System.Collections.Generic;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract class LanguageCharacteristics<TTokenizer, TSymbol, TSymbolType> internal abstract class LanguageCharacteristics<TTokenizer, TToken, TTokenType>
where TSymbolType : struct where TTokenType : struct
where TTokenizer : Tokenizer<TSymbol, TSymbolType> where TTokenizer : Tokenizer<TToken, TTokenType>
where TSymbol : SymbolBase<TSymbolType> where TToken : TokenBase<TTokenType>
{ {
public abstract string GetSample(TSymbolType type); public abstract string GetSample(TTokenType type);
public abstract TTokenizer CreateTokenizer(ITextDocument source); public abstract TTokenizer CreateTokenizer(ITextDocument source);
public abstract TSymbolType FlipBracket(TSymbolType bracket); public abstract TTokenType FlipBracket(TTokenType bracket);
public abstract TSymbol CreateMarkerSymbol(); public abstract TToken CreateMarkerToken();
public virtual IEnumerable<TSymbol> TokenizeString(string content) public virtual IEnumerable<TToken> TokenizeString(string content)
{ {
return TokenizeString(SourceLocation.Zero, content); return TokenizeString(SourceLocation.Zero, content);
} }
public virtual IEnumerable<TSymbol> TokenizeString(SourceLocation start, string input) public virtual IEnumerable<TToken> TokenizeString(SourceLocation start, string input)
{ {
using (var reader = new SeekableTextReader(input, start.FilePath)) using (var reader = new SeekableTextReader(input, start.FilePath))
{ {
var tok = CreateTokenizer(reader); var tok = CreateTokenizer(reader);
TSymbol sym; TToken token;
while ((sym = tok.NextSymbol()) != null) while ((token = tok.NextToken()) != null)
{ {
yield return sym; yield return token;
} }
} }
} }
public virtual bool IsWhiteSpace(TSymbol symbol) public virtual bool IsWhiteSpace(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.WhiteSpace); return IsKnownTokenType(token, KnownTokenType.WhiteSpace);
} }
public virtual bool IsNewLine(TSymbol symbol) public virtual bool IsNewLine(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.NewLine); return IsKnownTokenType(token, KnownTokenType.NewLine);
} }
public virtual bool IsIdentifier(TSymbol symbol) public virtual bool IsIdentifier(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.Identifier); return IsKnownTokenType(token, KnownTokenType.Identifier);
} }
public virtual bool IsKeyword(TSymbol symbol) public virtual bool IsKeyword(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.Keyword); return IsKnownTokenType(token, KnownTokenType.Keyword);
} }
public virtual bool IsTransition(TSymbol symbol) public virtual bool IsTransition(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.Transition); return IsKnownTokenType(token, KnownTokenType.Transition);
} }
public virtual bool IsCommentStart(TSymbol symbol) public virtual bool IsCommentStart(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.CommentStart); return IsKnownTokenType(token, KnownTokenType.CommentStart);
} }
public virtual bool IsCommentStar(TSymbol symbol) public virtual bool IsCommentStar(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.CommentStar); return IsKnownTokenType(token, KnownTokenType.CommentStar);
} }
public virtual bool IsCommentBody(TSymbol symbol) public virtual bool IsCommentBody(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.CommentBody); return IsKnownTokenType(token, KnownTokenType.CommentBody);
} }
public virtual bool IsUnknown(TSymbol symbol) public virtual bool IsUnknown(TToken token)
{ {
return IsKnownSymbolType(symbol, KnownSymbolType.Unknown); return IsKnownTokenType(token, KnownTokenType.Unknown);
} }
public virtual bool IsKnownSymbolType(TSymbol symbol, KnownSymbolType type) public virtual bool IsKnownTokenType(TToken token, KnownTokenType type)
{ {
return symbol != null && Equals(symbol.Type, GetKnownSymbolType(type)); return token != null && Equals(token.Type, GetKnownTokenType(type));
} }
public virtual Tuple<TSymbol, TSymbol> SplitSymbol(TSymbol symbol, int splitAt, TSymbolType leftType) public virtual Tuple<TToken, TToken> SplitToken(TToken token, int splitAt, TTokenType leftType)
{ {
var left = CreateSymbol(symbol.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray); var left = CreateToken(token.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray);
TSymbol right = null; TToken right = null;
if (splitAt < symbol.Content.Length) if (splitAt < token.Content.Length)
{ {
right = CreateSymbol(symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors); right = CreateToken(token.Content.Substring(splitAt), token.Type, token.Errors);
} }
return Tuple.Create(left, right); return Tuple.Create(left, right);
} }
public abstract TSymbolType GetKnownSymbolType(KnownSymbolType type); public abstract TTokenType GetKnownTokenType(KnownTokenType type);
public virtual bool KnowsSymbolType(KnownSymbolType type) public virtual bool KnowsTokenType(KnownTokenType type)
{ {
return type == KnownSymbolType.Unknown || !Equals(GetKnownSymbolType(type), GetKnownSymbolType(KnownSymbolType.Unknown)); return type == KnownTokenType.Unknown || !Equals(GetKnownTokenType(type), GetKnownTokenType(KnownTokenType.Unknown));
} }
protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList<RazorDiagnostic> errors); protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList<RazorDiagnostic> errors);
} }
} }

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs
View File

@ -67,7 +67,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
builder.Kind = SpanKindInternal.Markup; builder.Kind = SpanKindInternal.Markup;
builder.ChunkGenerator = new MarkupChunkGenerator(); builder.ChunkGenerator = new MarkupChunkGenerator();
foreach (ISymbol sym in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content)) foreach (IToken sym in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content))
{ {
builder.Accept(sym); builder.Accept(sym);
} }

28
src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs
View File

@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public ISpanChunkGenerator ChunkGenerator { get; private set; } public ISpanChunkGenerator ChunkGenerator { get; private set; }
public SpanKindInternal Kind { get; private set; } public SpanKindInternal Kind { get; private set; }
public IReadOnlyList<ISymbol> Symbols { get; private set; } public IReadOnlyList<IToken> Tokens { get; private set; }
// Allow test code to re-link spans // Allow test code to re-link spans
public Span Previous { get; internal set; } public Span Previous { get; internal set; }
@ -42,9 +42,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var length = 0; var length = 0;
if (_content == null) if (_content == null)
{ {
for (var i = 0; i < Symbols.Count; i++) for (var i = 0; i < Tokens.Count; i++)
{ {
length += Symbols[i].Content.Length; length += Tokens[i].Content.Length;
} }
} }
else else
@ -67,19 +67,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
if (_content == null) if (_content == null)
{ {
var symbolCount = Symbols.Count; var tokenCount = Tokens.Count;
if (symbolCount == 1) if (tokenCount == 1)
{ {
// Perf: no StringBuilder allocation if not necessary // Perf: no StringBuilder allocation if not necessary
_content = Symbols[0].Content; _content = Tokens[0].Content;
} }
else else
{ {
var builder = new StringBuilder(); var builder = new StringBuilder();
for (var i = 0; i < symbolCount; i++) for (var i = 0; i < tokenCount; i++)
{ {
var symbol = Symbols[i]; var token = Tokens[i];
builder.Append(symbol.Content); builder.Append(token.Content);
} }
_content = builder.ToString(); _content = builder.ToString();
@ -93,11 +93,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void ReplaceWith(SpanBuilder builder) public void ReplaceWith(SpanBuilder builder)
{ {
Kind = builder.Kind; Kind = builder.Kind;
Symbols = builder.Symbols; Tokens = builder.Tokens;
for (var i = 0; i <Symbols.Count; i++) for (var i = 0; i <Tokens.Count; i++)
{ {
Symbols[i].Parent = this; Tokens[i].Parent = this;
} }
EditHandler = builder.EditHandler; EditHandler = builder.EditHandler;
@ -122,7 +122,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
builder.Append("> Gen: <"); builder.Append("> Gen: <");
builder.Append(ChunkGenerator.ToString()); builder.Append(ChunkGenerator.ToString());
builder.Append("> {"); builder.Append("> {");
builder.Append(string.Join(";", Symbols.GroupBy(sym => sym.GetType()).Select(grp => string.Concat(grp.Key.Name, ":", grp.Count())))); builder.Append(string.Join(";", Tokens.GroupBy(sym => sym.GetType()).Select(grp => string.Concat(grp.Key.Name, ":", grp.Count()))));
builder.Append("}"); builder.Append("}");
return builder.ToString(); return builder.ToString();
} }
@ -166,7 +166,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind.Equals(other.Kind) && Kind.Equals(other.Kind) &&
EditHandler.Equals(other.EditHandler) && EditHandler.Equals(other.EditHandler) &&
ChunkGenerator.Equals(other.ChunkGenerator) && ChunkGenerator.Equals(other.ChunkGenerator) &&
Symbols.SequenceEqual(other.Symbols); Tokens.SequenceEqual(other.Tokens);
} }
public override int GetHashCode() public override int GetHashCode()

36
src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs
View File

@ -10,7 +10,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal class SpanBuilder internal class SpanBuilder
{ {
private SourceLocation _start; private SourceLocation _start;
private List<ISymbol> _symbols; private List<IToken> _tokens;
private SourceLocationTracker _tracker; private SourceLocationTracker _tracker;
public SpanBuilder(Span original) public SpanBuilder(Span original)
@ -20,7 +20,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_start = original.Start; _start = original.Start;
ChunkGenerator = original.ChunkGenerator; ChunkGenerator = original.ChunkGenerator;
_symbols = new List<ISymbol>(original.Symbols); _tokens = new List<IToken>(original.Tokens);
_tracker = new SourceLocationTracker(original.Start); _tracker = new SourceLocationTracker(original.Start);
} }
@ -49,16 +49,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SpanKindInternal Kind { get; set; } public SpanKindInternal Kind { get; set; }
public IReadOnlyList<ISymbol> Symbols public IReadOnlyList<IToken> Tokens
{ {
get get
{ {
if (_symbols == null) if (_tokens == null)
{ {
_symbols = new List<ISymbol>(); _tokens = new List<IToken>();
} }
return _symbols; return _tokens;
} }
} }
@ -68,10 +68,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Need to potentially allocate a new list because Span.ReplaceWith takes ownership // Need to potentially allocate a new list because Span.ReplaceWith takes ownership
// of the original list. // of the original list.
_symbols = null; _tokens = null;
_symbols = new List<ISymbol>(); _tokens = new List<IToken>();
EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<ISymbol>()); EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<IToken>());
ChunkGenerator = SpanChunkGenerator.Null; ChunkGenerator = SpanChunkGenerator.Null;
Start = SourceLocation.Undefined; Start = SourceLocation.Undefined;
} }
@ -80,23 +80,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
var span = new Span(this); var span = new Span(this);
for (var i = 0; i < span.Symbols.Count; i++) for (var i = 0; i < span.Tokens.Count; i++)
{ {
var symbol = span.Symbols[i]; var token = span.Tokens[i];
symbol.Parent = span; token.Parent = span;
} }
return span; return span;
} }
public void ClearSymbols() public void ClearTokens()
{ {
_symbols?.Clear(); _tokens?.Clear();
} }
public void Accept(ISymbol symbol) public void Accept(IToken token)
{ {
if (symbol == null) if (token == null)
{ {
return; return;
} }
@ -106,8 +106,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
throw new InvalidOperationException("SpanBuilder must have a valid location"); throw new InvalidOperationException("SpanBuilder must have a valid location");
} }
_symbols.Add(symbol); _tokens.Add(token);
_tracker.UpdateLocation(symbol.Content); _tracker.UpdateLocation(token.Content);
} }
} }
} }

10
src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs
View File

@ -11,12 +11,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
private static readonly int TypeHashCode = typeof(SpanEditHandler).GetHashCode(); private static readonly int TypeHashCode = typeof(SpanEditHandler).GetHashCode();
public SpanEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer) public SpanEditHandler(Func<string, IEnumerable<IToken>> tokenizer)
: this(tokenizer, AcceptedCharactersInternal.Any) : this(tokenizer, AcceptedCharactersInternal.Any)
{ {
} }
public SpanEditHandler(Func<string, IEnumerable<ISymbol>> tokenizer, AcceptedCharactersInternal accepted) public SpanEditHandler(Func<string, IEnumerable<IToken>> tokenizer, AcceptedCharactersInternal accepted)
{ {
AcceptedCharacters = accepted; AcceptedCharacters = accepted;
Tokenizer = tokenizer; Tokenizer = tokenizer;
@ -24,9 +24,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public AcceptedCharactersInternal AcceptedCharacters { get; set; } public AcceptedCharactersInternal AcceptedCharacters { get; set; }
public Func<string, IEnumerable<ISymbol>> Tokenizer { get; set; } public Func<string, IEnumerable<IToken>> Tokenizer { get; set; }
public static SpanEditHandler CreateDefault(Func<string, IEnumerable<ISymbol>> tokenizer) public static SpanEditHandler CreateDefault(Func<string, IEnumerable<IToken>> tokenizer)
{ {
return new SpanEditHandler(tokenizer); return new SpanEditHandler(tokenizer);
} }
@ -69,7 +69,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
var newContent = change.GetEditedContent(target); var newContent = change.GetEditedContent(target);
var newSpan = new SpanBuilder(target); var newSpan = new SpanBuilder(target);
newSpan.ClearSymbols(); newSpan.ClearTokens();
foreach (var token in Tokenizer(newContent)) foreach (var token in Tokenizer(newContent))
{ {
newSpan.Accept(token); newSpan.Accept(token);

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/SyntaxTreeNode.cs
View File

@ -28,7 +28,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// <param name="node">The node to compare this node with</param> /// <param name="node">The node to compare this node with</param>
/// <returns> /// <returns>
/// true if the provided node has all the same content and metadata, though the specific quantity and type of /// true if the provided node has all the same content and metadata, though the specific quantity and type of
/// symbols may be different. /// tokens may be different.
/// </returns> /// </returns>
public abstract bool EquivalentTo(SyntaxTreeNode node); public abstract bool EquivalentTo(SyntaxTreeNode node);

138
src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs
View File

@ -42,8 +42,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// We skip the first child "<tagname" and take everything up to the ending portion of the tag ">" or "/>". // We skip the first child "<tagname" and take everything up to the ending portion of the tag ">" or "/>".
// The -2 accounts for both the start and end tags. If the tag does not have a valid structure then there's // The -2 accounts for both the start and end tags. If the tag does not have a valid structure then there's
// no end tag to ignore. // no end tag to ignore.
var symbolOffset = validStructure ? 2 : 1; var tokenOffset = validStructure ? 2 : 1;
var attributeChildren = tagBlock.Children.Skip(1).Take(tagBlock.Children.Count() - symbolOffset); var attributeChildren = tagBlock.Children.Skip(1).Take(tagBlock.Children.Count() - tokenOffset);
var processedBoundAttributeNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase); var processedBoundAttributeNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var child in attributeChildren) foreach (var child in attributeChildren)
@ -149,8 +149,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind = span.Kind Kind = span.Kind
}; };
// Will contain symbols that represent a single attribute value: <input| class="btn"| /> // Will contain tokens that represent a single attribute value: <input| class="btn"| />
var htmlSymbols = span.Symbols.OfType<HtmlSymbol>().ToArray(); var htmlTokens = span.Tokens.OfType<HtmlToken>().ToArray();
var capturedAttributeValueStart = false; var capturedAttributeValueStart = false;
var attributeValueStartLocation = span.Start; var attributeValueStartLocation = span.Start;
@ -158,53 +158,53 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// TryParseBlock() variation of attribute parsing. // TryParseBlock() variation of attribute parsing.
var attributeValueStyle = AttributeStructure.DoubleQuotes; var attributeValueStyle = AttributeStructure.DoubleQuotes;
// The symbolOffset is initialized to 0 to expect worst case: "class=". If a quote is found later on for // The tokenOffset is initialized to 0 to expect worst case: "class=". If a quote is found later on for
// the attribute value the symbolOffset is adjusted accordingly. // the attribute value the tokenOffset is adjusted accordingly.
var symbolOffset = 0; var tokenOffset = 0;
string name = null; string name = null;
// Iterate down through the symbols to find the name and the start of the value. // Iterate down through the tokens to find the name and the start of the value.
// We subtract the symbolOffset so we don't accept an ending quote of a span. // We subtract the tokenOffset so we don't accept an ending quote of a span.
for (var i = 0; i < htmlSymbols.Length - symbolOffset; i++) for (var i = 0; i < htmlTokens.Length - tokenOffset; i++)
{ {
var symbol = htmlSymbols[i]; var token = htmlTokens[i];
if (afterEquals) if (afterEquals)
{ {
// We've captured all leading whitespace, the attribute name, and an equals with an optional // We've captured all leading whitespace, the attribute name, and an equals with an optional
// quote/double quote. We're now at: " asp-for='|...'" or " asp-for=|..." // quote/double quote. We're now at: " asp-for='|...'" or " asp-for=|..."
// The goal here is to capture all symbols until the end of the attribute. Note this will not // The goal here is to capture all tokens until the end of the attribute. Note this will not
// consume an ending quote due to the symbolOffset. // consume an ending quote due to the tokenOffset.
// When symbols are accepted into SpanBuilders, their locations get altered to be offset by the // When tokens are accepted into SpanBuilders, their locations get altered to be offset by the
// parent which is why we need to mark our start location prior to adding the symbol. // parent which is why we need to mark our start location prior to adding the token.
// This is needed to know the location of the attribute value start within the document. // This is needed to know the location of the attribute value start within the document.
if (!capturedAttributeValueStart) if (!capturedAttributeValueStart)
{ {
capturedAttributeValueStart = true; capturedAttributeValueStart = true;
attributeValueStartLocation = symbol.Start; attributeValueStartLocation = token.Start;
} }
builder.Accept(symbol); builder.Accept(token);
} }
else if (name == null && HtmlMarkupParser.IsValidAttributeNameSymbol(symbol)) else if (name == null && HtmlMarkupParser.IsValidAttributeNameToken(token))
{ {
// We've captured all leading whitespace prior to the attribute name. // We've captured all leading whitespace prior to the attribute name.
// We're now at: " |asp-for='...'" or " |asp-for=..." // We're now at: " |asp-for='...'" or " |asp-for=..."
// The goal here is to capture the attribute name. // The goal here is to capture the attribute name.
var nameBuilder = new StringBuilder(); var nameBuilder = new StringBuilder();
// Move the indexer past the attribute name symbols. // Move the indexer past the attribute name tokens.
for (var j = i; j < htmlSymbols.Length; j++) for (var j = i; j < htmlTokens.Length; j++)
{ {
var nameSymbol = htmlSymbols[j]; var nameToken = htmlTokens[j];
if (!HtmlMarkupParser.IsValidAttributeNameSymbol(nameSymbol)) if (!HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
{ {
break; break;
} }
nameBuilder.Append(nameSymbol.Content); nameBuilder.Append(nameToken.Content);
i++; i++;
} }
@ -213,67 +213,67 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
name = nameBuilder.ToString(); name = nameBuilder.ToString();
attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, name); attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, name);
} }
else if (symbol.Type == HtmlSymbolType.Equals) else if (token.Type == HtmlTokenType.Equals)
{ {
// We've captured all leading whitespace and the attribute name. // We've captured all leading whitespace and the attribute name.
// We're now at: " asp-for|='...'" or " asp-for|=..." // We're now at: " asp-for|='...'" or " asp-for|=..."
// The goal here is to consume the equal sign and the optional single/double-quote. // The goal here is to consume the equal sign and the optional single/double-quote.
// The coming symbols will either be a quote or value (in the case that the value is unquoted). // The coming tokens will either be a quote or value (in the case that the value is unquoted).
SourceLocation symbolStartLocation; SourceLocation tokenStartLocation;
// Skip the whitespace preceding the start of the attribute value. // Skip the whitespace preceding the start of the attribute value.
do do
{ {
i++; // Start from the symbol after '='. i++; // Start from the token after '='.
} while (i < htmlSymbols.Length && } while (i < htmlTokens.Length &&
(htmlSymbols[i].Type == HtmlSymbolType.WhiteSpace || (htmlTokens[i].Type == HtmlTokenType.WhiteSpace ||
htmlSymbols[i].Type == HtmlSymbolType.NewLine)); htmlTokens[i].Type == HtmlTokenType.NewLine));
// Check for attribute start values, aka single or double quote // Check for attribute start values, aka single or double quote
if (i < htmlSymbols.Length && IsQuote(htmlSymbols[i])) if (i < htmlTokens.Length && IsQuote(htmlTokens[i]))
{ {
if (htmlSymbols[i].Type == HtmlSymbolType.SingleQuote) if (htmlTokens[i].Type == HtmlTokenType.SingleQuote)
{ {
attributeValueStyle = AttributeStructure.SingleQuotes; attributeValueStyle = AttributeStructure.SingleQuotes;
} }
symbolStartLocation = htmlSymbols[i].Start; tokenStartLocation = htmlTokens[i].Start;
// If there's a start quote then there must be an end quote to be valid, skip it. // If there's a start quote then there must be an end quote to be valid, skip it.
symbolOffset = 1; tokenOffset = 1;
} }
else else
{ {
// We are at the symbol after equals. Go back to equals to ensure we don't skip past that symbol. // We are at the token after equals. Go back to equals to ensure we don't skip past that token.
i--; i--;
symbolStartLocation = symbol.Start; tokenStartLocation = token.Start;
} }
attributeValueStartLocation = new SourceLocation( attributeValueStartLocation = new SourceLocation(
symbolStartLocation.FilePath, tokenStartLocation.FilePath,
symbolStartLocation.AbsoluteIndex + 1, tokenStartLocation.AbsoluteIndex + 1,
symbolStartLocation.LineIndex, tokenStartLocation.LineIndex,
symbolStartLocation.CharacterIndex + 1); tokenStartLocation.CharacterIndex + 1);
afterEquals = true; afterEquals = true;
} }
else if (symbol.Type == HtmlSymbolType.WhiteSpace) else if (token.Type == HtmlTokenType.WhiteSpace)
{ {
// We're at the start of the attribute, this branch may be hit on the first iterations of // We're at the start of the attribute, this branch may be hit on the first iterations of
// the loop since the parser separates attributes with their spaces included as symbols. // the loop since the parser separates attributes with their spaces included as tokens.
// We're at: "| asp-for='...'" or "| asp-for=..." // We're at: "| asp-for='...'" or "| asp-for=..."
// Note: This will not be hit even for situations like asp-for ="..." because the core Razor // Note: This will not be hit even for situations like asp-for ="..." because the core Razor
// parser currently does not know how to handle attributes in that format. This will be addressed // parser currently does not know how to handle attributes in that format. This will be addressed
// by https://github.com/aspnet/Razor/issues/123. // by https://github.com/aspnet/Razor/issues/123.
attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, symbol.Content); attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, token.Content);
} }
} }
// After all symbols have been added we need to set the builders start position so we do not indirectly // After all tokens have been added we need to set the builders start position so we do not indirectly
// modify the span's start location. // modify the span's start location.
builder.Start = attributeValueStartLocation; builder.Start = attributeValueStartLocation;
@ -341,14 +341,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return TryParseSpan(childSpan, descriptors, errorSink, processedBoundAttributeNames); return TryParseSpan(childSpan, descriptors, errorSink, processedBoundAttributeNames);
} }
var nameSymbols = childSpan var nameTokens = childSpan
.Symbols .Tokens
.OfType<HtmlSymbol>() .OfType<HtmlToken>()
.SkipWhile(symbol => !HtmlMarkupParser.IsValidAttributeNameSymbol(symbol)) // Skip prefix .SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token)) // Skip prefix
.TakeWhile(nameSymbol => HtmlMarkupParser.IsValidAttributeNameSymbol(nameSymbol)) .TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
.Select(nameSymbol => nameSymbol.Content); .Select(nameToken => nameToken.Content);
var name = string.Concat(nameSymbols); var name = string.Concat(nameTokens);
if (string.IsNullOrEmpty(name)) if (string.IsNullOrEmpty(name))
{ {
var location = new SourceSpan(childSpan.Start, childSpan.Length); var location = new SourceSpan(childSpan.Start, childSpan.Length);
@ -362,12 +362,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames); var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames);
var firstChild = builder.Children[0] as Span; var firstChild = builder.Children[0] as Span;
if (firstChild != null && firstChild.Symbols[0] is HtmlSymbol) if (firstChild != null && firstChild.Tokens[0] is HtmlToken)
{ {
var htmlSymbol = firstChild.Symbols[firstChild.Symbols.Count - 1] as HtmlSymbol; var htmlToken = firstChild.Tokens[firstChild.Tokens.Count - 1] as HtmlToken;
switch (htmlSymbol.Type) switch (htmlToken.Type)
{ {
case HtmlSymbolType.Equals: case HtmlTokenType.Equals:
if (builder.Children.Count == 2 && if (builder.Children.Count == 2 &&
builder.Children[1] is Span value && builder.Children[1] is Span value &&
value.Kind == SpanKindInternal.Markup) value.Kind == SpanKindInternal.Markup)
@ -385,10 +385,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
result.AttributeStructure = AttributeStructure.DoubleQuotes; result.AttributeStructure = AttributeStructure.DoubleQuotes;
} }
break; break;
case HtmlSymbolType.DoubleQuote: case HtmlTokenType.DoubleQuote:
result.AttributeStructure = AttributeStructure.DoubleQuotes; result.AttributeStructure = AttributeStructure.DoubleQuotes;
break; break;
case HtmlSymbolType.SingleQuote: case HtmlTokenType.SingleQuote:
result.AttributeStructure = AttributeStructure.SingleQuotes; result.AttributeStructure = AttributeStructure.SingleQuotes;
break; break;
default: default:
@ -407,12 +407,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var endSpan = (Span)endNode; var endSpan = (Span)endNode;
// In some malformed cases e.g. <p bar="false', the last Span (false' in the ex.) may contain more // In some malformed cases e.g. <p bar="false', the last Span (false' in the ex.) may contain more
// than a single HTML symbol. Do not ignore those other symbols. // than a single HTML token. Do not ignore those other tokens.
var symbolCount = endSpan.Symbols.Count(); var tokenCount = endSpan.Tokens.Count();
var endSymbol = symbolCount == 1 ? (HtmlSymbol)endSpan.Symbols.First() : null; var endToken = tokenCount == 1 ? (HtmlToken)endSpan.Tokens.First() : null;
// Checking to see if it's a quoted attribute, if so we should remove end quote // Checking to see if it's a quoted attribute, if so we should remove end quote
if (endSymbol != null && IsQuote(endSymbol)) if (endToken != null && IsQuote(endToken))
{ {
builder.Children.RemoveAt(builder.Children.Count - 1); builder.Children.RemoveAt(builder.Children.Count - 1);
} }
@ -612,12 +612,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Attributes must have at least one non-whitespace character to represent the tagName (even if its a C# // Attributes must have at least one non-whitespace character to represent the tagName (even if its a C#
// expression). // expression).
var firstNonWhitespaceSymbol = span var firstNonWhitespaceToken = span
.Symbols .Tokens
.OfType<HtmlSymbol>() .OfType<HtmlToken>()
.First(sym => sym.Type != HtmlSymbolType.WhiteSpace && sym.Type != HtmlSymbolType.NewLine); .First(token => token.Type != HtmlTokenType.WhiteSpace && token.Type != HtmlTokenType.NewLine);
var location = new SourceSpan(firstNonWhitespaceSymbol.Start, attributeName.Length); var location = new SourceSpan(firstNonWhitespaceToken.Start, attributeName.Length);
return location; return location;
} }
@ -716,10 +716,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return firstBoundAttribute; return firstBoundAttribute;
} }
private static bool IsQuote(HtmlSymbol htmlSymbol) private static bool IsQuote(HtmlToken htmlToken)
{ {
return htmlSymbol.Type == HtmlSymbolType.DoubleQuote || return htmlToken.Type == HtmlTokenType.DoubleQuote ||
htmlSymbol.Type == HtmlSymbolType.SingleQuote; htmlToken.Type == HtmlTokenType.SingleQuote;
} }
private static void ConfigureNonStringAttribute(SpanBuilder builder, bool isDuplicateAttribute) private static void ConfigureNonStringAttribute(SpanBuilder builder, bool isDuplicateAttribute)

60
src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperParseTreeRewriter.cs
View File

@ -370,12 +370,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// We can assume the first span will always contain attributename=" and the last span will always // We can assume the first span will always contain attributename=" and the last span will always
// contain the final quote. Therefore, if the values not quoted there's no ending quote to skip. // contain the final quote. Therefore, if the values not quoted there's no ending quote to skip.
var childOffset = 0; var childOffset = 0;
if (childSpan.Symbols.Count > 0) if (childSpan.Tokens.Count > 0)
{ {
var potentialQuote = childSpan.Symbols[childSpan.Symbols.Count - 1] as HtmlSymbol; var potentialQuote = childSpan.Tokens[childSpan.Tokens.Count - 1] as HtmlToken;
if (potentialQuote != null && if (potentialQuote != null &&
(potentialQuote.Type == HtmlSymbolType.DoubleQuote || (potentialQuote.Type == HtmlTokenType.DoubleQuote ||
potentialQuote.Type == HtmlSymbolType.SingleQuote)) potentialQuote.Type == HtmlTokenType.SingleQuote))
{ {
childOffset = 1; childOffset = 1;
} }
@ -391,9 +391,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else else
{ {
var valueChildSpan = (Span)valueChild; var valueChildSpan = (Span)valueChild;
for (var k = 0; k < valueChildSpan.Symbols.Count; k++) for (var k = 0; k < valueChildSpan.Tokens.Count; k++)
{ {
_attributeValueBuilder.Append(valueChildSpan.Symbols[k].Content); _attributeValueBuilder.Append(valueChildSpan.Tokens[k].Content);
} }
} }
} }
@ -404,43 +404,43 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var afterEquals = false; var afterEquals = false;
var atValue = false; var atValue = false;
var endValueMarker = childSpan.Symbols.Count; var endValueMarker = childSpan.Tokens.Count;
// Entire attribute is a string // Entire attribute is a string
for (var j = 0; j < endValueMarker; j++) for (var j = 0; j < endValueMarker; j++)
{ {
var htmlSymbol = (HtmlSymbol)childSpan.Symbols[j]; var htmlToken = (HtmlToken)childSpan.Tokens[j];
if (!afterEquals) if (!afterEquals)
{ {
afterEquals = htmlSymbol.Type == HtmlSymbolType.Equals; afterEquals = htmlToken.Type == HtmlTokenType.Equals;
continue; continue;
} }
if (!atValue) if (!atValue)
{ {
atValue = htmlSymbol.Type != HtmlSymbolType.WhiteSpace && atValue = htmlToken.Type != HtmlTokenType.WhiteSpace &&
htmlSymbol.Type != HtmlSymbolType.NewLine; htmlToken.Type != HtmlTokenType.NewLine;
if (atValue) if (atValue)
{ {
if (htmlSymbol.Type == HtmlSymbolType.DoubleQuote || if (htmlToken.Type == HtmlTokenType.DoubleQuote ||
htmlSymbol.Type == HtmlSymbolType.SingleQuote) htmlToken.Type == HtmlTokenType.SingleQuote)
{ {
endValueMarker--; endValueMarker--;
} }
else else
{ {
// Current symbol is considered the value (unquoted). Add its content to the // Current token is considered the value (unquoted). Add its content to the
// attribute value builder before we move past it. // attribute value builder before we move past it.
_attributeValueBuilder.Append(htmlSymbol.Content); _attributeValueBuilder.Append(htmlToken.Content);
} }
} }
continue; continue;
} }
_attributeValueBuilder.Append(htmlSymbol.Content); _attributeValueBuilder.Append(htmlToken.Content);
} }
} }
@ -642,11 +642,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// If our tag end is not a markup span it means it's some sort of code SyntaxTreeNode (not a valid format) // If our tag end is not a markup span it means it's some sort of code SyntaxTreeNode (not a valid format)
if (tagEnd != null && tagEnd.Kind == SpanKindInternal.Markup) if (tagEnd != null && tagEnd.Kind == SpanKindInternal.Markup)
{ {
var endSymbol = tagEnd.Symbols.Count > 0 ? var endToken = tagEnd.Tokens.Count > 0 ?
tagEnd.Symbols[tagEnd.Symbols.Count - 1] as HtmlSymbol : tagEnd.Tokens[tagEnd.Tokens.Count - 1] as HtmlToken :
null; null;
if (endSymbol != null && endSymbol.Type == HtmlSymbolType.CloseAngle) if (endToken != null && endToken.Type == HtmlTokenType.CloseAngle)
{ {
return false; return false;
} }
@ -793,25 +793,25 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
var childSpan = (Span)child; var childSpan = (Span)child;
HtmlSymbol textSymbol = null; HtmlToken textToken = null;
for (var i = 0; i < childSpan.Symbols.Count; i++) for (var i = 0; i < childSpan.Tokens.Count; i++)
{ {
var symbol = childSpan.Symbols[i] as HtmlSymbol; var token = childSpan.Tokens[i] as HtmlToken;
if (symbol != null && if (token != null &&
(symbol.Type & (HtmlSymbolType.WhiteSpace | HtmlSymbolType.Text)) == symbol.Type) (token.Type & (HtmlTokenType.WhiteSpace | HtmlTokenType.Text)) == token.Type)
{ {
textSymbol = symbol; textToken = token;
break; break;
} }
} }
if (textSymbol == null) if (textToken == null)
{ {
return null; return null;
} }
return textSymbol.Type == HtmlSymbolType.WhiteSpace ? null : textSymbol.Content; return textToken.Type == HtmlTokenType.WhiteSpace ? null : textToken.Content;
} }
private static bool IsEndTag(Block tagBlock) private static bool IsEndTag(Block tagBlock)
@ -820,10 +820,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var childSpan = (Span)tagBlock.Children.First(); var childSpan = (Span)tagBlock.Children.First();
// We grab the symbol that could be forward slash // We grab the token that could be forward slash
var relevantSymbol = (HtmlSymbol)childSpan.Symbols[childSpan.Symbols.Count == 1 ? 0 : 1]; var relevantToken = (HtmlToken)childSpan.Tokens[childSpan.Tokens.Count == 1 ? 0 : 1];
return relevantSymbol.Type == HtmlSymbolType.ForwardSlash; return relevantToken.Type == HtmlTokenType.ForwardSlash;
} }
internal static bool IsComment(Span span) internal static bool IsComment(Span span)

14
src/Microsoft.AspNetCore.Razor.Language/Legacy/SymbolBase.cs → src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenBase.cs
View File

@ -8,9 +8,9 @@ using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract class SymbolBase<TType> : ISymbol where TType : struct internal abstract class TokenBase<TType> : IToken where TType : struct
{ {
protected SymbolBase( protected TokenBase(
string content, string content,
TType type, TType type,
IReadOnlyList<RazorDiagnostic> errors) IReadOnlyList<RazorDiagnostic> errors)
@ -43,15 +43,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
var tracker = new SourceLocationTracker(Parent.Start); var tracker = new SourceLocationTracker(Parent.Start);
for (var i = 0; i < Parent.Symbols.Count; i++) for (var i = 0; i < Parent.Tokens.Count; i++)
{ {
var symbol = Parent.Symbols[i]; var token = Parent.Tokens[i];
if (object.ReferenceEquals(this, symbol)) if (object.ReferenceEquals(this, token))
{ {
break; break;
} }
tracker.UpdateLocation(symbol.Content); tracker.UpdateLocation(token.Content);
} }
return tracker.CurrentLocation; return tracker.CurrentLocation;
@ -60,7 +60,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var other = obj as SymbolBase<TType>; var other = obj as TokenBase<TType>;
return other != null && return other != null &&
string.Equals(Content, other.Content, StringComparison.Ordinal) && string.Equals(Content, other.Content, StringComparison.Ordinal) &&
Type.Equals(other.Type); Type.Equals(other.Type);

90
src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs
View File

@ -8,9 +8,9 @@ using System.Text;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract partial class Tokenizer<TSymbol, TSymbolType> : ITokenizer internal abstract partial class Tokenizer<TToken, TTokenType> : ITokenizer
where TSymbolType : struct where TTokenType : struct
where TSymbol : SymbolBase<TSymbolType> where TToken : TokenBase<TTokenType>
{ {
protected Tokenizer(ITextDocument source) protected Tokenizer(ITextDocument source)
{ {
@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Source = source; Source = source;
Buffer = new StringBuilder(); Buffer = new StringBuilder();
CurrentErrors = new List<RazorDiagnostic>(); CurrentErrors = new List<RazorDiagnostic>();
StartSymbol(); StartToken();
} }
protected List<RazorDiagnostic> CurrentErrors { get; } protected List<RazorDiagnostic> CurrentErrors { get; }
@ -31,7 +31,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected int? CurrentState { get; set; } protected int? CurrentState { get; set; }
protected TSymbol CurrentSymbol { get; private set; } protected TToken CurrentToken { get; private set; }
public ITextDocument Source { get; private set; } public ITextDocument Source { get; private set; }
@ -42,9 +42,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return Source.Peek() == -1; } get { return Source.Peek() == -1; }
} }
public abstract TSymbolType RazorCommentStarType { get; } public abstract TTokenType RazorCommentStarType { get; }
public abstract TSymbolType RazorCommentType { get; } public abstract TTokenType RazorCommentType { get; }
public abstract TSymbolType RazorCommentTransitionType { get; } public abstract TTokenType RazorCommentTransitionType { get; }
protected bool HaveContent protected bool HaveContent
{ {
@ -64,27 +64,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SourceLocation CurrentStart { get; private set; } public SourceLocation CurrentStart { get; private set; }
protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList<RazorDiagnostic> errors); protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList<RazorDiagnostic> errors);
protected abstract StateResult Dispatch(); protected abstract StateResult Dispatch();
ISymbol ITokenizer.NextSymbol() IToken ITokenizer.NextToken()
{ {
return NextSymbol(); return NextToken();
} }
public virtual TSymbol NextSymbol() public virtual TToken NextToken()
{ {
// Post-Condition: Buffer should be empty at the start of Next() // Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
StartSymbol(); StartToken();
if (EndOfFile) if (EndOfFile)
{ {
return null; return null;
} }
var symbol = Turn(); var token = Turn();
// Post-Condition: Buffer should be empty at the end of Next() // Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
@ -92,10 +92,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Post-Condition: Token should be non-zero length unless we're at EOF. // Post-Condition: Token should be non-zero length unless we're at EOF.
Debug.Assert(EndOfFile || !CurrentStart.Equals(CurrentLocation)); Debug.Assert(EndOfFile || !CurrentStart.Equals(CurrentLocation));
return symbol; return token;
} }
protected virtual TSymbol Turn() protected virtual TToken Turn()
{ {
if (CurrentState != null) if (CurrentState != null)
{ {
@ -105,19 +105,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var next = Dispatch(); var next = Dispatch();
CurrentState = next.State; CurrentState = next.State;
CurrentSymbol = next.Result; CurrentToken = next.Result;
} }
while (CurrentState != null && CurrentSymbol == null); while (CurrentState != null && CurrentToken == null);
if (CurrentState == null) if (CurrentState == null)
{ {
return default(TSymbol); // Terminated return default(TToken); // Terminated
} }
return CurrentSymbol; return CurrentToken;
} }
return default(TSymbol); return default(TToken);
} }
public void Reset() public void Reset()
@ -149,7 +149,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to /// Returns a result containing the specified output and indicating that the next call to
/// <see cref="Turn"/> should invoke the provided state. /// <see cref="Turn"/> should invoke the provided state.
/// </summary> /// </summary>
protected StateResult Transition(int state, TSymbol result) protected StateResult Transition(int state, TToken result)
{ {
return new StateResult(state, result); return new StateResult(state, result);
} }
@ -159,7 +159,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new StateResult((int)state, result: null); return new StateResult((int)state, result: null);
} }
protected StateResult Transition(RazorCommentTokenizerState state, TSymbol result) protected StateResult Transition(RazorCommentTokenizerState state, TToken result)
{ {
return new StateResult((int)state, result); return new StateResult((int)state, result);
} }
@ -180,18 +180,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to /// Returns a result containing the specified output and indicating that the next call to
/// <see cref="Turn"/> should re-invoke the current state. /// <see cref="Turn"/> should re-invoke the current state.
/// </summary> /// </summary>
protected StateResult Stay(TSymbol result) protected StateResult Stay(TToken result)
{ {
return new StateResult(CurrentState, result); return new StateResult(CurrentState, result);
} }
protected TSymbol Single(TSymbolType type) protected TToken Single(TTokenType type)
{ {
TakeCurrent(); TakeCurrent();
return EndSymbol(type); return EndToken(type);
} }
protected void StartSymbol() protected void StartToken()
{ {
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
Debug.Assert(CurrentErrors.Count == 0); Debug.Assert(CurrentErrors.Count == 0);
@ -199,9 +199,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
CurrentStart = CurrentLocation; CurrentStart = CurrentLocation;
} }
protected TSymbol EndSymbol(TSymbolType type) protected TToken EndToken(TTokenType type)
{ {
TSymbol symbol = null; TToken token = null;
if (HaveContent) if (HaveContent)
{ {
// Perf: Don't allocate a new errors array unless necessary. // Perf: Don't allocate a new errors array unless necessary.
@ -211,18 +211,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
errors[i] = CurrentErrors[i]; errors[i] = CurrentErrors[i];
} }
var symbolContent = GetSymbolContent(type); var tokenContent = GetTokenContent(type);
Debug.Assert(string.Equals(symbolContent, Buffer.ToString(), StringComparison.Ordinal)); Debug.Assert(string.Equals(tokenContent, Buffer.ToString(), StringComparison.Ordinal));
symbol = CreateSymbol(symbolContent, type, errors); token = CreateToken(tokenContent, type, errors);
Buffer.Clear(); Buffer.Clear();
CurrentErrors.Clear(); CurrentErrors.Clear();
} }
return symbol; return token;
} }
protected virtual string GetSymbolContent(TSymbolType type) protected virtual string GetTokenContent(TTokenType type)
{ {
return Buffer.ToString(); return Buffer.ToString();
} }
@ -272,13 +272,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
if (CurrentCharacter != '*') if (CurrentCharacter != '*')
{ {
// We've been moved since last time we were asked for a symbol... reset the state // We've been moved since last time we were asked for a token... reset the state
return Transition(StartState); return Transition(StartState);
} }
AssertCurrent('*'); AssertCurrent('*');
TakeCurrent(); TakeCurrent();
return Transition(1002, EndSymbol(RazorCommentStarType)); return Transition(1002, EndToken(RazorCommentStarType));
} }
protected StateResult RazorCommentBody() protected StateResult RazorCommentBody()
@ -292,7 +292,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
return Transition( return Transition(
RazorCommentTokenizerState.StarAfterRazorCommentBody, RazorCommentTokenizerState.StarAfterRazorCommentBody,
EndSymbol(RazorCommentType)); EndToken(RazorCommentType));
} }
else else
{ {
@ -306,7 +306,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
return Transition(StartState, EndSymbol(RazorCommentType)); return Transition(StartState, EndToken(RazorCommentType));
} }
protected StateResult StarAfterRazorCommentBody() protected StateResult StarAfterRazorCommentBody()
@ -314,15 +314,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AssertCurrent('*'); AssertCurrent('*');
TakeCurrent(); TakeCurrent();
return Transition( return Transition(
RazorCommentTokenizerState.AtSymbolAfterRazorCommentBody, RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
EndSymbol(RazorCommentStarType)); EndToken(RazorCommentStarType));
} }
protected StateResult AtSymbolAfterRazorCommentBody() protected StateResult AtTokenAfterRazorCommentBody()
{ {
AssertCurrent('@'); AssertCurrent('@');
TakeCurrent(); TakeCurrent();
return Transition(StartState, EndSymbol(RazorCommentTransitionType)); return Transition(StartState, EndToken(RazorCommentTransitionType));
} }
/// <summary> /// <summary>
@ -392,12 +392,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
EscapedRazorCommentTransition, EscapedRazorCommentTransition,
RazorCommentBody, RazorCommentBody,
StarAfterRazorCommentBody, StarAfterRazorCommentBody,
AtSymbolAfterRazorCommentBody, AtTokenAfterRazorCommentBody,
} }
protected struct StateResult protected struct StateResult
{ {
public StateResult(int? state, TSymbol result) public StateResult(int? state, TToken result)
{ {
State = state; State = state;
Result = result; Result = result;
@ -405,7 +405,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public int? State { get; } public int? State { get; }
public TSymbol Result { get; } public TToken Result { get; }
} }
private static LookaheadToken BeginLookahead(ITextBuffer buffer) private static LookaheadToken BeginLookahead(ITextBuffer buffer)

262
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
View File

@ -8,20 +8,20 @@ using System.Linq;
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal abstract partial class TokenizerBackedParser<TTokenizer, TSymbol, TSymbolType> : ParserBase internal abstract partial class TokenizerBackedParser<TTokenizer, TToken, TTokenType> : ParserBase
where TSymbolType : struct where TTokenType : struct
where TTokenizer : Tokenizer<TSymbol, TSymbolType> where TTokenizer : Tokenizer<TToken, TTokenType>
where TSymbol : SymbolBase<TSymbolType> where TToken : TokenBase<TTokenType>
{ {
private readonly TokenizerView<TTokenizer, TSymbol, TSymbolType> _tokenizer; private readonly TokenizerView<TTokenizer, TToken, TTokenType> _tokenizer;
protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TSymbol, TSymbolType> language, ParserContext context) protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TToken, TTokenType> language, ParserContext context)
: base(context) : base(context)
{ {
Language = language; Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source); var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView<TTokenizer, TSymbol, TSymbolType>(languageTokenizer); _tokenizer = new TokenizerView<TTokenizer, TToken, TTokenType>(languageTokenizer);
Span = new SpanBuilder(CurrentLocation); Span = new SpanBuilder(CurrentLocation);
} }
@ -29,12 +29,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected Action<SpanBuilder> SpanConfig { get; set; } protected Action<SpanBuilder> SpanConfig { get; set; }
protected TSymbol CurrentSymbol protected TToken CurrentToken
{ {
get { return _tokenizer.Current; } get { return _tokenizer.Current; }
} }
protected TSymbol PreviousSymbol { get; private set; } protected TToken PreviousToken { get; private set; }
protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation; protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _tokenizer.EndOfFile; } get { return _tokenizer.EndOfFile; }
} }
protected LanguageCharacteristics<TTokenizer, TSymbol, TSymbolType> Language { get; } protected LanguageCharacteristics<TTokenizer, TToken, TTokenType> Language { get; }
protected virtual void HandleEmbeddedTransition() protected virtual void HandleEmbeddedTransition()
{ {
@ -58,7 +58,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override void BuildSpan(SpanBuilder span, SourceLocation start, string content) public override void BuildSpan(SpanBuilder span, SourceLocation start, string content)
{ {
foreach (ISymbol sym in Language.TokenizeString(start, content)) foreach (IToken sym in Language.TokenizeString(start, content))
{ {
span.Accept(sym); span.Accept(sym);
} }
@ -72,7 +72,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
protected TSymbol Lookahead(int count) protected TToken Lookahead(int count)
{ {
if (count < 0) if (count < 0)
{ {
@ -80,42 +80,42 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
else if (count == 0) else if (count == 0)
{ {
return CurrentSymbol; return CurrentToken;
} }
// We add 1 in order to store the current symbol. // We add 1 in order to store the current token.
var symbols = new TSymbol[count + 1]; var tokens = new TToken[count + 1];
var currentSymbol = CurrentSymbol; var currentToken = CurrentToken;
symbols[0] = currentSymbol; tokens[0] = currentToken;
// We need to look forward "count" many times. // We need to look forward "count" many times.
for (var i = 1; i <= count; i++) for (var i = 1; i <= count; i++)
{ {
NextToken(); NextToken();
symbols[i] = CurrentSymbol; tokens[i] = CurrentToken;
} }
// Restore Tokenizer's location to where it was pointing before the look-ahead. // Restore Tokenizer's location to where it was pointing before the look-ahead.
for (var i = count; i >= 0; i--) for (var i = count; i >= 0; i--)
{ {
PutBack(symbols[i]); PutBack(tokens[i]);
} }
// The PutBacks above will set CurrentSymbol to null. EnsureCurrent will set our CurrentSymbol to the // The PutBacks above will set CurrentToken to null. EnsureCurrent will set our CurrentToken to the
// next symbol. // next token.
EnsureCurrent(); EnsureCurrent();
return symbols[count]; return tokens[count];
} }
/// <summary> /// <summary>
/// Looks forward until the specified condition is met. /// Looks forward until the specified condition is met.
/// </summary> /// </summary>
/// <param name="condition">A predicate accepting the symbol being evaluated and the list of symbols which have been looped through.</param> /// <param name="condition">A predicate accepting the token being evaluated and the list of tokens which have been looped through.</param>
/// <returns>true, if the condition was met. false - if the condition wasn't met and the last symbol has already been processed.</returns> /// <returns>true, if the condition was met. false - if the condition wasn't met and the last token has already been processed.</returns>
/// <remarks>The list of previous symbols is passed in the reverse order. So the last processed element will be the first one in the list.</remarks> /// <remarks>The list of previous tokens is passed in the reverse order. So the last processed element will be the first one in the list.</remarks>
protected bool LookaheadUntil(Func<TSymbol, IEnumerable<TSymbol>, bool> condition) protected bool LookaheadUntil(Func<TToken, IEnumerable<TToken>, bool> condition)
{ {
if (condition == null) if (condition == null)
{ {
@ -124,8 +124,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var matchFound = false; var matchFound = false;
var symbols = new List<TSymbol>(); var tokens = new List<TToken>();
symbols.Add(CurrentSymbol); tokens.Add(CurrentToken);
while (true) while (true)
{ {
@ -134,8 +134,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break; break;
} }
symbols.Add(CurrentSymbol); tokens.Add(CurrentToken);
if (condition(CurrentSymbol, symbols)) if (condition(CurrentToken, tokens))
{ {
matchFound = true; matchFound = true;
break; break;
@ -143,13 +143,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
// Restore Tokenizer's location to where it was pointing before the look-ahead. // Restore Tokenizer's location to where it was pointing before the look-ahead.
for (var i = symbols.Count - 1; i >= 0; i--) for (var i = tokens.Count - 1; i >= 0; i--)
{ {
PutBack(symbols[i]); PutBack(tokens[i]);
} }
// The PutBacks above will set CurrentSymbol to null. EnsureCurrent will set our CurrentSymbol to the // The PutBacks above will set CurrentToken to null. EnsureCurrent will set our CurrentToken to the
// next symbol. // next token.
EnsureCurrent(); EnsureCurrent();
return matchFound; return matchFound;
@ -157,57 +157,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected internal bool NextToken() protected internal bool NextToken()
{ {
PreviousSymbol = CurrentSymbol; PreviousToken = CurrentToken;
return _tokenizer.Next(); return _tokenizer.Next();
} }
// Helpers // Helpers
[Conditional("DEBUG")] [Conditional("DEBUG")]
internal void Assert(TSymbolType expectedType) internal void Assert(TTokenType expectedType)
{ {
Debug.Assert(!EndOfFile && SymbolTypeEquals(CurrentSymbol.Type, expectedType)); Debug.Assert(!EndOfFile && TokenTypeEquals(CurrentToken.Type, expectedType));
} }
abstract protected bool SymbolTypeEquals(TSymbolType x, TSymbolType y); abstract protected bool TokenTypeEquals(TTokenType x, TTokenType y);
protected internal void PutBack(TSymbol symbol) protected internal void PutBack(TToken token)
{ {
if (symbol != null) if (token != null)
{ {
_tokenizer.PutBack(symbol); _tokenizer.PutBack(token);
} }
} }
/// <summary> /// <summary>
/// Put the specified symbols back in the input stream. The provided list MUST be in the ORDER THE SYMBOLS WERE READ. The /// Put the specified tokens back in the input stream. The provided list MUST be in the ORDER THE TOKENS WERE READ. The
/// list WILL be reversed and the Putback(TSymbol) will be called on each item. /// list WILL be reversed and the Putback(TToken) will be called on each item.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// If a document contains symbols: a, b, c, d, e, f /// If a document contains tokens: a, b, c, d, e, f
/// and AcceptWhile or AcceptUntil is used to collect until d /// and AcceptWhile or AcceptUntil is used to collect until d
/// the list returned by AcceptWhile/Until will contain: a, b, c IN THAT ORDER /// the list returned by AcceptWhile/Until will contain: a, b, c IN THAT ORDER
/// that is the correct format for providing to this method. The caller of this method would, /// that is the correct format for providing to this method. The caller of this method would,
/// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order /// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order
/// </remarks> /// </remarks>
protected internal void PutBack(IEnumerable<TSymbol> symbols) protected internal void PutBack(IEnumerable<TToken> tokens)
{ {
foreach (TSymbol symbol in symbols.Reverse()) foreach (TToken token in tokens.Reverse())
{ {
PutBack(symbol); PutBack(token);
} }
} }
protected internal void PutCurrentBack() protected internal void PutCurrentBack()
{ {
if (!EndOfFile && CurrentSymbol != null) if (!EndOfFile && CurrentToken != null)
{ {
PutBack(CurrentSymbol); PutBack(CurrentToken);
} }
} }
protected internal bool Balance(BalancingModes mode) protected internal bool Balance(BalancingModes mode)
{ {
var left = CurrentSymbol.Type; var left = CurrentToken.Type;
var right = Language.FlipBracket(left); var right = Language.FlipBracket(left);
var start = CurrentStart; var start = CurrentStart;
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -223,13 +223,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Balance(mode, left, right, start); return Balance(mode, left, right, start);
} }
protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start) protected internal bool Balance(BalancingModes mode, TTokenType left, TTokenType right, SourceLocation start)
{ {
var startPosition = CurrentStart.AbsoluteIndex; var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1; var nesting = 1;
if (!EndOfFile) if (!EndOfFile)
{ {
var syms = new List<TSymbol>(); var syms = new List<TToken>();
do do
{ {
if (IsAtEmbeddedTransition( if (IsAtEmbeddedTransition(
@ -253,7 +253,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
if (nesting > 0) if (nesting > 0)
{ {
syms.Add(CurrentSymbol); syms.Add(CurrentToken);
} }
} }
while (nesting > 0 && NextToken()); while (nesting > 0 && NextToken());
@ -280,29 +280,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
else else
{ {
// Accept all the symbols we saw // Accept all the tokens we saw
Accept(syms); Accept(syms);
} }
} }
return nesting == 0; return nesting == 0;
} }
protected internal bool NextIs(TSymbolType type) protected internal bool NextIs(TTokenType type)
{ {
return NextIs(sym => sym != null && SymbolTypeEquals(type, sym.Type)); return NextIs(sym => sym != null && TokenTypeEquals(type, sym.Type));
} }
protected internal bool NextIs(params TSymbolType[] types) protected internal bool NextIs(params TTokenType[] types)
{ {
return NextIs(sym => sym != null && types.Any(t => SymbolTypeEquals(t, sym.Type))); return NextIs(sym => sym != null && types.Any(t => TokenTypeEquals(t, sym.Type)));
} }
protected internal bool NextIs(Func<TSymbol, bool> condition) protected internal bool NextIs(Func<TToken, bool> condition)
{ {
var cur = CurrentSymbol; var cur = CurrentToken;
if (NextToken()) if (NextToken())
{ {
var result = condition(CurrentSymbol); var result = condition(CurrentToken);
PutCurrentBack(); PutCurrentBack();
PutBack(cur); PutBack(cur);
EnsureCurrent(); EnsureCurrent();
@ -317,27 +317,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false; return false;
} }
protected internal bool Was(TSymbolType type) protected internal bool Was(TTokenType type)
{ {
return PreviousSymbol != null && SymbolTypeEquals(PreviousSymbol.Type, type); return PreviousToken != null && TokenTypeEquals(PreviousToken.Type, type);
} }
protected internal bool At(TSymbolType type) protected internal bool At(TTokenType type)
{ {
return !EndOfFile && CurrentSymbol != null && SymbolTypeEquals(CurrentSymbol.Type, type); return !EndOfFile && CurrentToken != null && TokenTypeEquals(CurrentToken.Type, type);
} }
protected internal bool AcceptAndMoveNext() protected internal bool AcceptAndMoveNext()
{ {
Accept(CurrentSymbol); Accept(CurrentToken);
return NextToken(); return NextToken();
} }
protected TSymbol AcceptSingleWhiteSpaceCharacter() protected TToken AcceptSingleWhiteSpaceCharacter()
{ {
if (Language.IsWhiteSpace(CurrentSymbol)) if (Language.IsWhiteSpace(CurrentToken))
{ {
Tuple<TSymbol, TSymbol> pair = Language.SplitSymbol(CurrentSymbol, 1, Language.GetKnownSymbolType(KnownSymbolType.WhiteSpace)); Tuple<TToken, TToken> pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
Accept(pair.Item1); Accept(pair.Item1);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None; Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
NextToken(); NextToken();
@ -346,32 +346,32 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null; return null;
} }
protected internal void Accept(IEnumerable<TSymbol> symbols) protected internal void Accept(IEnumerable<TToken> tokens)
{ {
foreach (TSymbol symbol in symbols) foreach (TToken token in tokens)
{ {
Accept(symbol); Accept(token);
} }
} }
protected internal void Accept(TSymbol symbol) protected internal void Accept(TToken token)
{ {
if (symbol != null) if (token != null)
{ {
foreach (var error in symbol.Errors) foreach (var error in token.Errors)
{ {
Context.ErrorSink.OnError(error); Context.ErrorSink.OnError(error);
} }
Span.Accept(symbol); Span.Accept(token);
} }
} }
protected internal bool AcceptAll(params TSymbolType[] types) protected internal bool AcceptAll(params TTokenType[] types)
{ {
foreach (TSymbolType type in types) foreach (TTokenType type in types)
{ {
if (CurrentSymbol == null || !SymbolTypeEquals(CurrentSymbol.Type, type)) if (CurrentToken == null || !TokenTypeEquals(CurrentToken.Type, type))
{ {
return false; return false;
} }
@ -380,11 +380,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true; return true;
} }
protected internal void AddMarkerSymbolIfNecessary() protected internal void AddMarkerTokenIfNecessary()
{ {
if (Span.Symbols.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharactersInternal.Any) if (Span.Tokens.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharactersInternal.Any)
{ {
Accept(Language.CreateMarkerSymbol()); Accept(Language.CreateMarkerToken());
} }
} }
@ -408,7 +408,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void Output() private void Output()
{ {
if (Span.Symbols.Count > 0) if (Span.Tokens.Count > 0)
{ {
var nextStart = Span.End; var nextStart = Span.End;
@ -462,23 +462,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Initialize(Span); Initialize(Span);
} }
protected internal void Expected(KnownSymbolType type) protected internal void Expected(KnownTokenType type)
{ {
Expected(Language.GetKnownSymbolType(type)); Expected(Language.GetKnownTokenType(type));
} }
protected internal void Expected(params TSymbolType[] types) protected internal void Expected(params TTokenType[] types)
{ {
Debug.Assert(!EndOfFile && CurrentSymbol != null && types.Contains(CurrentSymbol.Type)); Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Type));
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
protected internal bool Optional(KnownSymbolType type) protected internal bool Optional(KnownTokenType type)
{ {
return Optional(Language.GetKnownSymbolType(type)); return Optional(Language.GetKnownTokenType(type));
} }
protected internal bool Optional(TSymbolType type) protected internal bool Optional(TTokenType type)
{ {
if (At(type)) if (At(type))
{ {
@ -490,7 +490,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool EnsureCurrent() protected bool EnsureCurrent()
{ {
if (CurrentSymbol == null) if (CurrentToken == null)
{ {
return NextToken(); return NextToken();
} }
@ -498,62 +498,62 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true; return true;
} }
protected internal void AcceptWhile(TSymbolType type) protected internal void AcceptWhile(TTokenType type)
{ {
AcceptWhile(sym => SymbolTypeEquals(type, sym.Type)); AcceptWhile(sym => TokenTypeEquals(type, sym.Type));
} }
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format // We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
protected internal void AcceptWhile(TSymbolType type1, TSymbolType type2) protected internal void AcceptWhile(TTokenType type1, TTokenType type2)
{ {
AcceptWhile(sym => SymbolTypeEquals(type1, sym.Type) || SymbolTypeEquals(type2, sym.Type)); AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type));
} }
protected internal void AcceptWhile(TSymbolType type1, TSymbolType type2, TSymbolType type3) protected internal void AcceptWhile(TTokenType type1, TTokenType type2, TTokenType type3)
{ {
AcceptWhile(sym => SymbolTypeEquals(type1, sym.Type) || SymbolTypeEquals(type2, sym.Type) || SymbolTypeEquals(type3, sym.Type)); AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type) || TokenTypeEquals(type3, sym.Type));
} }
protected internal void AcceptWhile(params TSymbolType[] types) protected internal void AcceptWhile(params TTokenType[] types)
{ {
AcceptWhile(sym => types.Any(expected => SymbolTypeEquals(expected, sym.Type))); AcceptWhile(sym => types.Any(expected => TokenTypeEquals(expected, sym.Type)));
} }
protected internal void AcceptUntil(TSymbolType type) protected internal void AcceptUntil(TTokenType type)
{ {
AcceptWhile(sym => !SymbolTypeEquals(type, sym.Type)); AcceptWhile(sym => !TokenTypeEquals(type, sym.Type));
} }
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format // We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
protected internal void AcceptUntil(TSymbolType type1, TSymbolType type2) protected internal void AcceptUntil(TTokenType type1, TTokenType type2)
{ {
AcceptWhile(sym => !SymbolTypeEquals(type1, sym.Type) && !SymbolTypeEquals(type2, sym.Type)); AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type));
} }
protected internal void AcceptUntil(TSymbolType type1, TSymbolType type2, TSymbolType type3) protected internal void AcceptUntil(TTokenType type1, TTokenType type2, TTokenType type3)
{ {
AcceptWhile(sym => !SymbolTypeEquals(type1, sym.Type) && !SymbolTypeEquals(type2, sym.Type) && !SymbolTypeEquals(type3, sym.Type)); AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type) && !TokenTypeEquals(type3, sym.Type));
} }
protected internal void AcceptUntil(params TSymbolType[] types) protected internal void AcceptUntil(params TTokenType[] types)
{ {
AcceptWhile(sym => types.All(expected => !SymbolTypeEquals(expected, sym.Type))); AcceptWhile(sym => types.All(expected => !TokenTypeEquals(expected, sym.Type)));
} }
protected internal void AcceptWhile(Func<TSymbol, bool> condition) protected internal void AcceptWhile(Func<TToken, bool> condition)
{ {
Accept(ReadWhileLazy(condition)); Accept(ReadWhileLazy(condition));
} }
protected internal IEnumerable<TSymbol> ReadWhile(Func<TSymbol, bool> condition) protected internal IEnumerable<TToken> ReadWhile(Func<TToken, bool> condition)
{ {
return ReadWhileLazy(condition).ToList(); return ReadWhileLazy(condition).ToList();
} }
protected TSymbol AcceptWhiteSpaceInLines() protected TToken AcceptWhiteSpaceInLines()
{ {
TSymbol lastWs = null; TToken lastWs = null;
while (Language.IsWhiteSpace(CurrentSymbol) || Language.IsNewLine(CurrentSymbol)) while (Language.IsWhiteSpace(CurrentToken) || Language.IsNewLine(CurrentToken))
{ {
// Capture the previous whitespace node // Capture the previous whitespace node
if (lastWs != null) if (lastWs != null)
@ -561,14 +561,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(lastWs); Accept(lastWs);
} }
if (Language.IsWhiteSpace(CurrentSymbol)) if (Language.IsWhiteSpace(CurrentToken))
{ {
lastWs = CurrentSymbol; lastWs = CurrentToken;
} }
else if (Language.IsNewLine(CurrentSymbol)) else if (Language.IsNewLine(CurrentToken))
{ {
// Accept newline and reset last whitespace tracker // Accept newline and reset last whitespace tracker
Accept(CurrentSymbol); Accept(CurrentToken);
lastWs = null; lastWs = null;
} }
@ -579,18 +579,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool AtIdentifier(bool allowKeywords) protected bool AtIdentifier(bool allowKeywords)
{ {
return CurrentSymbol != null && return CurrentToken != null &&
(Language.IsIdentifier(CurrentSymbol) || (Language.IsIdentifier(CurrentToken) ||
(allowKeywords && Language.IsKeyword(CurrentSymbol))); (allowKeywords && Language.IsKeyword(CurrentToken)));
} }
// Don't open this to sub classes because it's lazy but it looks eager. // Don't open this to sub classes because it's lazy but it looks eager.
// You have to advance the Enumerable to read the next characters. // You have to advance the Enumerable to read the next characters.
internal IEnumerable<TSymbol> ReadWhileLazy(Func<TSymbol, bool> condition) internal IEnumerable<TToken> ReadWhileLazy(Func<TToken, bool> condition)
{ {
while (EnsureCurrent() && condition(CurrentSymbol)) while (EnsureCurrent() && condition(CurrentToken))
{ {
yield return CurrentSymbol; yield return CurrentToken;
NextToken(); NextToken();
} }
} }
@ -620,9 +620,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected void RazorComment() protected void RazorComment()
{ {
if (!Language.KnowsSymbolType(KnownSymbolType.CommentStart) || if (!Language.KnowsTokenType(KnownTokenType.CommentStart) ||
!Language.KnowsSymbolType(KnownSymbolType.CommentStar) || !Language.KnowsTokenType(KnownTokenType.CommentStar) ||
!Language.KnowsSymbolType(KnownSymbolType.CommentBody)) !Language.KnowsTokenType(KnownTokenType.CommentBody))
{ {
throw new InvalidOperationException(Resources.Language_Does_Not_Support_RazorComment); throw new InvalidOperationException(Resources.Language_Does_Not_Support_RazorComment);
} }
@ -634,18 +634,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator(); Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator();
var start = CurrentStart; var start = CurrentStart;
Expected(KnownSymbolType.CommentStart); Expected(KnownTokenType.CommentStart);
Output(SpanKindInternal.Transition, AcceptedCharactersInternal.None); Output(SpanKindInternal.Transition, AcceptedCharactersInternal.None);
Expected(KnownSymbolType.CommentStar); Expected(KnownTokenType.CommentStar);
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None); Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None);
Optional(KnownSymbolType.CommentBody); Optional(KnownTokenType.CommentBody);
AddMarkerSymbolIfNecessary(); AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Comment); Output(SpanKindInternal.Comment);
var errorReported = false; var errorReported = false;
if (!Optional(KnownSymbolType.CommentStar)) if (!Optional(KnownTokenType.CommentStar))
{ {
errorReported = true; errorReported = true;
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
@ -657,7 +657,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None); Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None);
} }
if (!Optional(KnownSymbolType.CommentStart)) if (!Optional(KnownTokenType.CommentStart))
{ {
if (!errorReported) if (!errorReported)
{ {

16
src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
View File

@ -3,10 +3,10 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
internal class TokenizerView<TTokenizer, TSymbol, TSymbolType> internal class TokenizerView<TTokenizer, TToken, TTokenType>
where TSymbolType : struct where TTokenType : struct
where TTokenizer : Tokenizer<TSymbol, TSymbolType> where TTokenizer : Tokenizer<TToken, TTokenType>
where TSymbol : SymbolBase<TSymbolType> where TToken : TokenBase<TTokenType>
{ {
public TokenizerView(TTokenizer tokenizer) public TokenizerView(TTokenizer tokenizer)
{ {
@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public TTokenizer Tokenizer { get; private set; } public TTokenizer Tokenizer { get; private set; }
public bool EndOfFile { get; private set; } public bool EndOfFile { get; private set; }
public TSymbol Current { get; private set; } public TToken Current { get; private set; }
public ITextDocument Source public ITextDocument Source
{ {
@ -24,14 +24,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public bool Next() public bool Next()
{ {
Current = Tokenizer.NextSymbol(); Current = Tokenizer.NextToken();
EndOfFile = (Current == null); EndOfFile = (Current == null);
return !EndOfFile; return !EndOfFile;
} }
public void PutBack(TSymbol symbol) public void PutBack(TToken token)
{ {
Source.Position -= symbol.Content.Length; Source.Position -= token.Content.Length;
Current = null; Current = null;
EndOfFile = Source.Position >= Source.Length; EndOfFile = Source.Position >= Source.Length;
Tokenizer.Reset(); Tokenizer.Reset();

2
src/Microsoft.AspNetCore.Razor.Language/Legacy/WhiteSpaceRewriter.cs
View File

@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Add this node to the parent // Add this node to the parent
var builder = new SpanBuilder(ws); var builder = new SpanBuilder(ws);
builder.ClearSymbols(); builder.ClearTokens();
FillSpan(builder, ws.Start, ws.Content); FillSpan(builder, ws.Start, ws.Content);
parent.Children.Add(builder.Build()); parent.Children.Add(builder.Build());

116
src/Microsoft.AspNetCore.Razor.Language/Properties/Resources.Designer.cs generated
View File

@ -139,128 +139,128 @@ namespace Microsoft.AspNetCore.Razor.Language
/// <summary> /// <summary>
/// &lt;&lt;character literal&gt;&gt; /// &lt;&lt;character literal&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_CharacterLiteral internal static string CSharpToken_CharacterLiteral
{ {
get => GetString("CSharpSymbol_CharacterLiteral"); get => GetString("CSharpToken_CharacterLiteral");
} }
/// <summary> /// <summary>
/// &lt;&lt;character literal&gt;&gt; /// &lt;&lt;character literal&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_CharacterLiteral() internal static string FormatCSharpToken_CharacterLiteral()
=> GetString("CSharpSymbol_CharacterLiteral"); => GetString("CSharpToken_CharacterLiteral");
/// <summary> /// <summary>
/// &lt;&lt;comment&gt;&gt; /// &lt;&lt;comment&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_Comment internal static string CSharpToken_Comment
{ {
get => GetString("CSharpSymbol_Comment"); get => GetString("CSharpToken_Comment");
} }
/// <summary> /// <summary>
/// &lt;&lt;comment&gt;&gt; /// &lt;&lt;comment&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_Comment() internal static string FormatCSharpToken_Comment()
=> GetString("CSharpSymbol_Comment"); => GetString("CSharpToken_Comment");
/// <summary> /// <summary>
/// &lt;&lt;identifier&gt;&gt; /// &lt;&lt;identifier&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_Identifier internal static string CSharpToken_Identifier
{ {
get => GetString("CSharpSymbol_Identifier"); get => GetString("CSharpToken_Identifier");
} }
/// <summary> /// <summary>
/// &lt;&lt;identifier&gt;&gt; /// &lt;&lt;identifier&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_Identifier() internal static string FormatCSharpToken_Identifier()
=> GetString("CSharpSymbol_Identifier"); => GetString("CSharpToken_Identifier");
/// <summary> /// <summary>
/// &lt;&lt;integer literal&gt;&gt; /// &lt;&lt;integer literal&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_IntegerLiteral internal static string CSharpToken_IntegerLiteral
{ {
get => GetString("CSharpSymbol_IntegerLiteral"); get => GetString("CSharpToken_IntegerLiteral");
} }
/// <summary> /// <summary>
/// &lt;&lt;integer literal&gt;&gt; /// &lt;&lt;integer literal&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_IntegerLiteral() internal static string FormatCSharpToken_IntegerLiteral()
=> GetString("CSharpSymbol_IntegerLiteral"); => GetString("CSharpToken_IntegerLiteral");
/// <summary> /// <summary>
/// &lt;&lt;keyword&gt;&gt; /// &lt;&lt;keyword&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_Keyword internal static string CSharpToken_Keyword
{ {
get => GetString("CSharpSymbol_Keyword"); get => GetString("CSharpToken_Keyword");
} }
/// <summary> /// <summary>
/// &lt;&lt;keyword&gt;&gt; /// &lt;&lt;keyword&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_Keyword() internal static string FormatCSharpToken_Keyword()
=> GetString("CSharpSymbol_Keyword"); => GetString("CSharpToken_Keyword");
/// <summary> /// <summary>
/// &lt;&lt;newline sequence&gt;&gt; /// &lt;&lt;newline sequence&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_Newline internal static string CSharpToken_Newline
{ {
get => GetString("CSharpSymbol_Newline"); get => GetString("CSharpToken_Newline");
} }
/// <summary> /// <summary>
/// &lt;&lt;newline sequence&gt;&gt; /// &lt;&lt;newline sequence&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_Newline() internal static string FormatCSharpToken_Newline()
=> GetString("CSharpSymbol_Newline"); => GetString("CSharpToken_Newline");
/// <summary> /// <summary>
/// &lt;&lt;real literal&gt;&gt; /// &lt;&lt;real literal&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_RealLiteral internal static string CSharpToken_RealLiteral
{ {
get => GetString("CSharpSymbol_RealLiteral"); get => GetString("CSharpToken_RealLiteral");
} }
/// <summary> /// <summary>
/// &lt;&lt;real literal&gt;&gt; /// &lt;&lt;real literal&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_RealLiteral() internal static string FormatCSharpToken_RealLiteral()
=> GetString("CSharpSymbol_RealLiteral"); => GetString("CSharpToken_RealLiteral");
/// <summary> /// <summary>
/// &lt;&lt;string literal&gt;&gt; /// &lt;&lt;string literal&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_StringLiteral internal static string CSharpToken_StringLiteral
{ {
get => GetString("CSharpSymbol_StringLiteral"); get => GetString("CSharpToken_StringLiteral");
} }
/// <summary> /// <summary>
/// &lt;&lt;string literal&gt;&gt; /// &lt;&lt;string literal&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_StringLiteral() internal static string FormatCSharpToken_StringLiteral()
=> GetString("CSharpSymbol_StringLiteral"); => GetString("CSharpToken_StringLiteral");
/// <summary> /// <summary>
/// &lt;&lt;white space&gt;&gt; /// &lt;&lt;white space&gt;&gt;
/// </summary> /// </summary>
internal static string CSharpSymbol_Whitespace internal static string CSharpToken_Whitespace
{ {
get => GetString("CSharpSymbol_Whitespace"); get => GetString("CSharpToken_Whitespace");
} }
/// <summary> /// <summary>
/// &lt;&lt;white space&gt;&gt; /// &lt;&lt;white space&gt;&gt;
/// </summary> /// </summary>
internal static string FormatCSharpSymbol_Whitespace() internal static string FormatCSharpToken_Whitespace()
=> GetString("CSharpSymbol_Whitespace"); => GetString("CSharpToken_Whitespace");
/// <summary> /// <summary>
/// The document type '{0}' does not support the extension '{1}'. /// The document type '{0}' does not support the extension '{1}'.
@ -489,58 +489,58 @@ namespace Microsoft.AspNetCore.Razor.Language
/// <summary> /// <summary>
/// &lt;&lt;newline sequence&gt;&gt; /// &lt;&lt;newline sequence&gt;&gt;
/// </summary> /// </summary>
internal static string HtmlSymbol_NewLine internal static string HtmlToken_NewLine
{ {
get => GetString("HtmlSymbol_NewLine"); get => GetString("HtmlToken_NewLine");
} }
/// <summary> /// <summary>
/// &lt;&lt;newline sequence&gt;&gt; /// &lt;&lt;newline sequence&gt;&gt;
/// </summary> /// </summary>
internal static string FormatHtmlSymbol_NewLine() internal static string FormatHtmlToken_NewLine()
=> GetString("HtmlSymbol_NewLine"); => GetString("HtmlToken_NewLine");
/// <summary> /// <summary>
/// &lt;&lt;razor comment&gt;&gt; /// &lt;&lt;razor comment&gt;&gt;
/// </summary> /// </summary>
internal static string HtmlSymbol_RazorComment internal static string HtmlToken_RazorComment
{ {
get => GetString("HtmlSymbol_RazorComment"); get => GetString("HtmlToken_RazorComment");
} }
/// <summary> /// <summary>
/// &lt;&lt;razor comment&gt;&gt; /// &lt;&lt;razor comment&gt;&gt;
/// </summary> /// </summary>
internal static string FormatHtmlSymbol_RazorComment() internal static string FormatHtmlToken_RazorComment()
=> GetString("HtmlSymbol_RazorComment"); => GetString("HtmlToken_RazorComment");
/// <summary> /// <summary>
/// &lt;&lt;text&gt;&gt; /// &lt;&lt;text&gt;&gt;
/// </summary> /// </summary>
internal static string HtmlSymbol_Text internal static string HtmlToken_Text
{ {
get => GetString("HtmlSymbol_Text"); get => GetString("HtmlToken_Text");
} }
/// <summary> /// <summary>
/// &lt;&lt;text&gt;&gt; /// &lt;&lt;text&gt;&gt;
/// </summary> /// </summary>
internal static string FormatHtmlSymbol_Text() internal static string FormatHtmlToken_Text()
=> GetString("HtmlSymbol_Text"); => GetString("HtmlToken_Text");
/// <summary> /// <summary>
/// &lt;&lt;white space&gt;&gt; /// &lt;&lt;white space&gt;&gt;
/// </summary> /// </summary>
internal static string HtmlSymbol_WhiteSpace internal static string HtmlToken_WhiteSpace
{ {
get => GetString("HtmlSymbol_WhiteSpace"); get => GetString("HtmlToken_WhiteSpace");
} }
/// <summary> /// <summary>
/// &lt;&lt;white space&gt;&gt; /// &lt;&lt;white space&gt;&gt;
/// </summary> /// </summary>
internal static string FormatHtmlSymbol_WhiteSpace() internal static string FormatHtmlToken_WhiteSpace()
=> GetString("HtmlSymbol_WhiteSpace"); => GetString("HtmlToken_WhiteSpace");
/// <summary> /// <summary>
/// Specify the base class for the current document. /// Specify the base class for the current document.
@ -711,7 +711,7 @@ namespace Microsoft.AspNetCore.Razor.Language
=> GetString("KeyMustNotBeNull"); => GetString("KeyMustNotBeNull");
/// <summary> /// <summary>
/// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known symbol types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment /// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known token types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
/// </summary> /// </summary>
internal static string Language_Does_Not_Support_RazorComment internal static string Language_Does_Not_Support_RazorComment
{ {
@ -719,7 +719,7 @@ namespace Microsoft.AspNetCore.Razor.Language
} }
/// <summary> /// <summary>
/// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known symbol types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment /// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known token types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
/// </summary> /// </summary>
internal static string FormatLanguage_Does_Not_Support_RazorComment() internal static string FormatLanguage_Does_Not_Support_RazorComment()
=> GetString("Language_Does_Not_Support_RazorComment"); => GetString("Language_Does_Not_Support_RazorComment");
@ -1351,16 +1351,16 @@ namespace Microsoft.AspNetCore.Razor.Language
/// <summary> /// <summary>
/// &lt;&lt;unknown&gt;&gt; /// &lt;&lt;unknown&gt;&gt;
/// </summary> /// </summary>
internal static string Symbol_Unknown internal static string Token_Unknown
{ {
get => GetString("Symbol_Unknown"); get => GetString("Token_Unknown");
} }
/// <summary> /// <summary>
/// &lt;&lt;unknown&gt;&gt; /// &lt;&lt;unknown&gt;&gt;
/// </summary> /// </summary>
internal static string FormatSymbol_Unknown() internal static string FormatToken_Unknown()
=> GetString("Symbol_Unknown"); => GetString("Token_Unknown");
/// <summary> /// <summary>
/// Invalid tag helper bound property '{1}' on tag helper '{0}'. Tag helpers cannot bind to HTML attributes with name '{2}' because the name contains a '{3}' character. /// Invalid tag helper bound property '{1}' on tag helper '{0}'. Tag helpers cannot bind to HTML attributes with name '{2}' because the name contains a '{3}' character.

30
src/Microsoft.AspNetCore.Razor.Language/Resources.resx
View File

@ -146,31 +146,31 @@
<data name="CodeWriter_InvalidNewLine" xml:space="preserve"> <data name="CodeWriter_InvalidNewLine" xml:space="preserve">
<value>Invalid newline sequence '{0}'. Support newline sequences are '\r\n' and '\n'.</value> <value>Invalid newline sequence '{0}'. Support newline sequences are '\r\n' and '\n'.</value>
</data> </data>
<data name="CSharpSymbol_CharacterLiteral" xml:space="preserve"> <data name="CSharpToken_CharacterLiteral" xml:space="preserve">
<value>&lt;&lt;character literal&gt;&gt;</value> <value>&lt;&lt;character literal&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_Comment" xml:space="preserve"> <data name="CSharpToken_Comment" xml:space="preserve">
<value>&lt;&lt;comment&gt;&gt;</value> <value>&lt;&lt;comment&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_Identifier" xml:space="preserve"> <data name="CSharpToken_Identifier" xml:space="preserve">
<value>&lt;&lt;identifier&gt;&gt;</value> <value>&lt;&lt;identifier&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_IntegerLiteral" xml:space="preserve"> <data name="CSharpToken_IntegerLiteral" xml:space="preserve">
<value>&lt;&lt;integer literal&gt;&gt;</value> <value>&lt;&lt;integer literal&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_Keyword" xml:space="preserve"> <data name="CSharpToken_Keyword" xml:space="preserve">
<value>&lt;&lt;keyword&gt;&gt;</value> <value>&lt;&lt;keyword&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_Newline" xml:space="preserve"> <data name="CSharpToken_Newline" xml:space="preserve">
<value>&lt;&lt;newline sequence&gt;&gt;</value> <value>&lt;&lt;newline sequence&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_RealLiteral" xml:space="preserve"> <data name="CSharpToken_RealLiteral" xml:space="preserve">
<value>&lt;&lt;real literal&gt;&gt;</value> <value>&lt;&lt;real literal&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_StringLiteral" xml:space="preserve"> <data name="CSharpToken_StringLiteral" xml:space="preserve">
<value>&lt;&lt;string literal&gt;&gt;</value> <value>&lt;&lt;string literal&gt;&gt;</value>
</data> </data>
<data name="CSharpSymbol_Whitespace" xml:space="preserve"> <data name="CSharpToken_Whitespace" xml:space="preserve">
<value>&lt;&lt;white space&gt;&gt;</value> <value>&lt;&lt;white space&gt;&gt;</value>
</data> </data>
<data name="Diagnostic_CodeTarget_UnsupportedExtension" xml:space="preserve"> <data name="Diagnostic_CodeTarget_UnsupportedExtension" xml:space="preserve">
@ -221,16 +221,16 @@
<data name="FunctionsDirective_Description" xml:space="preserve"> <data name="FunctionsDirective_Description" xml:space="preserve">
<value>Specify a C# code block.</value> <value>Specify a C# code block.</value>
</data> </data>
<data name="HtmlSymbol_NewLine" xml:space="preserve"> <data name="HtmlToken_NewLine" xml:space="preserve">
<value>&lt;&lt;newline sequence&gt;&gt;</value> <value>&lt;&lt;newline sequence&gt;&gt;</value>
</data> </data>
<data name="HtmlSymbol_RazorComment" xml:space="preserve"> <data name="HtmlToken_RazorComment" xml:space="preserve">
<value>&lt;&lt;razor comment&gt;&gt;</value> <value>&lt;&lt;razor comment&gt;&gt;</value>
</data> </data>
<data name="HtmlSymbol_Text" xml:space="preserve"> <data name="HtmlToken_Text" xml:space="preserve">
<value>&lt;&lt;text&gt;&gt;</value> <value>&lt;&lt;text&gt;&gt;</value>
</data> </data>
<data name="HtmlSymbol_WhiteSpace" xml:space="preserve"> <data name="HtmlToken_WhiteSpace" xml:space="preserve">
<value>&lt;&lt;white space&gt;&gt;</value> <value>&lt;&lt;white space&gt;&gt;</value>
</data> </data>
<data name="InheritsDirective_Description" xml:space="preserve"> <data name="InheritsDirective_Description" xml:space="preserve">
@ -270,7 +270,7 @@
<value>The key must not be null.</value> <value>The key must not be null.</value>
</data> </data>
<data name="Language_Does_Not_Support_RazorComment" xml:space="preserve"> <data name="Language_Does_Not_Support_RazorComment" xml:space="preserve">
<value>Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known symbol types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment</value> <value>Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known token types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment</value>
</data> </data>
<data name="MismatchedContentEncoding" xml:space="preserve"> <data name="MismatchedContentEncoding" xml:space="preserve">
<value>The specified encoding '{0}' does not match the content's encoding '{1}'.</value> <value>The specified encoding '{0}' does not match the content's encoding '{1}'.</value>
@ -423,7 +423,7 @@ Instead, wrap the contents of the block in "{{}}":
<value>@section Header { ... }</value> <value>@section Header { ... }</value>
<comment>In CSHTML, the @section keyword is case-sensitive and lowercase (as with all C# keywords)</comment> <comment>In CSHTML, the @section keyword is case-sensitive and lowercase (as with all C# keywords)</comment>
</data> </data>
<data name="Symbol_Unknown" xml:space="preserve"> <data name="Token_Unknown" xml:space="preserve">
<value>&lt;&lt;unknown&gt;&gt;</value> <value>&lt;&lt;unknown&gt;&gt;</value>
</data> </data>
<data name="TagHelper_InvalidBoundAttributeName" xml:space="preserve"> <data name="TagHelper_InvalidBoundAttributeName" xml:space="preserve">

15
src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netcore.json Normal file
View File

@ -0,0 +1,15 @@
[
{
"TypeId": "protected enum Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer<T0, T1>+RazorCommentTokenizerState where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase<T1> where T1 : struct",
"Kind": "Removal"
},
{
"TypeId": "protected struct Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer<T0, T1>+StateResult where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase<T1> where T1 : struct",
"Kind": "Removal"
},
{
"TypeId": "protected class Microsoft.AspNetCore.Razor.Language.Legacy.CSharpCodeParser+Block",
"MemberId": "public .ctor(Microsoft.AspNetCore.Razor.Language.Legacy.CSharpSymbol symbol, Microsoft.AspNetCore.Razor.Language.SourceLocation start)",
"Kind": "Removal"
}
]

15
src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netframework.json Normal file
View File

@ -0,0 +1,15 @@
[
{
"TypeId": "protected enum Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer<T0, T1>+RazorCommentTokenizerState where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase<T1> where T1 : struct",
"Kind": "Removal"
},
{
"TypeId": "protected struct Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer<T0, T1>+StateResult where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase<T1> where T1 : struct",
"Kind": "Removal"
},
{
"TypeId": "protected class Microsoft.AspNetCore.Razor.Language.Legacy.CSharpCodeParser+Block",
"MemberId": "public .ctor(Microsoft.AspNetCore.Razor.Language.Legacy.CSharpSymbol symbol, Microsoft.AspNetCore.Razor.Language.SourceLocation start)",
"Kind": "Removal"
}
]

4
src/Microsoft.VisualStudio.Editor.Razor/BraceSmartIndenter.cs
View File

@ -278,9 +278,9 @@ namespace Microsoft.VisualStudio.Editor.Razor
{ {
// We only support whitespace based content. Any non-whitespace content is an unkonwn to us // We only support whitespace based content. Any non-whitespace content is an unkonwn to us
// in regards to indentation. // in regards to indentation.
for (var i = 0; i < owner.Symbols.Count; i++) for (var i = 0; i < owner.Tokens.Count; i++)
{ {
if (!string.IsNullOrWhiteSpace(owner.Symbols[i].Content)) if (!string.IsNullOrWhiteSpace(owner.Tokens[i].Content))
{ {
return true; return true;
} }

6
src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
View File

@ -142,9 +142,9 @@ namespace Microsoft.VisualStudio.Editor.Razor
internal static bool IsCSharpOpenCurlyBrace(SyntaxTreeNode currentChild) internal static bool IsCSharpOpenCurlyBrace(SyntaxTreeNode currentChild)
{ {
return currentChild is Span currentSpan && return currentChild is Span currentSpan &&
currentSpan.Symbols.Count == 1 && currentSpan.Tokens.Count == 1 &&
currentSpan.Symbols[0] is CSharpSymbol symbol && currentSpan.Tokens[0] is CSharpToken symbol &&
symbol.Type == CSharpSymbolType.LeftBrace; symbol.Type == CSharpTokenType.LeftBrace;
} }
} }
} }

10
src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
View File

@ -162,7 +162,7 @@ namespace Microsoft.VisualStudio.Editor.Razor
} }
if (owner.ChunkGenerator is ExpressionChunkGenerator && if (owner.ChunkGenerator is ExpressionChunkGenerator &&
owner.Symbols.All(IsDirectiveCompletableSymbol) && owner.Tokens.All(IsDirectiveCompletableSymbol) &&
// Do not provide IntelliSense for explicit expressions. Explicit expressions will usually look like: // Do not provide IntelliSense for explicit expressions. Explicit expressions will usually look like:
// [@] [(] [DateTime.Now] [)] // [@] [(] [DateTime.Now] [)]
owner.Parent?.Children.Count > 1 && owner.Parent?.Children.Count > 1 &&
@ -204,16 +204,16 @@ namespace Microsoft.VisualStudio.Editor.Razor
return false; return false;
} }
private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.ISymbol symbol) private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.IToken symbol)
{ {
if (!(symbol is CSharpSymbol csharpSymbol)) if (!(symbol is CSharpToken csharpSymbol))
{ {
return false; return false;
} }
return csharpSymbol.Type == CSharpSymbolType.Identifier || return csharpSymbol.Type == CSharpTokenType.Identifier ||
// Marker symbol // Marker symbol
csharpSymbol.Type == CSharpSymbolType.Unknown; csharpSymbol.Type == CSharpTokenType.Unknown;
} }
} }
} }

4
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
View File

@ -13,7 +13,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Arrange // Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero); var spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlSymbol("hello", HtmlSymbolType.Text)); spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
var span = spanBuilder.Build(); var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder() var blockBuilder = new BlockBuilder()
{ {
@ -29,7 +29,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var parentBlock = blockBuilder.Build(); var parentBlock = blockBuilder.Build();
var originalBlockLength = parentBlock.Length; var originalBlockLength = parentBlock.Length;
spanBuilder = new SpanBuilder(SourceLocation.Zero); spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlSymbol("hi", HtmlSymbolType.Text)); spanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
span.ReplaceWith(spanBuilder); span.ReplaceWith(spanBuilder);
// Wire up parents now so we can re-trigger ChildChanged to cause cache refresh. // Wire up parents now so we can re-trigger ChildChanged to cause cache refresh.

6
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
View File

@ -8,13 +8,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public class CSharpLanguageCharacteristicsTest public class CSharpLanguageCharacteristicsTest
{ {
[Fact] [Fact]
public void GetSample_RightShiftAssign_ReturnsCorrectSymbol() public void GetSample_RightShiftAssign_ReturnsCorrectToken()
{ {
// Arrange & Act // Arrange & Act
var symbol = CSharpLanguageCharacteristics.Instance.GetSample(CSharpSymbolType.RightShiftAssign); var token = CSharpLanguageCharacteristics.Instance.GetSample(CSharpTokenType.RightShiftAssign);
// Assert // Assert
Assert.Equal(">>=", symbol); Assert.Equal(">>=", token);
} }
} }
} }

44
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
View File

@ -7,16 +7,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
{ {
private new CSharpSymbol IgnoreRemaining => (CSharpSymbol)base.IgnoreRemaining; private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
[Fact] [Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment() public void Next_Ignores_Star_At_EOF_In_RazorComment()
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *", "@* Foo * Bar * Baz *",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol(" Foo * Bar * Baz *", CSharpSymbolType.RazorComment)); new CSharpToken(" Foo * Bar * Baz *", CSharpTokenType.RazorComment));
} }
[Fact] [Fact]
@ -24,11 +24,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *@", "@* Foo * Bar * Baz *@",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol(" Foo * Bar * Baz ", CSharpSymbolType.RazorComment), new CSharpToken(" Foo * Bar * Baz ", CSharpTokenType.RazorComment),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition)); new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
} }
[Fact] [Fact]
@ -36,59 +36,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo Bar Baz *@", "@* Foo Bar Baz *@",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol(" Foo Bar Baz ", CSharpSymbolType.RazorComment), new CSharpToken(" Foo Bar Baz ", CSharpTokenType.RazorComment),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition)); new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
} }
[Fact] [Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment() public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{ {
TestTokenizer("// Foo Bar Baz", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment)); TestTokenizer("// Foo Bar Baz", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment));
} }
[Fact] [Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline() public void Single_Line_Comment_Is_Terminated_By_Newline()
{ {
TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("// Foo Bar Baz\na", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect() public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{ {
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol("// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpToken("// Foo/*Bar*/ Baz", CSharpTokenType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment() public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{ {
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment)); TestTokenizer("/* Foo\nBar\nBaz */", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment));
} }
[Fact] [Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence() public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{ {
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF() public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{ {
TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol("/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz", new CSharpToken("/* Foo\nBar\nBaz", CSharpTokenType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence() public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{ {
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol("/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpToken("/* Foo/*\nBar\nBaz*/", CSharpTokenType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence() public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{ {
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol("/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpToken("/* Foo\nBar\nBaz* */", CSharpTokenType.Comment), IgnoreRemaining);
} }
} }
} }

26
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
View File

@ -10,73 +10,73 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact] [Fact]
public void Simple_Identifier_Is_Recognized() public void Simple_Identifier_Is_Recognized()
{ {
TestTokenizer("foo", new CSharpSymbol("foo", CSharpSymbolType.Identifier)); TestTokenizer("foo", new CSharpToken("foo", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized() public void Identifier_Starting_With_Underscore_Is_Recognized()
{ {
TestTokenizer("_foo", new CSharpSymbol("_foo", CSharpSymbolType.Identifier)); TestTokenizer("_foo", new CSharpToken("_foo", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Digits() public void Identifier_Can_Contain_Digits()
{ {
TestTokenizer("foo4", new CSharpSymbol("foo4", CSharpSymbolType.Identifier)); TestTokenizer("foo4", new CSharpToken("foo4", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Titlecase_Letter() public void Identifier_Can_Start_With_Titlecase_Letter()
{ {
TestTokenizer("ῼfoo", new CSharpSymbol("ῼfoo", CSharpSymbolType.Identifier)); TestTokenizer("ῼfoo", new CSharpToken("ῼfoo", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Letter_Modifier() public void Identifier_Can_Start_With_Letter_Modifier()
{ {
TestTokenizer("ᵊfoo", new CSharpSymbol("ᵊfoo", CSharpSymbolType.Identifier)); TestTokenizer("ᵊfoo", new CSharpToken("ᵊfoo", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Other_Letter() public void Identifier_Can_Start_With_Other_Letter()
{ {
TestTokenizer("ƻfoo", new CSharpSymbol("ƻfoo", CSharpSymbolType.Identifier)); TestTokenizer("ƻfoo", new CSharpToken("ƻfoo", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Number_Letter() public void Identifier_Can_Start_With_Number_Letter()
{ {
TestTokenizer("Ⅽool", new CSharpSymbol("Ⅽool", CSharpSymbolType.Identifier)); TestTokenizer("Ⅽool", new CSharpToken("Ⅽool", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark() public void Identifier_Can_Contain_Non_Spacing_Mark()
{ {
TestTokenizer("foo\u0300", new CSharpSymbol("foo\u0300", CSharpSymbolType.Identifier)); TestTokenizer("foo\u0300", new CSharpToken("foo\u0300", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark() public void Identifier_Can_Contain_Spacing_Combining_Mark()
{ {
TestTokenizer("fooः", new CSharpSymbol("fooः", CSharpSymbolType.Identifier)); TestTokenizer("fooः", new CSharpToken("fooः", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Non_English_Digit() public void Identifier_Can_Contain_Non_English_Digit()
{ {
TestTokenizer("foo١", new CSharpSymbol("foo١", CSharpSymbolType.Identifier)); TestTokenizer("foo١", new CSharpToken("foo١", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Connector_Punctuation() public void Identifier_Can_Contain_Connector_Punctuation()
{ {
TestTokenizer("foo‿bar", new CSharpSymbol("foo‿bar", CSharpSymbolType.Identifier)); TestTokenizer("foo‿bar", new CSharpToken("foo‿bar", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Format_Character() public void Identifier_Can_Contain_Format_Character()
{ {
TestTokenizer("foo؃bar", new CSharpSymbol("foo؃bar", CSharpSymbolType.Identifier)); TestTokenizer("foo؃bar", new CSharpToken("foo؃bar", CSharpTokenType.Identifier));
} }
[Fact] [Fact]
@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType) private void TestKeyword(string keyword, CSharpKeyword keywordType)
{ {
TestTokenizer(keyword, new CSharpSymbol(keyword, CSharpSymbolType.Keyword) { Keyword = keywordType }); TestTokenizer(keyword, new CSharpToken(keyword, CSharpTokenType.Keyword) { Keyword = keywordType });
} }
} }
} }

160
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
View File

@ -8,280 +8,280 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase
{ {
private new CSharpSymbol IgnoreRemaining => (CSharpSymbol)base.IgnoreRemaining; private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
[Fact] [Fact]
public void Simple_Integer_Literal_Is_Recognized() public void Simple_Integer_Literal_Is_Recognized()
{ {
TestSingleToken("01189998819991197253", CSharpSymbolType.IntegerLiteral); TestSingleToken("01189998819991197253", CSharpTokenType.IntegerLiteral);
} }
[Fact] [Fact]
public void Integer_Type_Suffix_Is_Recognized() public void Integer_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("42U", CSharpSymbolType.IntegerLiteral); TestSingleToken("42U", CSharpTokenType.IntegerLiteral);
TestSingleToken("42u", CSharpSymbolType.IntegerLiteral); TestSingleToken("42u", CSharpTokenType.IntegerLiteral);
TestSingleToken("42L", CSharpSymbolType.IntegerLiteral); TestSingleToken("42L", CSharpTokenType.IntegerLiteral);
TestSingleToken("42l", CSharpSymbolType.IntegerLiteral); TestSingleToken("42l", CSharpTokenType.IntegerLiteral);
TestSingleToken("42UL", CSharpSymbolType.IntegerLiteral); TestSingleToken("42UL", CSharpTokenType.IntegerLiteral);
TestSingleToken("42Ul", CSharpSymbolType.IntegerLiteral); TestSingleToken("42Ul", CSharpTokenType.IntegerLiteral);
TestSingleToken("42uL", CSharpSymbolType.IntegerLiteral); TestSingleToken("42uL", CSharpTokenType.IntegerLiteral);
TestSingleToken("42ul", CSharpSymbolType.IntegerLiteral); TestSingleToken("42ul", CSharpTokenType.IntegerLiteral);
TestSingleToken("42LU", CSharpSymbolType.IntegerLiteral); TestSingleToken("42LU", CSharpTokenType.IntegerLiteral);
TestSingleToken("42Lu", CSharpSymbolType.IntegerLiteral); TestSingleToken("42Lu", CSharpTokenType.IntegerLiteral);
TestSingleToken("42lU", CSharpSymbolType.IntegerLiteral); TestSingleToken("42lU", CSharpTokenType.IntegerLiteral);
TestSingleToken("42lu", CSharpSymbolType.IntegerLiteral); TestSingleToken("42lu", CSharpTokenType.IntegerLiteral);
} }
[Fact] [Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix() public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{ {
TestTokenizer("42a", new CSharpSymbol("42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); TestTokenizer("42a", new CSharpToken("42", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Simple_Hex_Literal_Is_Recognized() public void Simple_Hex_Literal_Is_Recognized()
{ {
TestSingleToken("0x0123456789ABCDEF", CSharpSymbolType.IntegerLiteral); TestSingleToken("0x0123456789ABCDEF", CSharpTokenType.IntegerLiteral);
} }
[Fact] [Fact]
public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal() public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal()
{ {
TestSingleToken("0xDEADBEEFU", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFU", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFu", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFu", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFL", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFL", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFl", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFl", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFUL", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFUL", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFUl", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFUl", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFuL", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFuL", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFul", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFul", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFLU", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFLU", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFLu", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFLu", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFlU", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFlU", CSharpTokenType.IntegerLiteral);
TestSingleToken("0xDEADBEEFlu", CSharpSymbolType.IntegerLiteral); TestSingleToken("0xDEADBEEFlu", CSharpTokenType.IntegerLiteral);
} }
[Fact] [Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix() public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{ {
TestTokenizer("0xDEADBEEFz", new CSharpSymbol("0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); TestTokenizer("0xDEADBEEFz", new CSharpToken("0xDEADBEEF", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal() public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{ {
TestTokenizer("3.a", new CSharpSymbol("3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); TestTokenizer("3.a", new CSharpToken("3", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Simple_Real_Literal_Is_Recognized() public void Simple_Real_Literal_Is_Recognized()
{ {
TestTokenizer("3.14159", new CSharpSymbol("3.14159", CSharpSymbolType.RealLiteral)); TestTokenizer("3.14159", new CSharpToken("3.14159", CSharpTokenType.RealLiteral));
} }
[Fact] [Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized() public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{ {
TestTokenizer(".14159", new CSharpSymbol(".14159", CSharpSymbolType.RealLiteral)); TestTokenizer(".14159", new CSharpToken(".14159", CSharpTokenType.RealLiteral));
} }
[Fact] [Fact]
public void Integer_With_Real_Type_Suffix_Is_Recognized() public void Integer_With_Real_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("42F", CSharpSymbolType.RealLiteral); TestSingleToken("42F", CSharpTokenType.RealLiteral);
TestSingleToken("42f", CSharpSymbolType.RealLiteral); TestSingleToken("42f", CSharpTokenType.RealLiteral);
TestSingleToken("42D", CSharpSymbolType.RealLiteral); TestSingleToken("42D", CSharpTokenType.RealLiteral);
TestSingleToken("42d", CSharpSymbolType.RealLiteral); TestSingleToken("42d", CSharpTokenType.RealLiteral);
TestSingleToken("42M", CSharpSymbolType.RealLiteral); TestSingleToken("42M", CSharpTokenType.RealLiteral);
TestSingleToken("42m", CSharpSymbolType.RealLiteral); TestSingleToken("42m", CSharpTokenType.RealLiteral);
} }
[Fact] [Fact]
public void Integer_With_Exponent_Is_Recognized() public void Integer_With_Exponent_Is_Recognized()
{ {
TestSingleToken("1e10", CSharpSymbolType.RealLiteral); TestSingleToken("1e10", CSharpTokenType.RealLiteral);
TestSingleToken("1E10", CSharpSymbolType.RealLiteral); TestSingleToken("1E10", CSharpTokenType.RealLiteral);
TestSingleToken("1e+10", CSharpSymbolType.RealLiteral); TestSingleToken("1e+10", CSharpTokenType.RealLiteral);
TestSingleToken("1E+10", CSharpSymbolType.RealLiteral); TestSingleToken("1E+10", CSharpTokenType.RealLiteral);
TestSingleToken("1e-10", CSharpSymbolType.RealLiteral); TestSingleToken("1e-10", CSharpTokenType.RealLiteral);
TestSingleToken("1E-10", CSharpSymbolType.RealLiteral); TestSingleToken("1E-10", CSharpTokenType.RealLiteral);
} }
[Fact] [Fact]
public void Real_Number_With_Type_Suffix_Is_Recognized() public void Real_Number_With_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("3.14F", CSharpSymbolType.RealLiteral); TestSingleToken("3.14F", CSharpTokenType.RealLiteral);
TestSingleToken("3.14f", CSharpSymbolType.RealLiteral); TestSingleToken("3.14f", CSharpTokenType.RealLiteral);
TestSingleToken("3.14D", CSharpSymbolType.RealLiteral); TestSingleToken("3.14D", CSharpTokenType.RealLiteral);
TestSingleToken("3.14d", CSharpSymbolType.RealLiteral); TestSingleToken("3.14d", CSharpTokenType.RealLiteral);
TestSingleToken("3.14M", CSharpSymbolType.RealLiteral); TestSingleToken("3.14M", CSharpTokenType.RealLiteral);
TestSingleToken("3.14m", CSharpSymbolType.RealLiteral); TestSingleToken("3.14m", CSharpTokenType.RealLiteral);
} }
[Fact] [Fact]
public void Real_Number_With_Exponent_Is_Recognized() public void Real_Number_With_Exponent_Is_Recognized()
{ {
TestSingleToken("3.14E10", CSharpSymbolType.RealLiteral); TestSingleToken("3.14E10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14e10", CSharpSymbolType.RealLiteral); TestSingleToken("3.14e10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14E+10", CSharpSymbolType.RealLiteral); TestSingleToken("3.14E+10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14e+10", CSharpSymbolType.RealLiteral); TestSingleToken("3.14e+10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14E-10", CSharpSymbolType.RealLiteral); TestSingleToken("3.14E-10", CSharpTokenType.RealLiteral);
TestSingleToken("3.14e-10", CSharpSymbolType.RealLiteral); TestSingleToken("3.14e-10", CSharpTokenType.RealLiteral);
} }
[Fact] [Fact]
public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized() public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized()
{ {
TestSingleToken("3.14E+10F", CSharpSymbolType.RealLiteral); TestSingleToken("3.14E+10F", CSharpTokenType.RealLiteral);
} }
[Fact] [Fact]
public void Single_Character_Literal_Is_Recognized() public void Single_Character_Literal_Is_Recognized()
{ {
TestSingleToken("'f'", CSharpSymbolType.CharacterLiteral); TestSingleToken("'f'", CSharpTokenType.CharacterLiteral);
} }
[Fact] [Fact]
public void Multi_Character_Literal_Is_Recognized() public void Multi_Character_Literal_Is_Recognized()
{ {
TestSingleToken("'foo'", CSharpSymbolType.CharacterLiteral); TestSingleToken("'foo'", CSharpTokenType.CharacterLiteral);
} }
[Fact] [Fact]
public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated() public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated()
{ {
TestSingleToken("'foo bar", CSharpSymbolType.CharacterLiteral); TestSingleToken("'foo bar", CSharpTokenType.CharacterLiteral);
} }
[Fact] [Fact]
public void Character_Literal_Not_Terminated_By_Escaped_Quote() public void Character_Literal_Not_Terminated_By_Escaped_Quote()
{ {
TestSingleToken("'foo\\'bar'", CSharpSymbolType.CharacterLiteral); TestSingleToken("'foo\\'bar'", CSharpTokenType.CharacterLiteral);
} }
[Fact] [Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated() public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{ {
TestTokenizer("'foo\n", new CSharpSymbol("'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\n", new CSharpToken("'foo", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("'foo\\\n", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\n", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer("'foo\\\nflarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\nflarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\" + Environment.NewLine, new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Allows_Escaped_Escape() public void Character_Literal_Allows_Escaped_Escape()
{ {
TestTokenizer("'foo\\\\'blah", new CSharpSymbol("'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\\'blah", new CSharpToken("'foo\\\\'", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Is_Recognized() public void String_Literal_Is_Recognized()
{ {
TestSingleToken("\"foo\"", CSharpSymbolType.StringLiteral); TestSingleToken("\"foo\"", CSharpTokenType.StringLiteral);
} }
[Fact] [Fact]
public void String_Literal_Is_Terminated_By_EOF_If_Unterminated() public void String_Literal_Is_Terminated_By_EOF_If_Unterminated()
{ {
TestSingleToken("\"foo bar", CSharpSymbolType.StringLiteral); TestSingleToken("\"foo bar", CSharpTokenType.StringLiteral);
} }
[Fact] [Fact]
public void String_Literal_Not_Terminated_By_Escaped_Quote() public void String_Literal_Not_Terminated_By_Escaped_Quote()
{ {
TestSingleToken("\"foo\\\"bar\"", CSharpSymbolType.StringLiteral); TestSingleToken("\"foo\\\"bar\"", CSharpTokenType.StringLiteral);
} }
[Fact] [Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated() public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{ {
TestTokenizer("\"foo\n", new CSharpSymbol("\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\n", new CSharpToken("\"foo", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("\"foo\\\n", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\n", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer("\"foo\\\nflarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\nflarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Allows_Escaped_Escape() public void String_Literal_Allows_Escaped_Escape()
{ {
TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol("\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\\\"blah", new CSharpToken("\"foo\\\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Can_Contain_Newlines() public void Verbatim_String_Literal_Can_Contain_Newlines()
{ {
TestSingleToken("@\"foo\nbar\nbaz\"", CSharpSymbolType.StringLiteral); TestSingleToken("@\"foo\nbar\nbaz\"", CSharpTokenType.StringLiteral);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote() public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote()
{ {
TestSingleToken("@\"foo\"\"bar\"", CSharpSymbolType.StringLiteral); TestSingleToken("@\"foo\"\"bar\"", CSharpTokenType.StringLiteral);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote() public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{ {
TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol("@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("@\"foo\\\"bar\"", new CSharpToken("@\"foo\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Verbatim_String_Literal_Is_Terminated_By_EOF() public void Verbatim_String_Literal_Is_Terminated_By_EOF()
{ {
TestSingleToken("@\"foo", CSharpSymbolType.StringLiteral); TestSingleToken("@\"foo", CSharpTokenType.StringLiteral);
} }
} }
} }

98
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
View File

@ -10,287 +10,287 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact] [Fact]
public void LeftBrace_Is_Recognized() public void LeftBrace_Is_Recognized()
{ {
TestSingleToken("{", CSharpSymbolType.LeftBrace); TestSingleToken("{", CSharpTokenType.LeftBrace);
} }
[Fact] [Fact]
public void Plus_Is_Recognized() public void Plus_Is_Recognized()
{ {
TestSingleToken("+", CSharpSymbolType.Plus); TestSingleToken("+", CSharpTokenType.Plus);
} }
[Fact] [Fact]
public void Assign_Is_Recognized() public void Assign_Is_Recognized()
{ {
TestSingleToken("=", CSharpSymbolType.Assign); TestSingleToken("=", CSharpTokenType.Assign);
} }
[Fact] [Fact]
public void Arrow_Is_Recognized() public void Arrow_Is_Recognized()
{ {
TestSingleToken("->", CSharpSymbolType.Arrow); TestSingleToken("->", CSharpTokenType.Arrow);
} }
[Fact] [Fact]
public void AndAssign_Is_Recognized() public void AndAssign_Is_Recognized()
{ {
TestSingleToken("&=", CSharpSymbolType.AndAssign); TestSingleToken("&=", CSharpTokenType.AndAssign);
} }
[Fact] [Fact]
public void RightBrace_Is_Recognized() public void RightBrace_Is_Recognized()
{ {
TestSingleToken("}", CSharpSymbolType.RightBrace); TestSingleToken("}", CSharpTokenType.RightBrace);
} }
[Fact] [Fact]
public void Minus_Is_Recognized() public void Minus_Is_Recognized()
{ {
TestSingleToken("-", CSharpSymbolType.Minus); TestSingleToken("-", CSharpTokenType.Minus);
} }
[Fact] [Fact]
public void LessThan_Is_Recognized() public void LessThan_Is_Recognized()
{ {
TestSingleToken("<", CSharpSymbolType.LessThan); TestSingleToken("<", CSharpTokenType.LessThan);
} }
[Fact] [Fact]
public void Equals_Is_Recognized() public void Equals_Is_Recognized()
{ {
TestSingleToken("==", CSharpSymbolType.Equals); TestSingleToken("==", CSharpTokenType.Equals);
} }
[Fact] [Fact]
public void OrAssign_Is_Recognized() public void OrAssign_Is_Recognized()
{ {
TestSingleToken("|=", CSharpSymbolType.OrAssign); TestSingleToken("|=", CSharpTokenType.OrAssign);
} }
[Fact] [Fact]
public void LeftBracket_Is_Recognized() public void LeftBracket_Is_Recognized()
{ {
TestSingleToken("[", CSharpSymbolType.LeftBracket); TestSingleToken("[", CSharpTokenType.LeftBracket);
} }
[Fact] [Fact]
public void Star_Is_Recognized() public void Star_Is_Recognized()
{ {
TestSingleToken("*", CSharpSymbolType.Star); TestSingleToken("*", CSharpTokenType.Star);
} }
[Fact] [Fact]
public void GreaterThan_Is_Recognized() public void GreaterThan_Is_Recognized()
{ {
TestSingleToken(">", CSharpSymbolType.GreaterThan); TestSingleToken(">", CSharpTokenType.GreaterThan);
} }
[Fact] [Fact]
public void NotEqual_Is_Recognized() public void NotEqual_Is_Recognized()
{ {
TestSingleToken("!=", CSharpSymbolType.NotEqual); TestSingleToken("!=", CSharpTokenType.NotEqual);
} }
[Fact] [Fact]
public void XorAssign_Is_Recognized() public void XorAssign_Is_Recognized()
{ {
TestSingleToken("^=", CSharpSymbolType.XorAssign); TestSingleToken("^=", CSharpTokenType.XorAssign);
} }
[Fact] [Fact]
public void RightBracket_Is_Recognized() public void RightBracket_Is_Recognized()
{ {
TestSingleToken("]", CSharpSymbolType.RightBracket); TestSingleToken("]", CSharpTokenType.RightBracket);
} }
[Fact] [Fact]
public void Slash_Is_Recognized() public void Slash_Is_Recognized()
{ {
TestSingleToken("/", CSharpSymbolType.Slash); TestSingleToken("/", CSharpTokenType.Slash);
} }
[Fact] [Fact]
public void QuestionMark_Is_Recognized() public void QuestionMark_Is_Recognized()
{ {
TestSingleToken("?", CSharpSymbolType.QuestionMark); TestSingleToken("?", CSharpTokenType.QuestionMark);
} }
[Fact] [Fact]
public void LessThanEqual_Is_Recognized() public void LessThanEqual_Is_Recognized()
{ {
TestSingleToken("<=", CSharpSymbolType.LessThanEqual); TestSingleToken("<=", CSharpTokenType.LessThanEqual);
} }
[Fact] [Fact]
public void LeftShift_Is_Not_Specially_Recognized() public void LeftShift_Is_Not_Specially_Recognized()
{ {
TestTokenizer("<<", TestTokenizer("<<",
new CSharpSymbol("<", CSharpSymbolType.LessThan), new CSharpToken("<", CSharpTokenType.LessThan),
new CSharpSymbol("<", CSharpSymbolType.LessThan)); new CSharpToken("<", CSharpTokenType.LessThan));
} }
[Fact] [Fact]
public void LeftParen_Is_Recognized() public void LeftParen_Is_Recognized()
{ {
TestSingleToken("(", CSharpSymbolType.LeftParenthesis); TestSingleToken("(", CSharpTokenType.LeftParenthesis);
} }
[Fact] [Fact]
public void Modulo_Is_Recognized() public void Modulo_Is_Recognized()
{ {
TestSingleToken("%", CSharpSymbolType.Modulo); TestSingleToken("%", CSharpTokenType.Modulo);
} }
[Fact] [Fact]
public void NullCoalesce_Is_Recognized() public void NullCoalesce_Is_Recognized()
{ {
TestSingleToken("??", CSharpSymbolType.NullCoalesce); TestSingleToken("??", CSharpTokenType.NullCoalesce);
} }
[Fact] [Fact]
public void GreaterThanEqual_Is_Recognized() public void GreaterThanEqual_Is_Recognized()
{ {
TestSingleToken(">=", CSharpSymbolType.GreaterThanEqual); TestSingleToken(">=", CSharpTokenType.GreaterThanEqual);
} }
[Fact] [Fact]
public void EqualGreaterThan_Is_Recognized() public void EqualGreaterThan_Is_Recognized()
{ {
TestSingleToken("=>", CSharpSymbolType.GreaterThanEqual); TestSingleToken("=>", CSharpTokenType.GreaterThanEqual);
} }
[Fact] [Fact]
public void RightParen_Is_Recognized() public void RightParen_Is_Recognized()
{ {
TestSingleToken(")", CSharpSymbolType.RightParenthesis); TestSingleToken(")", CSharpTokenType.RightParenthesis);
} }
[Fact] [Fact]
public void And_Is_Recognized() public void And_Is_Recognized()
{ {
TestSingleToken("&", CSharpSymbolType.And); TestSingleToken("&", CSharpTokenType.And);
} }
[Fact] [Fact]
public void DoubleColon_Is_Recognized() public void DoubleColon_Is_Recognized()
{ {
TestSingleToken("::", CSharpSymbolType.DoubleColon); TestSingleToken("::", CSharpTokenType.DoubleColon);
} }
[Fact] [Fact]
public void PlusAssign_Is_Recognized() public void PlusAssign_Is_Recognized()
{ {
TestSingleToken("+=", CSharpSymbolType.PlusAssign); TestSingleToken("+=", CSharpTokenType.PlusAssign);
} }
[Fact] [Fact]
public void Semicolon_Is_Recognized() public void Semicolon_Is_Recognized()
{ {
TestSingleToken(";", CSharpSymbolType.Semicolon); TestSingleToken(";", CSharpTokenType.Semicolon);
} }
[Fact] [Fact]
public void Tilde_Is_Recognized() public void Tilde_Is_Recognized()
{ {
TestSingleToken("~", CSharpSymbolType.Tilde); TestSingleToken("~", CSharpTokenType.Tilde);
} }
[Fact] [Fact]
public void DoubleOr_Is_Recognized() public void DoubleOr_Is_Recognized()
{ {
TestSingleToken("||", CSharpSymbolType.DoubleOr); TestSingleToken("||", CSharpTokenType.DoubleOr);
} }
[Fact] [Fact]
public void ModuloAssign_Is_Recognized() public void ModuloAssign_Is_Recognized()
{ {
TestSingleToken("%=", CSharpSymbolType.ModuloAssign); TestSingleToken("%=", CSharpTokenType.ModuloAssign);
} }
[Fact] [Fact]
public void Colon_Is_Recognized() public void Colon_Is_Recognized()
{ {
TestSingleToken(":", CSharpSymbolType.Colon); TestSingleToken(":", CSharpTokenType.Colon);
} }
[Fact] [Fact]
public void Not_Is_Recognized() public void Not_Is_Recognized()
{ {
TestSingleToken("!", CSharpSymbolType.Not); TestSingleToken("!", CSharpTokenType.Not);
} }
[Fact] [Fact]
public void DoubleAnd_Is_Recognized() public void DoubleAnd_Is_Recognized()
{ {
TestSingleToken("&&", CSharpSymbolType.DoubleAnd); TestSingleToken("&&", CSharpTokenType.DoubleAnd);
} }
[Fact] [Fact]
public void DivideAssign_Is_Recognized() public void DivideAssign_Is_Recognized()
{ {
TestSingleToken("/=", CSharpSymbolType.DivideAssign); TestSingleToken("/=", CSharpTokenType.DivideAssign);
} }
[Fact] [Fact]
public void Comma_Is_Recognized() public void Comma_Is_Recognized()
{ {
TestSingleToken(",", CSharpSymbolType.Comma); TestSingleToken(",", CSharpTokenType.Comma);
} }
[Fact] [Fact]
public void Xor_Is_Recognized() public void Xor_Is_Recognized()
{ {
TestSingleToken("^", CSharpSymbolType.Xor); TestSingleToken("^", CSharpTokenType.Xor);
} }
[Fact] [Fact]
public void Decrement_Is_Recognized() public void Decrement_Is_Recognized()
{ {
TestSingleToken("--", CSharpSymbolType.Decrement); TestSingleToken("--", CSharpTokenType.Decrement);
} }
[Fact] [Fact]
public void MultiplyAssign_Is_Recognized() public void MultiplyAssign_Is_Recognized()
{ {
TestSingleToken("*=", CSharpSymbolType.MultiplyAssign); TestSingleToken("*=", CSharpTokenType.MultiplyAssign);
} }
[Fact] [Fact]
public void Dot_Is_Recognized() public void Dot_Is_Recognized()
{ {
TestSingleToken(".", CSharpSymbolType.Dot); TestSingleToken(".", CSharpTokenType.Dot);
} }
[Fact] [Fact]
public void Or_Is_Recognized() public void Or_Is_Recognized()
{ {
TestSingleToken("|", CSharpSymbolType.Or); TestSingleToken("|", CSharpTokenType.Or);
} }
[Fact] [Fact]
public void Increment_Is_Recognized() public void Increment_Is_Recognized()
{ {
TestSingleToken("++", CSharpSymbolType.Increment); TestSingleToken("++", CSharpTokenType.Increment);
} }
[Fact] [Fact]
public void MinusAssign_Is_Recognized() public void MinusAssign_Is_Recognized()
{ {
TestSingleToken("-=", CSharpSymbolType.MinusAssign); TestSingleToken("-=", CSharpTokenType.MinusAssign);
} }
[Fact] [Fact]
public void RightShift_Is_Not_Specially_Recognized() public void RightShift_Is_Not_Specially_Recognized()
{ {
TestTokenizer(">>", TestTokenizer(">>",
new CSharpSymbol(">", CSharpSymbolType.GreaterThan), new CSharpToken(">", CSharpTokenType.GreaterThan),
new CSharpSymbol(">", CSharpSymbolType.GreaterThan)); new CSharpToken(">", CSharpTokenType.GreaterThan));
} }
[Fact] [Fact]
public void Hash_Is_Recognized() public void Hash_Is_Recognized()
{ {
TestSingleToken("#", CSharpSymbolType.Hash); TestSingleToken("#", CSharpTokenType.Hash);
} }
} }
} }

38
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
View File

@ -7,7 +7,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public class CSharpTokenizerTest : CSharpTokenizerTestBase public class CSharpTokenizerTest : CSharpTokenizerTestBase
{ {
private new CSharpSymbol IgnoreRemaining => (CSharpSymbol)base.IgnoreRemaining; private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
[Fact] [Fact]
public void Next_Returns_Null_When_EOF_Reached() public void Next_Returns_Null_When_EOF_Reached()
@ -20,8 +20,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\ra", "\r\ra",
new CSharpSymbol("\r", CSharpSymbolType.NewLine), new CSharpToken("\r", CSharpTokenType.NewLine),
new CSharpSymbol("\r", CSharpSymbolType.NewLine), new CSharpToken("\r", CSharpTokenType.NewLine),
IgnoreRemaining); IgnoreRemaining);
} }
@ -30,8 +30,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\n\na", "\n\na",
new CSharpSymbol("\n", CSharpSymbolType.NewLine), new CSharpToken("\n", CSharpTokenType.NewLine),
new CSharpSymbol("\n", CSharpSymbolType.NewLine), new CSharpToken("\n", CSharpTokenType.NewLine),
IgnoreRemaining); IgnoreRemaining);
} }
@ -41,8 +41,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// NEL: Unicode "Next Line" U+0085 // NEL: Unicode "Next Line" U+0085
TestTokenizer( TestTokenizer(
"\u0085\u0085a", "\u0085\u0085a",
new CSharpSymbol("\u0085", CSharpSymbolType.NewLine), new CSharpToken("\u0085", CSharpTokenType.NewLine),
new CSharpSymbol("\u0085", CSharpSymbolType.NewLine), new CSharpToken("\u0085", CSharpTokenType.NewLine),
IgnoreRemaining); IgnoreRemaining);
} }
@ -52,8 +52,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Line Separator" U+2028 // Unicode "Line Separator" U+2028
TestTokenizer( TestTokenizer(
"\u2028\u2028a", "\u2028\u2028a",
new CSharpSymbol("\u2028", CSharpSymbolType.NewLine), new CSharpToken("\u2028", CSharpTokenType.NewLine),
new CSharpSymbol("\u2028", CSharpSymbolType.NewLine), new CSharpToken("\u2028", CSharpTokenType.NewLine),
IgnoreRemaining); IgnoreRemaining);
} }
@ -63,8 +63,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Paragraph Separator" U+2029 // Unicode "Paragraph Separator" U+2029
TestTokenizer( TestTokenizer(
"\u2029\u2029a", "\u2029\u2029a",
new CSharpSymbol("\u2029", CSharpSymbolType.NewLine), new CSharpToken("\u2029", CSharpTokenType.NewLine),
new CSharpSymbol("\u2029", CSharpSymbolType.NewLine), new CSharpToken("\u2029", CSharpTokenType.NewLine),
IgnoreRemaining); IgnoreRemaining);
} }
@ -73,8 +73,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n\r\na", "\r\n\r\na",
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), new CSharpToken("\r\n", CSharpTokenType.NewLine),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), new CSharpToken("\r\n", CSharpTokenType.NewLine),
IgnoreRemaining); IgnoreRemaining);
} }
@ -83,15 +83,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
" \f\t\u000B \n ", " \f\t\u000B \n ",
new CSharpSymbol(" \f\t\u000B ", CSharpSymbolType.WhiteSpace), new CSharpToken(" \f\t\u000B ", CSharpTokenType.WhiteSpace),
new CSharpSymbol("\n", CSharpSymbolType.NewLine), new CSharpToken("\n", CSharpTokenType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace)); new CSharpToken(" ", CSharpTokenType.WhiteSpace));
} }
[Fact] [Fact]
public void Transition_Is_Recognized() public void Transition_Is_Recognized()
{ {
TestSingleToken("@", CSharpSymbolType.Transition); TestSingleToken("@", CSharpTokenType.Transition);
} }
[Fact] [Fact]
@ -99,8 +99,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@(", "@(",
new CSharpSymbol("@", CSharpSymbolType.Transition), new CSharpToken("@", CSharpTokenType.Transition),
new CSharpSymbol("(", CSharpSymbolType.LeftParenthesis)); new CSharpToken("(", CSharpTokenType.LeftParenthesis));
} }
} }
} }

10
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
View File

@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public abstract class CSharpTokenizerTestBase : TokenizerTestBase public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{ {
private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown); private static CSharpToken _ignoreRemaining = new CSharpToken(string.Empty, CSharpTokenType.Unknown);
internal override object IgnoreRemaining internal override object IgnoreRemaining
{ {
@ -17,14 +17,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source); return new CSharpTokenizer(source);
} }
internal void TestSingleToken(string text, CSharpSymbolType expectedSymbolType) internal void TestSingleToken(string text, CSharpTokenType expectedTokenType)
{ {
TestTokenizer(text, new CSharpSymbol(text, expectedSymbolType)); TestTokenizer(text, new CSharpToken(text, expectedTokenType));
} }
internal void TestTokenizer(string input, params CSharpSymbol[] expectedSymbols) internal void TestTokenizer(string input, params CSharpToken[] expectedTokens)
{ {
base.TestTokenizer<CSharpSymbol, CSharpSymbolType>(input, expectedSymbols); base.TestTokenizer<CSharpToken, CSharpTokenType>(input, expectedTokens);
} }
} }
} }

36
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
View File

@ -12,12 +12,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n @something \r\n @this is ignored", "\r\n @something \r\n @this is ignored",
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), new CSharpToken("\r\n", CSharpTokenType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpSymbol("@", CSharpSymbolType.Transition), new CSharpToken("@", CSharpTokenType.Transition),
new CSharpSymbol("something", CSharpSymbolType.Identifier), new CSharpToken("something", CSharpTokenType.Identifier),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine)); new CSharpToken("\r\n", CSharpTokenType.NewLine));
} }
[Fact] [Fact]
@ -25,18 +25,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@*included*@\r\n @something \"value\"\r\n @this is ignored", "@*included*@\r\n @something \"value\"\r\n @this is ignored",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol("included", CSharpSymbolType.RazorComment), new CSharpToken("included", CSharpTokenType.RazorComment),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), new CSharpToken("*", CSharpTokenType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), new CSharpToken("\r\n", CSharpTokenType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpSymbol("@", CSharpSymbolType.Transition), new CSharpToken("@", CSharpTokenType.Transition),
new CSharpSymbol("something", CSharpSymbolType.Identifier), new CSharpToken("something", CSharpTokenType.Identifier),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace), new CSharpToken(" ", CSharpTokenType.WhiteSpace),
new CSharpSymbol("\"value\"", CSharpSymbolType.StringLiteral), new CSharpToken("\"value\"", CSharpTokenType.StringLiteral),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine)); new CSharpToken("\r\n", CSharpTokenType.NewLine));
} }
internal override object CreateTokenizer(ITextDocument source) internal override object CreateTokenizer(ITextDocument source)

24
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
View File

@ -12,9 +12,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n <div>Ignored</div>", "\r\n <div>Ignored</div>",
new HtmlSymbol("\r\n", HtmlSymbolType.NewLine), new HtmlToken("\r\n", HtmlTokenType.NewLine),
new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace), new HtmlToken(" ", HtmlTokenType.WhiteSpace),
new HtmlSymbol("<", HtmlSymbolType.OpenAngle)); new HtmlToken("<", HtmlTokenType.OpenAngle));
} }
[Fact] [Fact]
@ -22,15 +22,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"\r\n @*included*@ <div>Ignored</div>", "\r\n @*included*@ <div>Ignored</div>",
new HtmlSymbol("\r\n", HtmlSymbolType.NewLine), new HtmlToken("\r\n", HtmlTokenType.NewLine),
new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace), new HtmlToken(" ", HtmlTokenType.WhiteSpace),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol("included", HtmlSymbolType.RazorComment), new HtmlToken("included", HtmlTokenType.RazorComment),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace), new HtmlToken(" ", HtmlTokenType.WhiteSpace),
new HtmlSymbol("<", HtmlSymbolType.OpenAngle)); new HtmlToken("<", HtmlTokenType.OpenAngle));
} }
internal override object CreateTokenizer(ITextDocument source) internal override object CreateTokenizer(ITextDocument source)

68
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
View File

@ -8,69 +8,69 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
{ {
public class HtmlMarkupParserTests public class HtmlMarkupParserTests
{ {
private static readonly HtmlSymbol doubleHyphenSymbol = new HtmlSymbol("--", HtmlSymbolType.DoubleHyphen); private static readonly HtmlToken doubleHyphenToken = new HtmlToken("--", HtmlTokenType.DoubleHyphen);
public static IEnumerable<object[]> NonDashSymbols public static IEnumerable<object[]> NonDashTokens
{ {
get get
{ {
yield return new[] { new HtmlSymbol("--", HtmlSymbolType.DoubleHyphen) }; yield return new[] { new HtmlToken("--", HtmlTokenType.DoubleHyphen) };
yield return new[] { new HtmlSymbol("asdf", HtmlSymbolType.Text) }; yield return new[] { new HtmlToken("asdf", HtmlTokenType.Text) };
yield return new[] { new HtmlSymbol(">", HtmlSymbolType.CloseAngle) }; yield return new[] { new HtmlToken(">", HtmlTokenType.CloseAngle) };
yield return new[] { new HtmlSymbol("<", HtmlSymbolType.OpenAngle) }; yield return new[] { new HtmlToken("<", HtmlTokenType.OpenAngle) };
yield return new[] { new HtmlSymbol("!", HtmlSymbolType.Bang) }; yield return new[] { new HtmlToken("!", HtmlTokenType.Bang) };
} }
} }
[Theory] [Theory]
[MemberData(nameof(NonDashSymbols))] [MemberData(nameof(NonDashTokens))]
public void IsHyphen_ReturnsFalseForNonDashSymbol(object symbol) public void IsHyphen_ReturnsFalseForNonDashToken(object token)
{ {
// Arrange // Arrange
var convertedSymbol = (HtmlSymbol)symbol; var convertedToken = (HtmlToken)token;
// Act & Assert // Act & Assert
Assert.False(HtmlMarkupParser.IsHyphen(convertedSymbol)); Assert.False(HtmlMarkupParser.IsHyphen(convertedToken));
} }
[Fact] [Fact]
public void IsHyphen_ReturnsTrueForADashSymbol() public void IsHyphen_ReturnsTrueForADashToken()
{ {
// Arrange // Arrange
var dashSymbol = new HtmlSymbol("-", HtmlSymbolType.Text); var dashToken = new HtmlToken("-", HtmlTokenType.Text);
// Act & Assert // Act & Assert
Assert.True(HtmlMarkupParser.IsHyphen(dashSymbol)); Assert.True(HtmlMarkupParser.IsHyphen(dashToken));
} }
[Fact] [Fact]
public void AcceptAllButLastDoubleHypens_ReturnsTheOnlyDoubleHyphenSymbol() public void AcceptAllButLastDoubleHypens_ReturnsTheOnlyDoubleHyphenToken()
{ {
// Arrange // Arrange
var sut = CreateTestParserForContent("-->"); var sut = CreateTestParserForContent("-->");
// Act // Act
var symbol = sut.AcceptAllButLastDoubleHyphens(); var token = sut.AcceptAllButLastDoubleHyphens();
// Assert // Assert
Assert.Equal(doubleHyphenSymbol, symbol); Assert.Equal(doubleHyphenToken, token);
Assert.True(sut.At(HtmlSymbolType.CloseAngle)); Assert.True(sut.At(HtmlTokenType.CloseAngle));
Assert.Equal(doubleHyphenSymbol, sut.PreviousSymbol); Assert.Equal(doubleHyphenToken, sut.PreviousToken);
} }
[Fact] [Fact]
public void AcceptAllButLastDoubleHypens_ReturnsTheDoubleHyphenSymbolAfterAcceptingTheDash() public void AcceptAllButLastDoubleHypens_ReturnsTheDoubleHyphenTokenAfterAcceptingTheDash()
{ {
// Arrange // Arrange
var sut = CreateTestParserForContent("--->"); var sut = CreateTestParserForContent("--->");
// Act // Act
var symbol = sut.AcceptAllButLastDoubleHyphens(); var token = sut.AcceptAllButLastDoubleHyphens();
// Assert // Assert
Assert.Equal(doubleHyphenSymbol, symbol); Assert.Equal(doubleHyphenToken, token);
Assert.True(sut.At(HtmlSymbolType.CloseAngle)); Assert.True(sut.At(HtmlTokenType.CloseAngle));
Assert.True(HtmlMarkupParser.IsHyphen(sut.PreviousSymbol)); Assert.True(HtmlMarkupParser.IsHyphen(sut.PreviousToken));
} }
[Fact] [Fact]
@ -117,7 +117,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsHtmlCommentAhead_ReturnsTrueForValidCommentTagWithExtraInfoAfter() public void IsHtmlCommentAhead_ReturnsTrueForValidCommentTagWithExtraInfoAfter()
{ {
// Arrange // Arrange
var sut = CreateTestParserForContent("-- comment --> the first part is a valid comment without the Open angle and bang symbols"); var sut = CreateTestParserForContent("-- comment --> the first part is a valid comment without the Open angle and bang tokens");
// Act & Assert // Act & Assert
Assert.True(sut.IsHtmlCommentAhead()); Assert.True(sut.IsHtmlCommentAhead());
@ -157,8 +157,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsFalseForAllowedContent() public void IsCommentContentEndingInvalid_ReturnsFalseForAllowedContent()
{ {
// Arrange // Arrange
var expectedSymbol1 = new HtmlSymbol("a", HtmlSymbolType.Text); var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text);
var sequence = Enumerable.Range((int)'a', 26).Select(item => new HtmlSymbol(((char)item).ToString(), HtmlSymbolType.Text)); var sequence = Enumerable.Range((int)'a', 26).Select(item => new HtmlToken(((char)item).ToString(), HtmlTokenType.Text));
// Act & Assert // Act & Assert
Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence)); Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -168,8 +168,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsTrueForDisallowedContent() public void IsCommentContentEndingInvalid_ReturnsTrueForDisallowedContent()
{ {
// Arrange // Arrange
var expectedSymbol1 = new HtmlSymbol("a", HtmlSymbolType.Text); var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text);
var sequence = new[] { new HtmlSymbol("<", HtmlSymbolType.OpenAngle), new HtmlSymbol("!", HtmlSymbolType.Bang), new HtmlSymbol("-", HtmlSymbolType.Text) }; var sequence = new[] { new HtmlToken("<", HtmlTokenType.OpenAngle), new HtmlToken("!", HtmlTokenType.Bang), new HtmlToken("-", HtmlTokenType.Text) };
// Act & Assert // Act & Assert
Assert.True(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence)); Assert.True(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -179,8 +179,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
public void IsCommentContentEndingInvalid_ReturnsFalseForEmptyContent() public void IsCommentContentEndingInvalid_ReturnsFalseForEmptyContent()
{ {
// Arrange // Arrange
var expectedSymbol1 = new HtmlSymbol("a", HtmlSymbolType.Text); var expectedToken1 = new HtmlToken("a", HtmlTokenType.Text);
var sequence = Array.Empty<HtmlSymbol>(); var sequence = Array.Empty<HtmlToken>();
// Act & Assert // Act & Assert
Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence)); Assert.False(HtmlMarkupParser.IsCommentContentEndingInvalid(sequence));
@ -188,9 +188,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
private class TestHtmlMarkupParser : HtmlMarkupParser private class TestHtmlMarkupParser : HtmlMarkupParser
{ {
public new HtmlSymbol PreviousSymbol public new HtmlToken PreviousToken
{ {
get => base.PreviousSymbol; get => base.PreviousToken;
} }
public new bool IsHtmlCommentAhead() public new bool IsHtmlCommentAhead()
@ -203,7 +203,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
this.EnsureCurrent(); this.EnsureCurrent();
} }
public new HtmlSymbol AcceptAllButLastDoubleHyphens() public new HtmlToken AcceptAllButLastDoubleHyphens()
{ {
return base.AcceptAllButLastDoubleHyphens(); return base.AcceptAllButLastDoubleHyphens();
} }

66
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlTokenizerTest.cs
View File

@ -17,113 +17,113 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void Text_Is_Recognized() public void Text_Is_Recognized()
{ {
TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384", TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384",
new HtmlSymbol("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text)); new HtmlToken("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlTokenType.Text));
} }
[Fact] [Fact]
public void Whitespace_Is_Recognized() public void Whitespace_Is_Recognized()
{ {
TestTokenizer(" \t\f ", TestTokenizer(" \t\f ",
new HtmlSymbol(" \t\f ", HtmlSymbolType.WhiteSpace)); new HtmlToken(" \t\f ", HtmlTokenType.WhiteSpace));
} }
[Fact] [Fact]
public void Newline_Is_Recognized() public void Newline_Is_Recognized()
{ {
TestTokenizer("\n\r\r\n", TestTokenizer("\n\r\r\n",
new HtmlSymbol("\n", HtmlSymbolType.NewLine), new HtmlToken("\n", HtmlTokenType.NewLine),
new HtmlSymbol("\r", HtmlSymbolType.NewLine), new HtmlToken("\r", HtmlTokenType.NewLine),
new HtmlSymbol("\r\n", HtmlSymbolType.NewLine)); new HtmlToken("\r\n", HtmlTokenType.NewLine));
} }
[Fact] [Fact]
public void Transition_Is_Not_Recognized_Mid_Text_If_Surrounded_By_Alphanumeric_Characters() public void Transition_Is_Not_Recognized_Mid_Text_If_Surrounded_By_Alphanumeric_Characters()
{ {
TestSingleToken("foo@bar", HtmlSymbolType.Text); TestSingleToken("foo@bar", HtmlTokenType.Text);
} }
[Fact] [Fact]
public void OpenAngle_Is_Recognized() public void OpenAngle_Is_Recognized()
{ {
TestSingleToken("<", HtmlSymbolType.OpenAngle); TestSingleToken("<", HtmlTokenType.OpenAngle);
} }
[Fact] [Fact]
public void Bang_Is_Recognized() public void Bang_Is_Recognized()
{ {
TestSingleToken("!", HtmlSymbolType.Bang); TestSingleToken("!", HtmlTokenType.Bang);
} }
[Fact] [Fact]
public void Solidus_Is_Recognized() public void Solidus_Is_Recognized()
{ {
TestSingleToken("/", HtmlSymbolType.ForwardSlash); TestSingleToken("/", HtmlTokenType.ForwardSlash);
} }
[Fact] [Fact]
public void QuestionMark_Is_Recognized() public void QuestionMark_Is_Recognized()
{ {
TestSingleToken("?", HtmlSymbolType.QuestionMark); TestSingleToken("?", HtmlTokenType.QuestionMark);
} }
[Fact] [Fact]
public void LeftBracket_Is_Recognized() public void LeftBracket_Is_Recognized()
{ {
TestSingleToken("[", HtmlSymbolType.LeftBracket); TestSingleToken("[", HtmlTokenType.LeftBracket);
} }
[Fact] [Fact]
public void CloseAngle_Is_Recognized() public void CloseAngle_Is_Recognized()
{ {
TestSingleToken(">", HtmlSymbolType.CloseAngle); TestSingleToken(">", HtmlTokenType.CloseAngle);
} }
[Fact] [Fact]
public void RightBracket_Is_Recognized() public void RightBracket_Is_Recognized()
{ {
TestSingleToken("]", HtmlSymbolType.RightBracket); TestSingleToken("]", HtmlTokenType.RightBracket);
} }
[Fact] [Fact]
public void Equals_Is_Recognized() public void Equals_Is_Recognized()
{ {
TestSingleToken("=", HtmlSymbolType.Equals); TestSingleToken("=", HtmlTokenType.Equals);
} }
[Fact] [Fact]
public void DoubleQuote_Is_Recognized() public void DoubleQuote_Is_Recognized()
{ {
TestSingleToken("\"", HtmlSymbolType.DoubleQuote); TestSingleToken("\"", HtmlTokenType.DoubleQuote);
} }
[Fact] [Fact]
public void SingleQuote_Is_Recognized() public void SingleQuote_Is_Recognized()
{ {
TestSingleToken("'", HtmlSymbolType.SingleQuote); TestSingleToken("'", HtmlTokenType.SingleQuote);
} }
[Fact] [Fact]
public void Transition_Is_Recognized() public void Transition_Is_Recognized()
{ {
TestSingleToken("@", HtmlSymbolType.Transition); TestSingleToken("@", HtmlTokenType.Transition);
} }
[Fact] [Fact]
public void DoubleHyphen_Is_Recognized() public void DoubleHyphen_Is_Recognized()
{ {
TestSingleToken("--", HtmlSymbolType.DoubleHyphen); TestSingleToken("--", HtmlTokenType.DoubleHyphen);
} }
[Fact] [Fact]
public void SingleHyphen_Is_Not_Recognized() public void SingleHyphen_Is_Not_Recognized()
{ {
TestSingleToken("-", HtmlSymbolType.Text); TestSingleToken("-", HtmlTokenType.Text);
} }
[Fact] [Fact]
public void SingleHyphen_Mid_Text_Is_Not_Recognized_As_Separate_Token() public void SingleHyphen_Mid_Text_Is_Not_Recognized_As_Separate_Token()
{ {
TestSingleToken("foo-bar", HtmlSymbolType.Text); TestSingleToken("foo-bar", HtmlTokenType.Text);
} }
[Fact] [Fact]
@ -131,9 +131,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *", "@* Foo * Bar * Baz *",
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol(" Foo * Bar * Baz *", HtmlSymbolType.RazorComment)); new HtmlToken(" Foo * Bar * Baz *", HtmlTokenType.RazorComment));
} }
[Fact] [Fact]
@ -141,11 +141,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo * Bar * Baz *@", "@* Foo * Bar * Baz *@",
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol(" Foo * Bar * Baz ", HtmlSymbolType.RazorComment), new HtmlToken(" Foo * Bar * Baz ", HtmlTokenType.RazorComment),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition)); new HtmlToken("@", HtmlTokenType.RazorCommentTransition));
} }
[Fact] [Fact]
@ -153,11 +153,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
TestTokenizer( TestTokenizer(
"@* Foo Bar Baz *@", "@* Foo Bar Baz *@",
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol(" Foo Bar Baz ", HtmlSymbolType.RazorComment), new HtmlToken(" Foo Bar Baz ", HtmlTokenType.RazorComment),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), new HtmlToken("*", HtmlTokenType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition)); new HtmlToken("@", HtmlTokenType.RazorCommentTransition));
} }
} }
} }

10
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlTokenizerTestBase.cs
View File

@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
public abstract class HtmlTokenizerTestBase : TokenizerTestBase public abstract class HtmlTokenizerTestBase : TokenizerTestBase
{ {
private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown); private static HtmlToken _ignoreRemaining = new HtmlToken(string.Empty, HtmlTokenType.Unknown);
internal override object IgnoreRemaining internal override object IgnoreRemaining
{ {
@ -17,14 +17,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source); return new HtmlTokenizer(source);
} }
internal void TestSingleToken(string text, HtmlSymbolType expectedSymbolType) internal void TestSingleToken(string text, HtmlTokenType expectedTokenType)
{ {
TestTokenizer(text, new HtmlSymbol(text, expectedSymbolType)); TestTokenizer(text, new HtmlToken(text, expectedTokenType));
} }
internal void TestTokenizer(string input, params HtmlSymbol[] expectedSymbols) internal void TestTokenizer(string input, params HtmlToken[] expectedTokens)
{ {
base.TestTokenizer<HtmlSymbol, HtmlSymbolType>(input, expectedSymbols); base.TestTokenizer<HtmlToken, HtmlTokenType>(input, expectedTokens);
} }
} }
} }

22
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/ImplicitExpressionEditHandlerTest.cs
View File

@ -159,7 +159,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_SingleLeftParenthesis_CountsCorrectly() public void TryUpdateBalanceCount_SingleLeftParenthesis_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("(", CSharpSymbolType.LeftParenthesis); var token = new CSharpToken("(", CSharpTokenType.LeftParenthesis);
var count = 0; var count = 0;
// Act // Act
@ -174,7 +174,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_SingleRightParenthesis_CountsCorrectly() public void TryUpdateBalanceCount_SingleRightParenthesis_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpSymbol(")", CSharpSymbolType.RightParenthesis); var token = new CSharpToken(")", CSharpTokenType.RightParenthesis);
var count = 2; var count = 2;
// Act // Act
@ -189,7 +189,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_IncompleteStringLiteral_CountsCorrectly() public void TryUpdateBalanceCount_IncompleteStringLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("\"((", CSharpSymbolType.StringLiteral); var token = new CSharpToken("\"((", CSharpTokenType.StringLiteral);
var count = 2; var count = 2;
// Act // Act
@ -204,7 +204,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_IncompleteCharacterLiteral_CountsCorrectly() public void TryUpdateBalanceCount_IncompleteCharacterLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("'((", CSharpSymbolType.CharacterLiteral); var token = new CSharpToken("'((", CSharpTokenType.CharacterLiteral);
var count = 2; var count = 2;
// Act // Act
@ -219,7 +219,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_CompleteStringLiteral_CountsCorrectly() public void TryUpdateBalanceCount_CompleteStringLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("\"((\"", CSharpSymbolType.StringLiteral); var token = new CSharpToken("\"((\"", CSharpTokenType.StringLiteral);
var count = 2; var count = 2;
// Act // Act
@ -234,7 +234,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_CompleteCharacterLiteral_CountsCorrectly() public void TryUpdateBalanceCount_CompleteCharacterLiteral_CountsCorrectly()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("'('", CSharpSymbolType.CharacterLiteral); var token = new CSharpToken("'('", CSharpTokenType.CharacterLiteral);
var count = 2; var count = 2;
// Act // Act
@ -249,7 +249,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesis_ReturnsFalse() public void TryUpdateBalanceCount_InvalidParenthesis_ReturnsFalse()
{ {
// Arrange // Arrange
var token = new CSharpSymbol(")", CSharpSymbolType.RightParenthesis); var token = new CSharpToken(")", CSharpTokenType.RightParenthesis);
var count = 0; var count = 0;
// Act // Act
@ -264,7 +264,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesisStringLiteral_ReturnsFalse() public void TryUpdateBalanceCount_InvalidParenthesisStringLiteral_ReturnsFalse()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("\")", CSharpSymbolType.StringLiteral); var token = new CSharpToken("\")", CSharpTokenType.StringLiteral);
var count = 0; var count = 0;
// Act // Act
@ -279,7 +279,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void TryUpdateBalanceCount_InvalidParenthesisCharacterLiteral_ReturnsFalse() public void TryUpdateBalanceCount_InvalidParenthesisCharacterLiteral_ReturnsFalse()
{ {
// Arrange // Arrange
var token = new CSharpSymbol("')", CSharpSymbolType.CharacterLiteral); var token = new CSharpToken("')", CSharpTokenType.CharacterLiteral);
var count = 0; var count = 0;
// Act // Act
@ -453,10 +453,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return span; return span;
} }
private static IReadOnlyList<CSharpSymbol> GetTokens(SourceLocation start, string content) private static IReadOnlyList<CSharpToken> GetTokens(SourceLocation start, string content)
{ {
var parent = GetSpan(start, content); var parent = GetSpan(start, content);
var tokens = parent.Symbols.Cast<CSharpSymbol>().ToArray(); var tokens = parent.Tokens.Cast<CSharpToken>().ToArray();
return tokens; return tokens;
} }
} }

10
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/SpanTest.cs
View File

@ -12,10 +12,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Arrange // Arrange
var builder = new SpanBuilder(SourceLocation.Zero); var builder = new SpanBuilder(SourceLocation.Zero);
builder.Accept(new HtmlSymbol("hello", HtmlSymbolType.Text)); builder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
var span = builder.Build(); var span = builder.Build();
var newBuilder = new SpanBuilder(SourceLocation.Zero); var newBuilder = new SpanBuilder(SourceLocation.Zero);
newBuilder.Accept(new HtmlSymbol("hi", HtmlSymbolType.Text)); newBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
var originalLength = span.Length; var originalLength = span.Length;
// Act // Act
@ -33,7 +33,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{ {
// Arrange // Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero); var spanBuilder = new SpanBuilder(SourceLocation.Zero);
spanBuilder.Accept(new HtmlSymbol("hello", HtmlSymbolType.Text)); spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
var span = spanBuilder.Build(); var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder() var blockBuilder = new BlockBuilder()
{ {
@ -44,7 +44,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.Parent = block; span.Parent = block;
var originalBlockLength = block.Length; var originalBlockLength = block.Length;
var newSpanBuilder = new SpanBuilder(SourceLocation.Zero); var newSpanBuilder = new SpanBuilder(SourceLocation.Zero);
newSpanBuilder.Accept(new HtmlSymbol("hi", HtmlSymbolType.Text)); newSpanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
// Act // Act
span.ReplaceWith(newSpanBuilder); span.ReplaceWith(newSpanBuilder);
@ -64,7 +64,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind = SpanKindInternal.Transition, Kind = SpanKindInternal.Transition,
ChunkGenerator = new ExpressionChunkGenerator(), ChunkGenerator = new ExpressionChunkGenerator(),
}; };
spanBuilder.Accept(new CSharpSymbol("@", CSharpSymbolType.Transition)); spanBuilder.Accept(new CSharpToken("@", CSharpTokenType.Transition));
var span = spanBuilder.Build(); var span = spanBuilder.Build();
// Act // Act

72
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/TokenizerLookaheadTest.cs
View File

@ -57,51 +57,51 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
[Fact] [Fact]
public void LookaheadUntil_PassesThePreviousSymbolsInTheSameOrder() public void LookaheadUntil_PassesThePreviousTokensInTheSameOrder()
{ {
// Arrange // Arrange
var tokenizer = CreateContentTokenizer("asdf--fvd--<"); var tokenizer = CreateContentTokenizer("asdf--fvd--<");
// Act // Act
var i = 3; var i = 3;
IEnumerable<HtmlSymbol> previousSymbols = null; IEnumerable<HtmlToken> previousTokens = null;
var symbolFound = tokenizer.LookaheadUntil((s, p) => var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{ {
previousSymbols = p; previousTokens = p;
return --i == 0; return --i == 0;
}); });
// Assert // Assert
Assert.Equal(4, previousSymbols.Count()); Assert.Equal(4, previousTokens.Count());
// For the very first element, there will be no previous items, so null is expected // For the very first element, there will be no previous items, so null is expected
var orderIndex = 0; var orderIndex = 0;
Assert.Null(previousSymbols.ElementAt(orderIndex++)); Assert.Null(previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlSymbol("asdf", HtmlSymbolType.Text), previousSymbols.ElementAt(orderIndex++)); Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlSymbol("--", HtmlSymbolType.DoubleHyphen), previousSymbols.ElementAt(orderIndex++)); Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), previousTokens.ElementAt(orderIndex++));
Assert.Equal(new HtmlSymbol("fvd", HtmlSymbolType.Text), previousSymbols.ElementAt(orderIndex++)); Assert.Equal(new HtmlToken("fvd", HtmlTokenType.Text), previousTokens.ElementAt(orderIndex++));
} }
[Fact] [Fact]
public void LookaheadUntil_ReturnsFalseAfterIteratingOverAllSymbolsIfConditionIsNotMet() public void LookaheadUntil_ReturnsFalseAfterIteratingOverAllTokensIfConditionIsNotMet()
{ {
// Arrange // Arrange
var tokenizer = CreateContentTokenizer("asdf--fvd"); var tokenizer = CreateContentTokenizer("asdf--fvd");
// Act // Act
var symbols = new Stack<HtmlSymbol>(); var tokens = new Stack<HtmlToken>();
var symbolFound = tokenizer.LookaheadUntil((s, p) => var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{ {
symbols.Push(s); tokens.Push(s);
return false; return false;
}); });
// Assert // Assert
Assert.False(symbolFound); Assert.False(tokenFound);
Assert.Equal(3, symbols.Count); Assert.Equal(3, tokens.Count);
Assert.Equal(new HtmlSymbol("fvd", HtmlSymbolType.Text), symbols.Pop()); Assert.Equal(new HtmlToken("fvd", HtmlTokenType.Text), tokens.Pop());
Assert.Equal(new HtmlSymbol("--", HtmlSymbolType.DoubleHyphen), symbols.Pop()); Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), tokens.Pop());
Assert.Equal(new HtmlSymbol("asdf", HtmlSymbolType.Text), symbols.Pop()); Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), tokens.Pop());
} }
[Fact] [Fact]
@ -111,18 +111,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokenizer = CreateContentTokenizer("asdf--fvd"); var tokenizer = CreateContentTokenizer("asdf--fvd");
// Act // Act
var symbols = new Stack<HtmlSymbol>(); var tokens = new Stack<HtmlToken>();
var symbolFound = tokenizer.LookaheadUntil((s, p) => var tokenFound = tokenizer.LookaheadUntil((s, p) =>
{ {
symbols.Push(s); tokens.Push(s);
return s.Type == HtmlSymbolType.DoubleHyphen; return s.Type == HtmlTokenType.DoubleHyphen;
}); });
// Assert // Assert
Assert.True(symbolFound); Assert.True(tokenFound);
Assert.Equal(2, symbols.Count); Assert.Equal(2, tokens.Count);
Assert.Equal(new HtmlSymbol("--", HtmlSymbolType.DoubleHyphen), symbols.Pop()); Assert.Equal(new HtmlToken("--", HtmlTokenType.DoubleHyphen), tokens.Pop());
Assert.Equal(new HtmlSymbol("asdf", HtmlSymbolType.Text), symbols.Pop()); Assert.Equal(new HtmlToken("asdf", HtmlTokenType.Text), tokens.Pop());
} }
private static TestTokenizerBackedParser CreateContentTokenizer(string content) private static TestTokenizerBackedParser CreateContentTokenizer(string content)
@ -135,7 +135,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return tokenizer; return tokenizer;
} }
private class ExposedTokenizer : Tokenizer<CSharpSymbol, CSharpSymbolType> private class ExposedTokenizer : Tokenizer<CSharpToken, CSharpTokenType>
{ {
public ExposedTokenizer(string input) public ExposedTokenizer(string input)
: base(new SeekableTextReader(input, filePath: null)) : base(new SeekableTextReader(input, filePath: null))
@ -150,7 +150,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public override CSharpSymbolType RazorCommentStarType public override CSharpTokenType RazorCommentStarType
{ {
get get
{ {
@ -158,7 +158,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public override CSharpSymbolType RazorCommentTransitionType public override CSharpTokenType RazorCommentTransitionType
{ {
get get
{ {
@ -166,7 +166,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
public override CSharpSymbolType RazorCommentType public override CSharpTokenType RazorCommentType
{ {
get get
{ {
@ -182,9 +182,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
protected override CSharpSymbol CreateSymbol( protected override CSharpToken CreateToken(
string content, string content,
CSharpSymbolType type, CSharpTokenType type,
IReadOnlyList<RazorDiagnostic> errors) IReadOnlyList<RazorDiagnostic> errors)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
@ -196,9 +196,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
} }
} }
private class TestTokenizerBackedParser : TokenizerBackedParser<HtmlTokenizer, HtmlSymbol, HtmlSymbolType> private class TestTokenizerBackedParser : TokenizerBackedParser<HtmlTokenizer, HtmlToken, HtmlTokenType>
{ {
internal TestTokenizerBackedParser(LanguageCharacteristics<HtmlTokenizer, HtmlSymbol, HtmlSymbolType> language, ParserContext context) : base(language, context) internal TestTokenizerBackedParser(LanguageCharacteristics<HtmlTokenizer, HtmlToken, HtmlTokenType> language, ParserContext context) : base(language, context)
{ {
} }
@ -207,12 +207,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
throw new NotImplementedException(); throw new NotImplementedException();
} }
protected override bool SymbolTypeEquals(HtmlSymbolType x, HtmlSymbolType y) protected override bool TokenTypeEquals(HtmlTokenType x, HtmlTokenType y)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
internal new bool LookaheadUntil(Func<HtmlSymbol, IEnumerable<HtmlSymbol>, bool> condition) internal new bool LookaheadUntil(Func<HtmlToken, IEnumerable<HtmlToken>, bool> condition)
{ {
return base.LookaheadUntil(condition); return base.LookaheadUntil(condition);
} }

4
test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/TokenizerTestBase.cs
View File

@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal void TestTokenizer<TSymbol, TSymbolType>(string input, params TSymbol[] expectedSymbols) internal void TestTokenizer<TSymbol, TSymbolType>(string input, params TSymbol[] expectedSymbols)
where TSymbolType : struct where TSymbolType : struct
where TSymbol : SymbolBase<TSymbolType> where TSymbol : TokenBase<TSymbolType>
{ {
// Arrange // Arrange
var success = true; var success = true;
@ -25,7 +25,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokenizer = (Tokenizer<TSymbol, TSymbolType>)CreateTokenizer(source); var tokenizer = (Tokenizer<TSymbol, TSymbolType>)CreateTokenizer(source);
var counter = 0; var counter = 0;
TSymbol current = null; TSymbol current = null;
while ((current = tokenizer.NextSymbol()) != null) while ((current = tokenizer.NextToken()) != null)
{ {
if (counter >= expectedSymbols.Length) if (counter >= expectedSymbols.Length)
{ {

20
test/Microsoft.AspNetCore.Razor.Language.Test/SourceChangeTest.cs
View File

@ -104,8 +104,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(0, 0, 0)); var builder = new SpanBuilder(new SourceLocation(0, 0, 0));
builder.Accept(new RawTextSymbol(new SourceLocation(0, 0, 0), "Hello, ")); builder.Accept(new RawTextToken(new SourceLocation(0, 0, 0), "Hello, "));
builder.Accept(new RawTextSymbol(new SourceLocation(7, 0, 7), "World")); builder.Accept(new RawTextToken(new SourceLocation(7, 0, 7), "World"));
var span = new Span(builder); var span = new Span(builder);
@ -123,8 +123,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextSymbol(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextSymbol(new SourceLocation(20, 0, 20), "World")); builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
var span = new Span(builder); var span = new Span(builder);
@ -142,8 +142,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextSymbol(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextSymbol(new SourceLocation(20, 0, 20), "World")); builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
var span = new Span(builder); var span = new Span(builder);
@ -161,8 +161,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextSymbol(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextSymbol(new SourceLocation(20, 0, 20), "World")); builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
var span = new Span(builder); var span = new Span(builder);
@ -180,8 +180,8 @@ namespace Microsoft.AspNetCore.Razor.Language
{ {
// Arrange // Arrange
var builder = new SpanBuilder(new SourceLocation(13, 0, 0)); var builder = new SpanBuilder(new SourceLocation(13, 0, 0));
builder.Accept(new RawTextSymbol(new SourceLocation(13, 0, 13), "Hello, ")); builder.Accept(new RawTextToken(new SourceLocation(13, 0, 13), "Hello, "));
builder.Accept(new RawTextSymbol(new SourceLocation(20, 0, 20), "World")); builder.Accept(new RawTextToken(new SourceLocation(20, 0, 20), "World"));
var span = new Span(builder); var span = new Span(builder);

16
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/FunctionsDirectiveAutoCompleteAtEOF.syntaxtree.txt
View File

@ -1,9 +1,9 @@
Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 11 - (0:0,0) Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 11 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Symbols:1 MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpSymbolType.Identifier;[functions]; CSharpTokenType.Identifier;[functions];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Symbols:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
Code span - Gen<Stmt> - [] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Symbols:1 Code span - Gen<Stmt> - [] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:1
CSharpSymbolType.Unknown;[]; CSharpTokenType.Unknown;[];

18
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/FunctionsDirectiveAutoCompleteAtStartOfFile.syntaxtree.txt
View File

@ -1,10 +1,10 @@
Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 16 - (0:0,0) Directive block - Gen<Directive:{functions;CodeBlock;Unrestricted} [RZ1006(10:0,10 [1] )]> - 16 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Symbols:1 MetaCode span - Gen<None> - [functions] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpSymbolType.Identifier;[functions]; CSharpTokenType.Identifier;[functions];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Symbols:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (10:0,10) - Tokens:1
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
Code span - Gen<Stmt> - [LFfoo] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Symbols:2 Code span - Gen<Stmt> - [LFfoo] - CodeBlockEditHandler;Accepts:Any;CodeBlock - (11:0,11) - Tokens:2
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];

28
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/SectionDirectiveAutoCompleteAtEOF.syntaxtree.txt
View File

@ -1,16 +1,16 @@
Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 17 - (0:0,0) Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 17 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Symbols:1 MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpSymbolType.Identifier;[section]; CSharpTokenType.Identifier;[section];
Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Symbols:1 Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Symbols:1 Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1
CSharpSymbolType.Identifier;[Header]; CSharpTokenType.Identifier;[Header];
Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Symbols:1 Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Symbols:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
Markup block - Gen<None> - 0 - (17:0,17) Markup block - Gen<None> - 0 - (17:0,17)
Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (17:0,17) - Symbols:1 Markup span - Gen<Markup> - [] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
HtmlSymbolType.Unknown;[]; HtmlTokenType.Unknown;[];

50
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/SectionDirectiveAutoCompleteAtStartOfFile.syntaxtree.txt
View File

@ -1,29 +1,29 @@
Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 29 - (0:0,0) Directive block - Gen<Directive:{section;RazorBlock;Unrestricted} [RZ1006(16:0,16 [1] )]> - 29 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Symbols:1 MetaCode span - Gen<None> - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpSymbolType.Identifier;[section]; CSharpTokenType.Identifier;[section];
Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Symbols:1 Code span - Gen<None> - [ ] - SpanEditHandler;Accepts:WhiteSpace - (8:0,8) - Tokens:1
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Symbols:1 Code span - Gen<DirectiveToken {SectionName;Member;Opt:False}> - [Header] - DirectiveTokenEditHandler;Accepts:NonWhiteSpace - (9:0,9) - Tokens:1
CSharpSymbolType.Identifier;[Header]; CSharpTokenType.Identifier;[Header];
Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Symbols:1 Markup span - Gen<None> - [ ] - SpanEditHandler;Accepts:AllWhiteSpace - (15:0,15) - Tokens:1
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Symbols:1 MetaCode span - Gen<None> - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[}];AtEnd - (16:0,16) - Tokens:1
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
Markup block - Gen<None> - 12 - (17:0,17) Markup block - Gen<None> - 12 - (17:0,17)
Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (17:0,17) - Symbols:1 Markup span - Gen<Markup> - [LF] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
HtmlSymbolType.NewLine;[LF]; HtmlTokenType.NewLine;[LF];
Tag block - Gen<None> - 3 - (19:1,0) Tag block - Gen<None> - 3 - (19:1,0)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:Any - (19:1,0) - Symbols:3 Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:Any - (19:1,0) - Tokens:3
HtmlSymbolType.OpenAngle;[<]; HtmlTokenType.OpenAngle;[<];
HtmlSymbolType.Text;[p]; HtmlTokenType.Text;[p];
HtmlSymbolType.CloseAngle;[>]; HtmlTokenType.CloseAngle;[>];
Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (22:1,3) - Symbols:1 Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (22:1,3) - Tokens:1
HtmlSymbolType.Text;[Foo]; HtmlTokenType.Text;[Foo];
Tag block - Gen<None> - 4 - (25:1,6) Tag block - Gen<None> - 4 - (25:1,6)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:Any - (25:1,6) - Symbols:4 Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:Any - (25:1,6) - Tokens:4
HtmlSymbolType.OpenAngle;[<]; HtmlTokenType.OpenAngle;[<];
HtmlSymbolType.ForwardSlash;[/]; HtmlTokenType.ForwardSlash;[/];
HtmlSymbolType.Text;[p]; HtmlTokenType.Text;[p];
HtmlSymbolType.CloseAngle;[>]; HtmlTokenType.CloseAngle;[>];

12
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/VerbatimBlockAutoCompleteAtEOF.syntaxtree.txt
View File

@ -1,7 +1,7 @@
Statement block - Gen<None> - 2 - (0:0,0) Statement block - Gen<None> - 2 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Symbols:1 MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
Code span - Gen<Stmt> - [] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Symbols:1 Code span - Gen<Stmt> - [] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1
CSharpSymbolType.Unknown;[]; CSharpTokenType.Unknown;[];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpAutoCompleteTest/VerbatimBlockAutoCompleteAtStartOfFile.syntaxtree.txt
View File

@ -1,21 +1,21 @@
Statement block - Gen<None> - 11 - (0:0,0) Statement block - Gen<None> - 11 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Symbols:1 MetaCode span - Gen<None> - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
Code span - Gen<Stmt> - [LF] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Symbols:1 Code span - Gen<Stmt> - [LF] - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[}];AtEOL - (2:0,2) - Tokens:1
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
Markup block - Gen<None> - 7 - (4:1,0) Markup block - Gen<None> - 7 - (4:1,0)
Tag block - Gen<None> - 3 - (4:1,0) Tag block - Gen<None> - 3 - (4:1,0)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (4:1,0) - Symbols:3 Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (4:1,0) - Tokens:3
HtmlSymbolType.OpenAngle;[<]; HtmlTokenType.OpenAngle;[<];
HtmlSymbolType.Text;[p]; HtmlTokenType.Text;[p];
HtmlSymbolType.CloseAngle;[>]; HtmlTokenType.CloseAngle;[>];
Tag block - Gen<None> - 4 - (7:1,3) Tag block - Gen<None> - 4 - (7:1,3)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (7:1,3) - Symbols:4 Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (7:1,3) - Tokens:4
HtmlSymbolType.OpenAngle;[<]; HtmlTokenType.OpenAngle;[<];
HtmlSymbolType.ForwardSlash;[/]; HtmlTokenType.ForwardSlash;[/];
HtmlSymbolType.Text;[p]; HtmlTokenType.Text;[p];
HtmlSymbolType.CloseAngle;[>]; HtmlTokenType.CloseAngle;[>];
Code span - Gen<Stmt> - [] - SpanEditHandler;Accepts:Any - (11:1,7) - Symbols:1 Code span - Gen<Stmt> - [] - SpanEditHandler;Accepts:Any - (11:1,7) - Tokens:1
CSharpSymbolType.Unknown;[]; CSharpTokenType.Unknown;[];

40
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/BalancingBracketsIgnoresStringLiteralCharactersAndBracketsInsideBlockComments.syntaxtree.txt
View File

@ -1,21 +1,21 @@
Statement block - Gen<None> - 54 - (0:0,0) Statement block - Gen<None> - 54 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) {LF /* bar } " */ ' baz } 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:19 Code span - Gen<Stmt> - [if(foo) {LF /* bar } " */ ' baz } 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:19
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Comment;[/* bar } " */]; CSharpTokenType.Comment;[/* bar } " */];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.CharacterLiteral;[' baz } ']; CSharpTokenType.CharacterLiteral;[' baz } '];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[zoop]; CSharpTokenType.Identifier;[zoop];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

36
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/BalancingBracketsIgnoresStringLiteralCharactersAndBracketsInsideSingleLineComments.syntaxtree.txt
View File

@ -1,19 +1,19 @@
Statement block - Gen<None> - 47 - (0:0,0) Statement block - Gen<None> - 47 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) {LF // bar } " baz 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:17 Code span - Gen<Stmt> - [if(foo) {LF // bar } " baz 'LF zoop();LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:17
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Comment;[// bar } " baz ']; CSharpTokenType.Comment;[// bar } " baz '];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[zoop]; CSharpTokenType.Identifier;[zoop];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

46
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/NestedCodeBlockWithAtDoesntCauseError.syntaxtree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 28 - (0:0,0) Statement block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:8 Code span - Gen<Stmt> - [if (true) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[true]; CSharpTokenType.Keyword;[true];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
Statement block - Gen<None> - 14 - (12:0,12) Statement block - Gen<None> - 14 - (12:0,12)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (13:0,13) - Symbols:8 Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:8
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[false]; CSharpTokenType.Keyword;[false];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (26:0,26) - Symbols:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (26:0,26) - Tokens:2
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

118
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockAcceptsElseIfWithNoCondition.syntaxtree.txt
View File

@ -1,60 +1,60 @@
Statement block - Gen<None> - 106 - (0:0,0) Statement block - Gen<None> - 106 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if { foo(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:58 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if { foo(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:58
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

32
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockAcceptsTrailingDotIntoImplicitExpressionWhenEmbeddedInCode.syntaxtree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 17 - (0:0,0) Statement block - Gen<None> - 17 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:7 Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
Expression block - Gen<Expr> - 5 - (10:0,10) Expression block - Gen<Expr> - 5 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
Code span - Gen<Expr> - [foo.] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Symbols:2 Code span - Gen<Expr> - [foo.] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:2
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (15:0,15) - Symbols:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (15:0,15) - Tokens:2
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

18
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockAllowsEmptyBlockStatement.syntaxtree.txt
View File

@ -1,10 +1,10 @@
Statement block - Gen<None> - 13 - (0:0,0) Statement block - Gen<None> - 13 - (0:0,0)
Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:8 Code span - Gen<Stmt> - [if(false) { }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:8
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[false]; CSharpTokenType.Keyword;[false];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

10
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCapturesNewlineAfterUsing.syntaxtree.txt
View File

@ -1,6 +1,6 @@
Directive block - Gen<None> - 11 - (0:0,0) Directive block - Gen<None> - 11 - (0:0,0)
Code span - Gen<Import: Foo;> - [using FooLF] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Symbols:4 Code span - Gen<Import: Foo;> - [using FooLF] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:4
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];

50
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCorrectlyParsesDoWhileBlock.syntaxtree.txt
View File

@ -1,26 +1,26 @@
Statement block - Gen<None> - 40 - (0:0,0) Statement block - Gen<None> - 40 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar);] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:24 Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar);] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:24
CSharpSymbolType.Keyword;[do]; CSharpTokenType.Keyword;[do];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[while]; CSharpTokenType.Keyword;[while];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.NotEqual;[!=]; CSharpTokenType.NotEqual;[!=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];

48
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCorrectlyParsesDoWhileBlockMissingSemicolon.syntaxtree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 39 - (0:0,0) Statement block - Gen<None> - 39 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar)] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:23 Code span - Gen<Stmt> - [do { var foo = bar; } while(foo != bar)] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:23
CSharpSymbolType.Keyword;[do]; CSharpTokenType.Keyword;[do];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[while]; CSharpTokenType.Keyword;[while];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.NotEqual;[!=]; CSharpTokenType.NotEqual;[!=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];

30
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCorrectlyParsesDoWhileBlockMissingWhileClauseEntirely.syntaxtree.txt
View File

@ -1,16 +1,16 @@
Statement block - Gen<None> - 21 - (0:0,0) Statement block - Gen<None> - 21 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; }] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:14 Code span - Gen<Stmt> - [do { var foo = bar; }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:14
CSharpSymbolType.Keyword;[do]; CSharpTokenType.Keyword;[do];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCorrectlyParsesDoWhileBlockMissingWhileCondition.syntaxtree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 27 - (0:0,0) Statement block - Gen<None> - 27 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:16 Code span - Gen<Stmt> - [do { var foo = bar; } while] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:16
CSharpSymbolType.Keyword;[do]; CSharpTokenType.Keyword;[do];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[while]; CSharpTokenType.Keyword;[while];

36
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCorrectlyParsesDoWhileBlockMissingWhileConditionWithSemicolon.syntaxtree.txt
View File

@ -1,19 +1,19 @@
Statement block - Gen<None> - 28 - (0:0,0) Statement block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Stmt> - [do { var foo = bar; } while;] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:17 Code span - Gen<Stmt> - [do { var foo = bar; } while;] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:17
CSharpSymbolType.Keyword;[do]; CSharpTokenType.Keyword;[do];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[while]; CSharpTokenType.Keyword;[while];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];

92
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockCorrectlyParsesMarkupInDoWhileBlock.syntaxtree.txt
View File

@ -1,50 +1,50 @@
Statement block - Gen<None> - 58 - (0:0,0) Statement block - Gen<None> - 58 - (0:0,0)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
Code span - Gen<Stmt> - [do { var foo = bar;] - SpanEditHandler;Accepts:Any - (1:0,1) - Symbols:12 Code span - Gen<Stmt> - [do { var foo = bar;] - SpanEditHandler;Accepts:Any - (1:0,1) - Tokens:12
CSharpSymbolType.Keyword;[do]; CSharpTokenType.Keyword;[do];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
Markup block - Gen<None> - 12 - (20:0,20) Markup block - Gen<None> - 12 - (20:0,20)
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (20:0,20) - Symbols:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:Any - (20:0,20) - Tokens:1
HtmlSymbolType.WhiteSpace;[ ]; HtmlTokenType.WhiteSpace;[ ];
Tag block - Gen<None> - 3 - (21:0,21) Tag block - Gen<None> - 3 - (21:0,21)
Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (21:0,21) - Symbols:3 Markup span - Gen<Markup> - [<p>] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:3
HtmlSymbolType.OpenAngle;[<]; HtmlTokenType.OpenAngle;[<];
HtmlSymbolType.Text;[p]; HtmlTokenType.Text;[p];
HtmlSymbolType.CloseAngle;[>]; HtmlTokenType.CloseAngle;[>];
Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (24:0,24) - Symbols:1 Markup span - Gen<Markup> - [Foo] - SpanEditHandler;Accepts:Any - (24:0,24) - Tokens:1
HtmlSymbolType.Text;[Foo]; HtmlTokenType.Text;[Foo];
Tag block - Gen<None> - 4 - (27:0,27) Tag block - Gen<None> - 4 - (27:0,27)
Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (27:0,27) - Symbols:4 Markup span - Gen<Markup> - [</p>] - SpanEditHandler;Accepts:None - (27:0,27) - Tokens:4
HtmlSymbolType.OpenAngle;[<]; HtmlTokenType.OpenAngle;[<];
HtmlSymbolType.ForwardSlash;[/]; HtmlTokenType.ForwardSlash;[/];
HtmlSymbolType.Text;[p]; HtmlTokenType.Text;[p];
HtmlSymbolType.CloseAngle;[>]; HtmlTokenType.CloseAngle;[>];
Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (31:0,31) - Symbols:1 Markup span - Gen<Markup> - [ ] - SpanEditHandler;Accepts:None - (31:0,31) - Tokens:1
HtmlSymbolType.WhiteSpace;[ ]; HtmlTokenType.WhiteSpace;[ ];
Code span - Gen<Stmt> - [foo++; } while (foo<bar>);] - SpanEditHandler;Accepts:None - (32:0,32) - Symbols:15 Code span - Gen<Stmt> - [foo++; } while (foo<bar>);] - SpanEditHandler;Accepts:None - (32:0,32) - Tokens:15
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.Increment;[++]; CSharpTokenType.Increment;[++];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[while]; CSharpTokenType.Keyword;[while];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.GreaterThan;[>]; CSharpTokenType.GreaterThan;[>];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];

72
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockDoesNotAllowMultipleFinallyBlocks.syntaxtree.txt
View File

@ -1,37 +1,37 @@
Statement block - Gen<None> - 55 - (0:0,0) Statement block - Gen<None> - 55 - (0:0,0)
Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:35 Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:35
CSharpSymbolType.Keyword;[try]; CSharpTokenType.Keyword;[try];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[finally]; CSharpTokenType.Keyword;[finally];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

32
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockDoesNotParseMarkupStatementOrExpressionOnSwitchCharacterNotFollowedByOpenAngleOrColon.syntaxtree.txt
View File

@ -1,17 +1,17 @@
Statement block - Gen<None> - 30 - (0:0,0) Statement block - Gen<None> - 30 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { @"Foo".ToString(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:15 Code span - Gen<Stmt> - [if(foo) { @"Foo".ToString(); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:15
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;[@"Foo"]; CSharpTokenType.StringLiteral;[@"Foo"];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[ToString]; CSharpTokenType.Identifier;[ToString];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

8
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockDoesntCaptureWhitespaceAfterUsing.syntaxtree.txt
View File

@ -1,5 +1,5 @@
Directive block - Gen<None> - 9 - (0:0,0) Directive block - Gen<None> - 9 - (0:0,0)
Code span - Gen<Import: Foo;> - [using Foo] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Symbols:3 Code span - Gen<Import: Foo;> - [using Foo] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:3
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];

24
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockHasErrorsIfNamespaceAliasMissingSemicolon.syntaxtree.txt
View File

@ -1,13 +1,13 @@
Directive block - Gen<None> - 29 - (0:0,0) Directive block - Gen<None> - 29 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz = FooBarBaz;> - [using Foo.Bar.Baz = FooBarBaz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Symbols:11 Code span - Gen<Import: Foo.Bar.Baz = FooBarBaz;> - [using Foo.Bar.Baz = FooBarBaz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:11
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[Baz]; CSharpTokenType.Identifier;[Baz];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[FooBarBaz]; CSharpTokenType.Identifier;[FooBarBaz];

16
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockHasErrorsIfNamespaceImportMissingSemicolon.syntaxtree.txt
View File

@ -1,9 +1,9 @@
Directive block - Gen<None> - 17 - (0:0,0) Directive block - Gen<None> - 17 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Symbols:7 Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:7
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[Baz]; CSharpTokenType.Identifier;[Baz];

188
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesElseIfBranchesOfIfStatement.syntaxtree.txt
View File

@ -1,95 +1,95 @@
Statement block - Gen<None> - 180 - (0:0,0) Statement block - Gen<None> - 180 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:93 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:93
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

44
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesExpressionOnSwitchCharacterFollowedByIdentifierStart.syntaxtree.txt
View File

@ -1,24 +1,24 @@
Statement block - Gen<None> - 25 - (0:0,0) Statement block - Gen<None> - 25 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:7 Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
Expression block - Gen<Expr> - 13 - (10:0,10) Expression block - Gen<Expr> - 13 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
Code span - Gen<Expr> - [foo[4].bar()] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Symbols:8 Code span - Gen<Expr> - [foo[4].bar()] - ImplicitExpressionEditHandler;Accepts:NonWhiteSpace;ImplicitExpression[ATD];K14 - (11:0,11) - Tokens:8
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.LeftBracket;[[]; CSharpTokenType.LeftBracket;[[];
CSharpSymbolType.IntegerLiteral;[4]; CSharpTokenType.IntegerLiteral;[4];
CSharpSymbolType.RightBracket;[]]; CSharpTokenType.RightBracket;[]];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (23:0,23) - Symbols:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (23:0,23) - Tokens:2
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

46
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesExpressionOnSwitchCharacterFollowedByOpenParen.syntaxtree.txt
View File

@ -1,25 +1,25 @@
Statement block - Gen<None> - 24 - (0:0,0) Statement block - Gen<None> - 24 - (0:0,0)
Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:7 Code span - Gen<Stmt> - [if(foo) { ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:7
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
Expression block - Gen<Expr> - 12 - (10:0,10) Expression block - Gen<Expr> - 12 - (10:0,10)
Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Symbols:1 Transition span - Gen<None> - [@] - SpanEditHandler;Accepts:None - (10:0,10) - Tokens:1
CSharpSymbolType.Transition;[@]; CSharpTokenType.Transition;[@];
MetaCode span - Gen<None> - [(] - SpanEditHandler;Accepts:None - (11:0,11) - Symbols:1 MetaCode span - Gen<None> - [(] - SpanEditHandler;Accepts:None - (11:0,11) - Tokens:1
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
Code span - Gen<Expr> - [foo + bar] - SpanEditHandler;Accepts:Any - (12:0,12) - Symbols:5 Code span - Gen<Expr> - [foo + bar] - SpanEditHandler;Accepts:Any - (12:0,12) - Tokens:5
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Plus;[+]; CSharpTokenType.Plus;[+];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[bar]; CSharpTokenType.Identifier;[bar];
MetaCode span - Gen<None> - [)] - SpanEditHandler;Accepts:None - (21:0,21) - Symbols:1 MetaCode span - Gen<None> - [)] - SpanEditHandler;Accepts:None - (21:0,21) - Tokens:1
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (22:0,22) - Symbols:2 Code span - Gen<Stmt> - [ }] - SpanEditHandler;Accepts:Any - (22:0,22) - Tokens:2
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

476
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesMultipleElseIfBranchesOfIfStatement.syntaxtree.txt
View File

@ -1,239 +1,239 @@
Statement block - Gen<None> - 459 - (0:0,0) Statement block - Gen<None> - 459 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:237 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:237
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

312
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesMultipleElseIfBranchesOfIfStatementFollowedByOneElseBranch.syntaxtree.txt
View File

@ -1,157 +1,157 @@
Statement block - Gen<None> - 313 - (0:0,0) Statement block - Gen<None> - 313 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else { Debug.WriteLine(@"bar } baz"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:155 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else { Debug.WriteLine(@"bar } baz"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:155
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

18
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesNamespaceAliasWithSemicolonForUsingKeywordIfIsInValidFormat.syntaxtree.txt
View File

@ -1,10 +1,10 @@
Directive block - Gen<None> - 28 - (0:0,0) Directive block - Gen<None> - 28 - (0:0,0)
Code span - Gen<Import: FooBarBaz = FooBarBaz;> - [using FooBarBaz = FooBarBaz;] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Symbols:8 Code span - Gen<Import: FooBarBaz = FooBarBaz;> - [using FooBarBaz = FooBarBaz;] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:8
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[FooBarBaz]; CSharpTokenType.Identifier;[FooBarBaz];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[FooBarBaz]; CSharpTokenType.Identifier;[FooBarBaz];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];

18
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockParsesNamespaceImportWithSemicolonForUsingKeywordIfIsInValidFormat.syntaxtree.txt
View File

@ -1,10 +1,10 @@
Directive block - Gen<None> - 18 - (0:0,0) Directive block - Gen<None> - 18 - (0:0,0)
Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz;] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Symbols:8 Code span - Gen<Import: Foo.Bar.Baz;> - [using Foo.Bar.Baz;] - SpanEditHandler;Accepts:AnyExceptNewline - (0:0,0) - Tokens:8
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[Baz]; CSharpTokenType.Identifier;[Baz];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];

90
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsForKeyword.syntaxtree.txt
View File

@ -1,46 +1,46 @@
Statement block - Gen<None> - 82 - (0:0,0) Statement block - Gen<None> - 82 - (0:0,0)
Code span - Gen<Stmt> - [for(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:44 Code span - Gen<Stmt> - [for(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:44
CSharpSymbolType.Keyword;[for]; CSharpTokenType.Keyword;[for];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

90
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsForeachKeyword.syntaxtree.txt
View File

@ -1,46 +1,46 @@
Statement block - Gen<None> - 86 - (0:0,0) Statement block - Gen<None> - 86 - (0:0,0)
Code span - Gen<Stmt> - [foreach(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:44 Code span - Gen<Stmt> - [foreach(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:44
CSharpSymbolType.Keyword;[foreach]; CSharpTokenType.Keyword;[foreach];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

90
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsIfKeywordWithNoElseBranches.syntaxtree.txt
View File

@ -1,46 +1,46 @@
Statement block - Gen<None> - 81 - (0:0,0) Statement block - Gen<None> - 81 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:44 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:44
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

34
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsLockKeyword.syntaxtree.txt
View File

@ -1,18 +1,18 @@
Statement block - Gen<None> - 44 - (0:0,0) Statement block - Gen<None> - 44 - (0:0,0)
Code span - Gen<Stmt> - [lock(foo) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:16 Code span - Gen<Stmt> - [lock(foo) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:16
CSharpSymbolType.Keyword;[lock]; CSharpTokenType.Keyword;[lock];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

106
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsSwitchKeyword.syntaxtree.txt
View File

@ -1,54 +1,54 @@
Statement block - Gen<None> - 161 - (0:0,0) Statement block - Gen<None> - 161 - (0:0,0)
Code span - Gen<Stmt> - [switch(foo) {LF case 0:LF break;LF case 1:LF {LF break;LF }LF case 2:LF return;LF default:LF return;LF}] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:52 Code span - Gen<Stmt> - [switch(foo) {LF case 0:LF break;LF case 1:LF {LF break;LF }LF case 2:LF return;LF default:LF return;LF}] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:52
CSharpSymbolType.Keyword;[switch]; CSharpTokenType.Keyword;[switch];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[case]; CSharpTokenType.Keyword;[case];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Colon;[:]; CSharpTokenType.Colon;[:];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[break]; CSharpTokenType.Keyword;[break];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[case]; CSharpTokenType.Keyword;[case];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[1]; CSharpTokenType.IntegerLiteral;[1];
CSharpSymbolType.Colon;[:]; CSharpTokenType.Colon;[:];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[break]; CSharpTokenType.Keyword;[break];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[case]; CSharpTokenType.Keyword;[case];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[2]; CSharpTokenType.IntegerLiteral;[2];
CSharpSymbolType.Colon;[:]; CSharpTokenType.Colon;[:];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[return]; CSharpTokenType.Keyword;[return];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[default]; CSharpTokenType.Keyword;[default];
CSharpSymbolType.Colon;[:]; CSharpTokenType.Colon;[:];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[return]; CSharpTokenType.Keyword;[return];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

90
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsUsingKeywordFollowedByParen.syntaxtree.txt
View File

@ -1,46 +1,46 @@
Statement block - Gen<None> - 84 - (0:0,0) Statement block - Gen<None> - 84 - (0:0,0)
Code span - Gen<Stmt> - [using(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:44 Code span - Gen<Stmt> - [using(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:44
CSharpSymbolType.Keyword;[using]; CSharpTokenType.Keyword;[using];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

90
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockSkipsParenthesisedExpressionAndThenBalancesBracesIfFirstIdentifierIsWhileKeyword.syntaxtree.txt
View File

@ -1,46 +1,46 @@
Statement block - Gen<None> - 84 - (0:0,0) Statement block - Gen<None> - 84 - (0:0,0)
Code span - Gen<Stmt> - [while(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:44 Code span - Gen<Stmt> - [while(int i = 0; i < 10; new Foo { Bar = "baz" }) { Debug.WriteLine(@"foo } bar"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:44
CSharpSymbolType.Keyword;[while]; CSharpTokenType.Keyword;[while];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

72
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockStopsParsingCatchClausesAfterFinallyBlock.syntaxtree.txt
View File

@ -1,37 +1,37 @@
Statement block - Gen<None> - 55 - (0:0,0) Statement block - Gen<None> - 55 - (0:0,0)
Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:35 Code span - Gen<Stmt> - [try { var foo = new { } } finally { var foo = new { } }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:35
CSharpSymbolType.Keyword;[try]; CSharpTokenType.Keyword;[try];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[finally]; CSharpTokenType.Keyword;[finally];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[var]; CSharpTokenType.Identifier;[var];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[foo]; CSharpTokenType.Identifier;[foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

216
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockStopsParsingCodeAfterElseBranch.syntaxtree.txt
View File

@ -1,109 +1,109 @@
Statement block - Gen<None> - 220 - (0:0,0) Statement block - Gen<None> - 220 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else { Debug.WriteLine(@"bar } baz"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Symbols:107 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF} else if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"bar } baz");LF} else { Debug.WriteLine(@"bar } baz"); }] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:107
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[else]; CSharpTokenType.Keyword;[else];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"bar } baz"]; CSharpTokenType.StringLiteral;[@"bar } baz"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

92
test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/CSharpBlockTest/ParseBlockStopsParsingIfIfStatementNotFollowedByElse.syntaxtree.txt
View File

@ -1,47 +1,47 @@
Statement block - Gen<None> - 87 - (0:0,0) Statement block - Gen<None> - 87 - (0:0,0)
Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Symbols:45 Code span - Gen<Stmt> - [if(int i = 0; i < 10; new Foo { Bar = "baz" }) {LF Debug.WriteLine(@"foo } bar");LF}] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:45
CSharpSymbolType.Keyword;[if]; CSharpTokenType.Keyword;[if];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.Keyword;[int]; CSharpTokenType.Keyword;[int];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[0]; CSharpTokenType.IntegerLiteral;[0];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[i]; CSharpTokenType.Identifier;[i];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LessThan;[<]; CSharpTokenType.LessThan;[<];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.IntegerLiteral;[10]; CSharpTokenType.IntegerLiteral;[10];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Keyword;[new]; CSharpTokenType.Keyword;[new];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Foo]; CSharpTokenType.Identifier;[Foo];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Bar]; CSharpTokenType.Identifier;[Bar];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Assign;[=]; CSharpTokenType.Assign;[=];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.StringLiteral;["baz"]; CSharpTokenType.StringLiteral;["baz"];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.LeftBrace;[{]; CSharpTokenType.LeftBrace;[{];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.WhiteSpace;[ ]; CSharpTokenType.WhiteSpace;[ ];
CSharpSymbolType.Identifier;[Debug]; CSharpTokenType.Identifier;[Debug];
CSharpSymbolType.Dot;[.]; CSharpTokenType.Dot;[.];
CSharpSymbolType.Identifier;[WriteLine]; CSharpTokenType.Identifier;[WriteLine];
CSharpSymbolType.LeftParenthesis;[(]; CSharpTokenType.LeftParenthesis;[(];
CSharpSymbolType.StringLiteral;[@"foo } bar"]; CSharpTokenType.StringLiteral;[@"foo } bar"];
CSharpSymbolType.RightParenthesis;[)]; CSharpTokenType.RightParenthesis;[)];
CSharpSymbolType.Semicolon;[;]; CSharpTokenType.Semicolon;[;];
CSharpSymbolType.NewLine;[LF]; CSharpTokenType.NewLine;[LF];
CSharpSymbolType.RightBrace;[}]; CSharpTokenType.RightBrace;[}];

Some files were not shown because too many files have changed in this diff Show More