diff --git a/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs b/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs
index dae31efbd2..a711567547 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorIntermediateNodeLoweringPhase.cs
@@ -593,14 +593,14 @@ namespace Microsoft.AspNetCore.Razor.Language
public override void VisitMarkupSpan(MarkupChunkGenerator chunkGenerator, Span span)
{
- if (span.Symbols.Count == 1)
+ if (span.Tokens.Count == 1)
{
- var symbol = span.Symbols[0] as HtmlSymbol;
- if (symbol != null &&
- symbol.Type == HtmlSymbolType.Unknown &&
- symbol.Content.Length == 0)
+ var token = span.Tokens[0] as HtmlToken;
+ if (token != null &&
+ token.Type == HtmlTokenType.Unknown &&
+ token.Content.Length == 0)
{
- // We don't want to create IR nodes for marker symbols.
+ // We don't want to create IR nodes for marker tokens.
return;
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs b/src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs
index 6abaa03f5c..308c274518 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/DirectiveTokenEditHandler.cs
@@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Razor.Language
{
internal class DirectiveTokenEditHandler : SpanEditHandler
{
- public DirectiveTokenEditHandler(Func> tokenizer) : base(tokenizer)
+ public DirectiveTokenEditHandler(Func> tokenizer) : base(tokenizer)
{
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Extensions/DesignTimeDirectiveTargetExtension.cs b/src/Microsoft.AspNetCore.Razor.Language/Extensions/DesignTimeDirectiveTargetExtension.cs
index bc73343044..316c255bfe 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Extensions/DesignTimeDirectiveTargetExtension.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Extensions/DesignTimeDirectiveTargetExtension.cs
@@ -59,8 +59,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
if (string.IsNullOrEmpty(node.Content))
{
- // This is most likely a marker symbol.
- WriteMarkerSymbol(context, node);
+ // This is most likely a marker token.
+ WriteMarkerToken(context, node);
break;
}
@@ -80,8 +80,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
if (string.IsNullOrEmpty(node.Content))
{
- // This is most likely a marker symbol.
- WriteMarkerSymbol(context, node);
+ // This is most likely a marker token.
+ WriteMarkerToken(context, node);
break;
}
@@ -102,8 +102,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
if (string.IsNullOrEmpty(node.Content))
{
- // This is most likely a marker symbol.
- WriteMarkerSymbol(context, node);
+ // This is most likely a marker token.
+ WriteMarkerToken(context, node);
break;
}
@@ -155,9 +155,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Extensions
context.CodeWriter.WriteLine("))();");
}
- private void WriteMarkerSymbol(CodeRenderingContext context, DirectiveTokenIntermediateNode node)
+ private void WriteMarkerToken(CodeRenderingContext context, DirectiveTokenIntermediateNode node)
{
- // We want to map marker symbols to a location in the generated document
+ // We want to map marker tokens to a location in the generated document
// that will provide CSharp intellisense.
context.AddSourceMappingFor(node);
context.CodeWriter.Write(" ");
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs
index bd73bbc1b2..3aecb29a52 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/AutoCompleteEditHandler.cs
@@ -11,18 +11,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
private static readonly int TypeHashCode = typeof(AutoCompleteEditHandler).GetHashCode();
- public AutoCompleteEditHandler(Func> tokenizer)
+ public AutoCompleteEditHandler(Func> tokenizer)
: base(tokenizer)
{
}
- public AutoCompleteEditHandler(Func> tokenizer, bool autoCompleteAtEndOfSpan)
+ public AutoCompleteEditHandler(Func> tokenizer, bool autoCompleteAtEndOfSpan)
: this(tokenizer)
{
AutoCompleteAtEndOfSpan = autoCompleteAtEndOfSpan;
}
- public AutoCompleteEditHandler(Func> tokenizer, AcceptedCharactersInternal accepted)
+ public AutoCompleteEditHandler(Func> tokenizer, AcceptedCharactersInternal accepted)
: base(tokenizer, accepted)
{
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
index b95916b622..a75c983fec 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
@@ -8,14 +8,14 @@ using System.Linq;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class CSharpCodeParser : TokenizerBackedParser
+ internal class CSharpCodeParser : TokenizerBackedParser
{
private static HashSet InvalidNonWhitespaceNameCharacters = new HashSet(new[]
{
'@', '!', '<', '/', '?', '[', '>', ']', '=', '"', '\'', '*'
});
- private static readonly Func IsValidStatementSpacingSymbol =
+ private static readonly Func IsValidStatementSpacingToken =
IsSpacingToken(includeNewLines: true, includeComments: true);
internal static readonly DirectiveDescriptor AddTagHelperDirectiveDescriptor = DirectiveDescriptor.CreateDirective(
@@ -102,7 +102,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public bool IsNested { get; set; }
- protected override bool SymbolTypeEquals(CSharpSymbolType x, CSharpSymbolType y) => x == y;
+ protected override bool TokenTypeEquals(CSharpTokenType x, CSharpTokenType y) => x == y;
protected void MapDirectives(Action handler, params string[] directives)
{
@@ -161,16 +161,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Conditional("DEBUG")]
internal void Assert(CSharpKeyword expectedKeyword)
{
- Debug.Assert(CurrentSymbol.Type == CSharpSymbolType.Keyword &&
- CurrentSymbol.Keyword.HasValue &&
- CurrentSymbol.Keyword.Value == expectedKeyword);
+ Debug.Assert(CurrentToken.Type == CSharpTokenType.Keyword &&
+ CurrentToken.Keyword.HasValue &&
+ CurrentToken.Keyword.Value == expectedKeyword);
}
protected internal bool At(CSharpKeyword keyword)
{
- return At(CSharpSymbolType.Keyword) &&
- CurrentSymbol.Keyword.HasValue &&
- CurrentSymbol.Keyword.Value == keyword;
+ return At(CSharpTokenType.Keyword) &&
+ CurrentToken.Keyword.HasValue &&
+ CurrentToken.Keyword.Value == keyword;
}
protected internal bool AcceptIf(CSharpKeyword keyword)
@@ -183,11 +183,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- protected static Func IsSpacingToken(bool includeNewLines, bool includeComments)
+ protected static Func IsSpacingToken(bool includeNewLines, bool includeComments)
{
- return sym => sym.Type == CSharpSymbolType.WhiteSpace ||
- (includeNewLines && sym.Type == CSharpSymbolType.NewLine) ||
- (includeComments && sym.Type == CSharpSymbolType.Comment);
+ return token => token.Type == CSharpTokenType.WhiteSpace ||
+ (includeNewLines && token.Type == CSharpTokenType.NewLine) ||
+ (includeComments && token.Type == CSharpTokenType.Comment);
}
public override void ParseBlock()
@@ -208,27 +208,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
- var current = CurrentSymbol;
- if (At(CSharpSymbolType.StringLiteral) &&
- CurrentSymbol.Content.Length > 0 &&
- CurrentSymbol.Content[0] == SyntaxConstants.TransitionCharacter)
+ var current = CurrentToken;
+ if (At(CSharpTokenType.StringLiteral) &&
+ CurrentToken.Content.Length > 0 &&
+ CurrentToken.Content[0] == SyntaxConstants.TransitionCharacter)
{
- var split = Language.SplitSymbol(CurrentSymbol, 1, CSharpSymbolType.Transition);
+ var split = Language.SplitToken(CurrentToken, 1, CSharpTokenType.Transition);
current = split.Item1;
// Back up to the end of the transition
Context.Source.Position -= split.Item2.Content.Length;
NextToken();
}
- else if (At(CSharpSymbolType.Transition))
+ else if (At(CSharpTokenType.Transition))
{
NextToken();
}
// Accept "@" if we see it, but if we don't, that's OK. We assume we were started for a good reason
- if (current.Type == CSharpSymbolType.Transition)
+ if (current.Type == CSharpTokenType.Transition)
{
- if (Span.Symbols.Count > 0)
+ if (Span.Tokens.Count > 0)
{
Output(SpanKindInternal.Code);
}
@@ -251,9 +251,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.ChunkGenerator = new StatementChunkGenerator();
}
- private void AtTransition(CSharpSymbol current)
+ private void AtTransition(CSharpToken current)
{
- Debug.Assert(current.Type == CSharpSymbolType.Transition);
+ Debug.Assert(current.Type == CSharpTokenType.Transition);
Accept(current);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
@@ -273,16 +273,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// What type of block is this?
if (!EndOfFile)
{
- if (CurrentSymbol.Type == CSharpSymbolType.LeftParenthesis)
+ if (CurrentToken.Type == CSharpTokenType.LeftParenthesis)
{
Context.Builder.CurrentBlock.Type = BlockKindInternal.Expression;
Context.Builder.CurrentBlock.ChunkGenerator = new ExpressionChunkGenerator();
ExplicitExpression();
return;
}
- else if (CurrentSymbol.Type == CSharpSymbolType.Identifier)
+ else if (CurrentToken.Type == CSharpTokenType.Identifier)
{
- if (TryGetDirectiveHandler(CurrentSymbol.Content, out var handler))
+ if (TryGetDirectiveHandler(CurrentToken.Content, out var handler))
{
Span.ChunkGenerator = SpanChunkGenerator.Null;
handler();
@@ -291,13 +291,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
if (string.Equals(
- CurrentSymbol.Content,
+ CurrentToken.Content,
SyntaxConstants.CSharp.HelperKeyword,
StringComparison.Ordinal))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_HelperDirectiveNotAvailable(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length)));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length)));
}
Context.Builder.CurrentBlock.Type = BlockKindInternal.Expression;
@@ -306,9 +306,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
}
- else if (CurrentSymbol.Type == CSharpSymbolType.Keyword)
+ else if (CurrentToken.Type == CSharpTokenType.Keyword)
{
- if (TryGetDirectiveHandler(CurrentSymbol.Content, out var handler))
+ if (TryGetDirectiveHandler(CurrentToken.Content, out var handler))
{
Span.ChunkGenerator = SpanChunkGenerator.Null;
handler();
@@ -320,7 +320,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
}
- else if (CurrentSymbol.Type == CSharpSymbolType.LeftBrace)
+ else if (CurrentToken.Type == CSharpTokenType.LeftBrace)
{
VerbatimBlock();
return;
@@ -330,7 +330,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Invalid character
Context.Builder.CurrentBlock.Type = BlockKindInternal.Expression;
Context.Builder.CurrentBlock.ChunkGenerator = new ExpressionChunkGenerator();
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Span.ChunkGenerator = new ExpressionChunkGenerator();
Span.EditHandler = new ImplicitExpressionEditHandler(
Language.TokenizeString,
@@ -339,11 +339,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
AcceptedCharacters = AcceptedCharactersInternal.NonWhiteSpace
};
- if (At(CSharpSymbolType.WhiteSpace) || At(CSharpSymbolType.NewLine))
+ if (At(CSharpTokenType.WhiteSpace) || At(CSharpTokenType.NewLine))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedWhiteSpaceAtStartOfCodeBlock(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length)));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length)));
}
else if (EndOfFile)
{
@@ -355,8 +355,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedCharacterAtStartOfCodeBlock(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length),
- CurrentSymbol.Content));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length),
+ CurrentToken.Content));
}
}
finally
@@ -369,7 +369,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void VerbatimBlock()
{
- Assert(CSharpSymbolType.LeftBrace);
+ Assert(CSharpTokenType.LeftBrace);
var block = new Block(Resources.BlockName_Code, CurrentStart);
AcceptAndMoveNext();
@@ -384,14 +384,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
CodeBlock(false, block);
Span.ChunkGenerator = new StatementChunkGenerator();
- AddMarkerSymbolIfNecessary();
- if (!At(CSharpSymbolType.RightBrace))
+ AddMarkerTokenIfNecessary();
+ if (!At(CSharpTokenType.RightBrace))
{
editHandler.AutoCompleteString = "}";
}
Output(SpanKindInternal.Code);
- if (Optional(CSharpSymbolType.RightBrace))
+ if (Optional(CSharpTokenType.RightBrace))
{
// Set up the "}" span
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
@@ -401,8 +401,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!IsNested)
{
EnsureCurrent();
- if (At(CSharpSymbolType.NewLine) ||
- (At(CSharpSymbolType.WhiteSpace) && NextIs(CSharpSymbolType.NewLine)))
+ if (At(CSharpTokenType.NewLine) ||
+ (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.NewLine)))
{
Context.NullGenerateWhitespaceAndNewLine = true;
}
@@ -456,13 +456,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
if (!EndOfFile)
{
- if (CurrentSymbol.Type == CSharpSymbolType.LeftParenthesis ||
- CurrentSymbol.Type == CSharpSymbolType.LeftBracket)
+ if (CurrentToken.Type == CSharpTokenType.LeftParenthesis ||
+ CurrentToken.Type == CSharpTokenType.LeftBracket)
{
// If we end within "(", whitespace is fine
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
- CSharpSymbolType right;
+ CSharpTokenType right;
bool success;
using (PushSpanConfig((span, prev) =>
@@ -471,13 +471,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
}))
{
- right = Language.FlipBracket(CurrentSymbol.Type);
+ right = Language.FlipBracket(CurrentToken.Type);
success = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
}
if (!success)
{
- AcceptUntil(CSharpSymbolType.LessThan);
+ AcceptUntil(CSharpTokenType.LessThan);
}
if (At(right))
{
@@ -488,22 +488,22 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
return MethodCallOrArrayIndex(acceptedCharacters);
}
- if (At(CSharpSymbolType.QuestionMark))
+ if (At(CSharpTokenType.QuestionMark))
{
var next = Lookahead(count: 1);
if (next != null)
{
- if (next.Type == CSharpSymbolType.Dot)
+ if (next.Type == CSharpTokenType.Dot)
{
// Accept null conditional dot operator (?.).
AcceptAndMoveNext();
AcceptAndMoveNext();
// If the next piece after the ?. is a keyword or identifier then we want to continue.
- return At(CSharpSymbolType.Identifier) || At(CSharpSymbolType.Keyword);
+ return At(CSharpTokenType.Identifier) || At(CSharpTokenType.Keyword);
}
- else if (next.Type == CSharpSymbolType.LeftBracket)
+ else if (next.Type == CSharpTokenType.LeftBracket)
{
// We're at the ? for a null conditional bracket operator (?[).
AcceptAndMoveNext();
@@ -513,12 +513,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
}
- else if (At(CSharpSymbolType.Dot))
+ else if (At(CSharpTokenType.Dot))
{
- var dot = CurrentSymbol;
+ var dot = CurrentToken;
if (NextToken())
{
- if (At(CSharpSymbolType.Identifier) || At(CSharpSymbolType.Keyword))
+ if (At(CSharpTokenType.Identifier) || At(CSharpTokenType.Keyword))
{
// Accept the dot and return to the start
Accept(dot);
@@ -526,7 +526,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else
{
- // Put the symbol back
+ // Put the token back
PutCurrentBack();
}
}
@@ -540,7 +540,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(dot);
}
}
- else if (!At(CSharpSymbolType.WhiteSpace) && !At(CSharpSymbolType.NewLine))
+ else if (!At(CSharpTokenType.WhiteSpace) && !At(CSharpTokenType.NewLine))
{
PutCurrentBack();
}
@@ -564,7 +564,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
if (insertMarkerIfNecessary && Context.Builder.LastAcceptedCharacters != AcceptedCharactersInternal.Any)
{
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
}
EnsureCurrent();
@@ -587,8 +587,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaptureWhitespaceAtEndOfCodeOnlyLine()
{
- var whitespace = ReadWhile(sym => sym.Type == CSharpSymbolType.WhiteSpace);
- if (At(CSharpSymbolType.NewLine))
+ var whitespace = ReadWhile(token => token.Type == CSharpTokenType.WhiteSpace);
+ if (At(CSharpTokenType.NewLine))
{
Accept(whitespace);
AcceptAndMoveNext();
@@ -610,7 +610,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ExplicitExpression()
{
var block = new Block(Resources.BlockName_ExplicitExpression, CurrentStart);
- Assert(CSharpSymbolType.LeftParenthesis);
+ Assert(CSharpTokenType.LeftParenthesis);
AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
@@ -621,28 +621,28 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
BalancingModes.BacktrackOnFailure |
BalancingModes.NoErrorOnFailure |
BalancingModes.AllowCommentsAndTemplates,
- CSharpSymbolType.LeftParenthesis,
- CSharpSymbolType.RightParenthesis,
+ CSharpTokenType.LeftParenthesis,
+ CSharpTokenType.RightParenthesis,
block.Start);
if (!success)
{
- AcceptUntil(CSharpSymbolType.LessThan);
+ AcceptUntil(CSharpTokenType.LessThan);
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF(
new SourceSpan(block.Start, contentLength: 1 /* ( */), block.Name, ")", "("));
}
- // If necessary, put an empty-content marker symbol here
- if (Span.Symbols.Count == 0)
+ // If necessary, put an empty-content marker token here
+ if (Span.Tokens.Count == 0)
{
- Accept(new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown));
+ Accept(new CSharpToken(string.Empty, CSharpTokenType.Unknown));
}
// Output the content span and then capture the ")"
Output(SpanKindInternal.Code);
}
- Optional(CSharpSymbolType.RightParenthesis);
+ Optional(CSharpTokenType.RightParenthesis);
if (!EndOfFile)
{
PutCurrentBack();
@@ -701,19 +701,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// No embedded transitions in C#, so ignore that param
return allowTemplatesAndComments
- && ((Language.IsTransition(CurrentSymbol)
- && NextIs(CSharpSymbolType.LessThan, CSharpSymbolType.Colon, CSharpSymbolType.DoubleColon))
- || Language.IsCommentStart(CurrentSymbol));
+ && ((Language.IsTransition(CurrentToken)
+ && NextIs(CSharpTokenType.LessThan, CSharpTokenType.Colon, CSharpTokenType.DoubleColon))
+ || Language.IsCommentStart(CurrentToken));
}
protected override void HandleEmbeddedTransition()
{
- if (Language.IsTransition(CurrentSymbol))
+ if (Language.IsTransition(CurrentToken))
{
PutCurrentBack();
Template();
}
- else if (Language.IsCommentStart(CurrentSymbol))
+ else if (Language.IsCommentStart(CurrentToken))
{
RazorComment();
}
@@ -761,7 +761,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_ReservedWord(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length), CurrentSymbol.Content));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), CurrentToken.Content));
AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
@@ -783,12 +783,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CaseStatement(bool topLevel)
{
- Assert(CSharpSymbolType.Keyword);
- Debug.Assert(CurrentSymbol.Keyword != null &&
- (CurrentSymbol.Keyword.Value == CSharpKeyword.Case ||
- CurrentSymbol.Keyword.Value == CSharpKeyword.Default));
- AcceptUntil(CSharpSymbolType.Colon);
- Optional(CSharpSymbolType.Colon);
+ Assert(CSharpTokenType.Keyword);
+ Debug.Assert(CurrentToken.Keyword != null &&
+ (CurrentToken.Keyword.Value == CSharpKeyword.Case ||
+ CurrentToken.Keyword.Value == CSharpKeyword.Default));
+ AcceptUntil(CSharpTokenType.Colon);
+ Optional(CSharpTokenType.Colon);
}
private void DoStatement(bool topLevel)
@@ -813,7 +813,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Assert(CSharpKeyword.While);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
- if (AcceptCondition() && Optional(CSharpSymbolType.Semicolon))
+ if (AcceptCondition() && Optional(CSharpTokenType.Semicolon))
{
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
}
@@ -828,16 +828,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void UsingKeyword(bool topLevel)
{
Assert(CSharpKeyword.Using);
- var block = new Block(CurrentSymbol, CurrentStart);
+ var block = new Block(CurrentToken, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
- if (At(CSharpSymbolType.LeftParenthesis))
+ if (At(CSharpTokenType.LeftParenthesis))
{
// using ( ==> Using Statement
UsingStatement(block);
}
- else if (At(CSharpSymbolType.Identifier) || At(CSharpKeyword.Static))
+ else if (At(CSharpTokenType.Identifier) || At(CSharpKeyword.Static))
{
// using Identifier ==> Using Declaration
if (!topLevel)
@@ -865,16 +865,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Context.Builder.CurrentBlock.Type = BlockKindInternal.Directive;
var start = CurrentStart;
- if (At(CSharpSymbolType.Identifier))
+ if (At(CSharpTokenType.Identifier))
{
// non-static using
NamespaceOrTypeName();
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
- if (At(CSharpSymbolType.Assign))
+ if (At(CSharpTokenType.Assign))
{
// Alias
Accept(whitespace);
- Assert(CSharpSymbolType.Assign);
+ Assert(CSharpTokenType.Assign);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@@ -898,13 +898,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.AnyExceptNewline;
Span.ChunkGenerator = new AddImportChunkGenerator(new LocationTagged(
- string.Concat(Span.Symbols.Skip(1).Select(s => s.Content)),
+ string.Concat(Span.Tokens.Skip(1).Select(s => s.Content)),
start));
// Optional ";"
if (EnsureCurrent())
{
- Optional(CSharpSymbolType.Semicolon);
+ Optional(CSharpTokenType.Semicolon);
}
}
@@ -920,15 +920,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var tokens = ReadWhile(token =>
{
var type = token.Type;
- if ((expectingDot && type == CSharpSymbolType.Dot) ||
- (!expectingDot && type == CSharpSymbolType.Identifier))
+ if ((expectingDot && type == CSharpTokenType.Dot) ||
+ (!expectingDot && type == CSharpTokenType.Identifier))
{
expectingDot = !expectingDot;
return true;
}
- if (type != CSharpSymbolType.WhiteSpace &&
- type != CSharpSymbolType.NewLine)
+ if (type != CSharpTokenType.WhiteSpace &&
+ type != CSharpTokenType.NewLine)
{
expectingDot = false;
currentIdentifierLength += token.Content.Length;
@@ -966,69 +966,69 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool NamespaceOrTypeName()
{
- if (Optional(CSharpSymbolType.LeftParenthesis))
+ if (Optional(CSharpTokenType.LeftParenthesis))
{
- while (!Optional(CSharpSymbolType.RightParenthesis) && !EndOfFile)
+ while (!Optional(CSharpTokenType.RightParenthesis) && !EndOfFile)
{
- Optional(CSharpSymbolType.WhiteSpace);
+ Optional(CSharpTokenType.WhiteSpace);
if (!NamespaceOrTypeName())
{
return false;
}
- Optional(CSharpSymbolType.WhiteSpace);
- Optional(CSharpSymbolType.Identifier);
- Optional(CSharpSymbolType.WhiteSpace);
- Optional(CSharpSymbolType.Comma);
+ Optional(CSharpTokenType.WhiteSpace);
+ Optional(CSharpTokenType.Identifier);
+ Optional(CSharpTokenType.WhiteSpace);
+ Optional(CSharpTokenType.Comma);
}
- if (At(CSharpSymbolType.WhiteSpace) && NextIs(CSharpSymbolType.QuestionMark))
+ if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.QuestionMark))
{
// Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext();
}
- Optional(CSharpSymbolType.QuestionMark); // Nullable
+ Optional(CSharpTokenType.QuestionMark); // Nullable
return true;
}
- else if (Optional(CSharpSymbolType.Identifier) || Optional(CSharpSymbolType.Keyword))
+ else if (Optional(CSharpTokenType.Identifier) || Optional(CSharpTokenType.Keyword))
{
- if (Optional(CSharpSymbolType.DoubleColon))
+ if (Optional(CSharpTokenType.DoubleColon))
{
- if (!Optional(CSharpSymbolType.Identifier))
+ if (!Optional(CSharpTokenType.Identifier))
{
- Optional(CSharpSymbolType.Keyword);
+ Optional(CSharpTokenType.Keyword);
}
}
- if (At(CSharpSymbolType.LessThan))
+ if (At(CSharpTokenType.LessThan))
{
TypeArgumentList();
}
- if (Optional(CSharpSymbolType.Dot))
+ if (Optional(CSharpTokenType.Dot))
{
NamespaceOrTypeName();
}
- if (At(CSharpSymbolType.WhiteSpace) && NextIs(CSharpSymbolType.QuestionMark))
+ if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.QuestionMark))
{
// Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext();
}
- Optional(CSharpSymbolType.QuestionMark); // Nullable
+ Optional(CSharpTokenType.QuestionMark); // Nullable
- if (At(CSharpSymbolType.WhiteSpace) && NextIs(CSharpSymbolType.LeftBracket))
+ if (At(CSharpTokenType.WhiteSpace) && NextIs(CSharpTokenType.LeftBracket))
{
// Only accept the whitespace if we are going to consume the next token.
AcceptAndMoveNext();
}
- while (At(CSharpSymbolType.LeftBracket))
+ while (At(CSharpTokenType.LeftBracket))
{
Balance(BalancingModes.None);
- Optional(CSharpSymbolType.RightBracket);
+ Optional(CSharpTokenType.RightBracket);
}
return true;
}
@@ -1040,14 +1040,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TypeArgumentList()
{
- Assert(CSharpSymbolType.LessThan);
+ Assert(CSharpTokenType.LessThan);
Balance(BalancingModes.None);
- Optional(CSharpSymbolType.GreaterThan);
+ Optional(CSharpTokenType.GreaterThan);
}
private void UsingStatement(Block block)
{
- Assert(CSharpSymbolType.LeftParenthesis);
+ Assert(CSharpTokenType.LeftParenthesis);
// Parse condition
if (AcceptCondition())
@@ -1136,7 +1136,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return;
}
- var block = new Block(CurrentSymbol, CurrentStart);
+ var block = new Block(CurrentToken, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@@ -1159,13 +1159,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!EndOfFile)
{
// Check for "{" to make sure we're at a block
- if (!At(CSharpSymbolType.LeftBrace))
+ if (!At(CSharpTokenType.LeftBrace))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_SingleLineControlFlowStatementsNotAllowed(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length),
- Language.GetSample(CSharpSymbolType.LeftBrace),
- CurrentSymbol.Content));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length),
+ Language.GetSample(CSharpTokenType.LeftBrace),
+ CurrentToken.Content));
}
// Parse the statement and then we're done
@@ -1175,8 +1175,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void UnconditionalBlock()
{
- Assert(CSharpSymbolType.Keyword);
- var block = new Block(CurrentSymbol, CurrentStart);
+ Assert(CSharpTokenType.Keyword);
+ var block = new Block(CurrentToken, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
ExpectCodeBlock(block);
@@ -1186,22 +1186,22 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Assert(CSharpKeyword.Catch);
- var block = new Block(CurrentSymbol, CurrentStart);
+ var block = new Block(CurrentToken, CurrentStart);
// Accept "catch"
AcceptAndMoveNext();
- AcceptWhile(IsValidStatementSpacingSymbol);
+ AcceptWhile(IsValidStatementSpacingToken);
// Parse the catch condition if present. If not present, let the C# compiler complain.
if (AcceptCondition())
{
- AcceptWhile(IsValidStatementSpacingSymbol);
+ AcceptWhile(IsValidStatementSpacingToken);
if (At(CSharpKeyword.When))
{
// Accept "when".
AcceptAndMoveNext();
- AcceptWhile(IsValidStatementSpacingSymbol);
+ AcceptWhile(IsValidStatementSpacingToken);
// Parse the filter condition if present. If not present, let the C# compiler complain.
if (!AcceptCondition())
@@ -1210,7 +1210,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
- AcceptWhile(IsValidStatementSpacingSymbol);
+ AcceptWhile(IsValidStatementSpacingToken);
}
ExpectCodeBlock(block);
@@ -1219,8 +1219,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ConditionalBlock(bool topLevel)
{
- Assert(CSharpSymbolType.Keyword);
- var block = new Block(CurrentSymbol, CurrentStart);
+ Assert(CSharpTokenType.Keyword);
+ var block = new Block(CurrentToken, CurrentStart);
ConditionalBlock(block);
if (topLevel)
{
@@ -1243,16 +1243,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool AcceptCondition()
{
- if (At(CSharpSymbolType.LeftParenthesis))
+ if (At(CSharpTokenType.LeftParenthesis))
{
var complete = Balance(BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
if (!complete)
{
- AcceptUntil(CSharpSymbolType.NewLine);
+ AcceptUntil(CSharpTokenType.NewLine);
}
else
{
- Optional(CSharpSymbolType.RightParenthesis);
+ Optional(CSharpTokenType.RightParenthesis);
}
return complete;
}
@@ -1280,16 +1280,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
- var type = CurrentSymbol.Type;
+ var type = CurrentToken.Type;
var loc = CurrentStart;
// Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text
- var isSingleLineMarkup = type == CSharpSymbolType.Transition &&
- (NextIs(CSharpSymbolType.Colon, CSharpSymbolType.DoubleColon));
+ var isSingleLineMarkup = type == CSharpTokenType.Transition &&
+ (NextIs(CSharpTokenType.Colon, CSharpTokenType.DoubleColon));
var isMarkup = isSingleLineMarkup ||
- type == CSharpSymbolType.LessThan ||
- (type == CSharpSymbolType.Transition && NextIs(CSharpSymbolType.LessThan));
+ type == CSharpTokenType.LessThan ||
+ (type == CSharpTokenType.Transition && NextIs(CSharpTokenType.LessThan));
if (Context.DesignTimeMode || !isMarkup)
{
@@ -1301,14 +1301,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else
{
- var nextSymbol = Lookahead(1);
+ var nextToken = Lookahead(1);
// MARKUP owns whitespace EXCEPT in DesignTimeMode.
PutCurrentBack();
// Put back the whitespace unless it precedes a '' tag.
- if (nextSymbol != null &&
- !string.Equals(nextSymbol.Content, SyntaxConstants.TextTagName, StringComparison.Ordinal))
+ if (nextToken != null &&
+ !string.Equals(nextToken.Content, SyntaxConstants.TextTagName, StringComparison.Ordinal))
{
PutBack(lastWhitespace);
}
@@ -1321,7 +1321,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (isMarkup)
{
- if (type == CSharpSymbolType.Transition && !isSingleLineMarkup)
+ if (type == CSharpTokenType.Transition && !isSingleLineMarkup)
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_AtInCodeMustBeFollowedByColonParenOrIdentifierStart(
@@ -1330,8 +1330,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Markup block
Output(SpanKindInternal.Code);
- if (Context.DesignTimeMode && CurrentSymbol != null &&
- (CurrentSymbol.Type == CSharpSymbolType.LessThan || CurrentSymbol.Type == CSharpSymbolType.Transition))
+ if (Context.DesignTimeMode && CurrentToken != null &&
+ (CurrentToken.Type == CSharpTokenType.LessThan || CurrentToken.Type == CSharpTokenType.Transition))
{
PutCurrentBack();
}
@@ -1344,33 +1344,33 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- private void HandleStatement(Block block, CSharpSymbolType type)
+ private void HandleStatement(Block block, CSharpTokenType type)
{
switch (type)
{
- case CSharpSymbolType.RazorCommentTransition:
+ case CSharpTokenType.RazorCommentTransition:
Output(SpanKindInternal.Code);
RazorComment();
Statement(block);
break;
- case CSharpSymbolType.LeftBrace:
+ case CSharpTokenType.LeftBrace:
// Verbatim Block
block = block ?? new Block(Resources.BlockName_Code, CurrentStart);
AcceptAndMoveNext();
CodeBlock(block);
break;
- case CSharpSymbolType.Keyword:
+ case CSharpTokenType.Keyword:
// Keyword block
HandleKeyword(false, StandardStatement);
break;
- case CSharpSymbolType.Transition:
+ case CSharpTokenType.Transition:
// Embedded Expression block
EmbeddedExpression();
break;
- case CSharpSymbolType.RightBrace:
+ case CSharpTokenType.RightBrace:
// Possible end of Code Block, just run the continuation
break;
- case CSharpSymbolType.Comment:
+ case CSharpTokenType.Comment:
AcceptAndMoveNext();
break;
default:
@@ -1383,11 +1383,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void EmbeddedExpression()
{
// First, verify the type of the block
- Assert(CSharpSymbolType.Transition);
- var transition = CurrentSymbol;
+ Assert(CSharpTokenType.Transition);
+ var transition = CurrentToken;
NextToken();
- if (At(CSharpSymbolType.Transition))
+ if (At(CSharpTokenType.Transition))
{
// Escaped "@"
Output(SpanKindInternal.Code);
@@ -1397,14 +1397,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Code);
- Assert(CSharpSymbolType.Transition);
+ Assert(CSharpTokenType.Transition);
AcceptAndMoveNext();
StandardStatement();
}
else
{
// Throw errors as necessary, but continue parsing
- if (At(CSharpSymbolType.LeftBrace))
+ if (At(CSharpTokenType.LeftBrace))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedNestedCodeBlock(
@@ -1416,7 +1416,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
PutBack(transition);
// Before exiting, add a marker span if necessary
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
NestedBlock();
}
@@ -1427,49 +1427,49 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
while (!EndOfFile)
{
var bookmark = CurrentStart.AbsoluteIndex;
- var read = ReadWhile(sym =>
- sym.Type != CSharpSymbolType.Semicolon &&
- sym.Type != CSharpSymbolType.RazorCommentTransition &&
- sym.Type != CSharpSymbolType.Transition &&
- sym.Type != CSharpSymbolType.LeftBrace &&
- sym.Type != CSharpSymbolType.LeftParenthesis &&
- sym.Type != CSharpSymbolType.LeftBracket &&
- sym.Type != CSharpSymbolType.RightBrace);
+ var read = ReadWhile(token =>
+ token.Type != CSharpTokenType.Semicolon &&
+ token.Type != CSharpTokenType.RazorCommentTransition &&
+ token.Type != CSharpTokenType.Transition &&
+ token.Type != CSharpTokenType.LeftBrace &&
+ token.Type != CSharpTokenType.LeftParenthesis &&
+ token.Type != CSharpTokenType.LeftBracket &&
+ token.Type != CSharpTokenType.RightBrace);
- if (At(CSharpSymbolType.LeftBrace) ||
- At(CSharpSymbolType.LeftParenthesis) ||
- At(CSharpSymbolType.LeftBracket))
+ if (At(CSharpTokenType.LeftBrace) ||
+ At(CSharpTokenType.LeftParenthesis) ||
+ At(CSharpTokenType.LeftBracket))
{
Accept(read);
if (Balance(BalancingModes.AllowCommentsAndTemplates | BalancingModes.BacktrackOnFailure))
{
- Optional(CSharpSymbolType.RightBrace);
+ Optional(CSharpTokenType.RightBrace);
}
else
{
// Recovery
- AcceptUntil(CSharpSymbolType.LessThan, CSharpSymbolType.RightBrace);
+ AcceptUntil(CSharpTokenType.LessThan, CSharpTokenType.RightBrace);
return;
}
}
- else if (At(CSharpSymbolType.Transition) && (NextIs(CSharpSymbolType.LessThan, CSharpSymbolType.Colon)))
+ else if (At(CSharpTokenType.Transition) && (NextIs(CSharpTokenType.LessThan, CSharpTokenType.Colon)))
{
Accept(read);
Output(SpanKindInternal.Code);
Template();
}
- else if (At(CSharpSymbolType.RazorCommentTransition))
+ else if (At(CSharpTokenType.RazorCommentTransition))
{
Accept(read);
RazorComment();
}
- else if (At(CSharpSymbolType.Semicolon))
+ else if (At(CSharpTokenType.Semicolon))
{
Accept(read);
AcceptAndMoveNext();
return;
}
- else if (At(CSharpSymbolType.RightBrace))
+ else if (At(CSharpTokenType.RightBrace))
{
Accept(read);
return;
@@ -1478,7 +1478,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.Source.Position = bookmark;
NextToken();
- AcceptUntil(CSharpSymbolType.LessThan, CSharpSymbolType.LeftBrace, CSharpSymbolType.RightBrace);
+ AcceptUntil(CSharpTokenType.LessThan, CSharpTokenType.LeftBrace, CSharpTokenType.RightBrace);
return;
}
}
@@ -1492,7 +1492,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void CodeBlock(bool acceptTerminatingBrace, Block block)
{
EnsureCurrent();
- while (!EndOfFile && !At(CSharpSymbolType.RightBrace))
+ while (!EndOfFile && !At(CSharpTokenType.RightBrace))
{
// Parse a statement, then return here
Statement();
@@ -1507,7 +1507,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else if (acceptTerminatingBrace)
{
- Assert(CSharpSymbolType.RightBrace);
+ Assert(CSharpTokenType.RightBrace);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
AcceptAndMoveNext();
}
@@ -1515,8 +1515,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void HandleKeyword(bool topLevel, Action fallback)
{
- Debug.Assert(CurrentSymbol.Type == CSharpSymbolType.Keyword && CurrentSymbol.Keyword != null);
- if (_keywordParsers.TryGetValue(CurrentSymbol.Keyword.Value, out var handler))
+ Debug.Assert(CurrentToken.Type == CSharpTokenType.Keyword && CurrentToken.Keyword != null);
+ if (_keywordParsers.TryGetValue(CurrentToken.Keyword.Value, out var handler))
{
handler(topLevel);
}
@@ -1526,12 +1526,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- private IEnumerable SkipToNextImportantToken()
+ private IEnumerable SkipToNextImportantToken()
{
while (!EndOfFile)
{
var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
- if (At(CSharpSymbolType.RazorCommentTransition))
+ if (At(CSharpTokenType.RazorCommentTransition))
{
Accept(whitespace);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
@@ -1542,13 +1542,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return whitespace;
}
}
- return Enumerable.Empty();
+ return Enumerable.Empty();
}
// Common code for Parsers, but FxCop REALLY doesn't like it in the base class.. moving it here for now.
protected override void OutputSpanBeforeRazorComment()
{
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Code);
}
@@ -1610,7 +1610,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!char.IsWhiteSpace(@char))
{
- var currentDirective = CurrentSymbol.Content;
+ var currentDirective = CurrentToken.Content;
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveMustAppearAtStartOfLine(
new SourceSpan(CurrentStart, currentDirective.Length), currentDirective));
@@ -1644,13 +1644,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
for (var i = 0; i < descriptor.Tokens.Count; i++)
{
- if (!At(CSharpSymbolType.WhiteSpace) &&
- !At(CSharpSymbolType.NewLine) &&
+ if (!At(CSharpTokenType.WhiteSpace) &&
+ !At(CSharpTokenType.NewLine) &&
!EndOfFile)
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveTokensMustBeSeparatedByWhitespace(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length), descriptor.Directive));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
return;
}
@@ -1664,10 +1664,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Code, AcceptedCharactersInternal.WhiteSpace);
- if (EndOfFile || At(CSharpSymbolType.NewLine))
+ if (EndOfFile || At(CSharpTokenType.NewLine))
{
- // Add a marker symbol to provide CSharp intellisense when we start typing the directive token.
- AddMarkerSymbolIfNecessary();
+ // Add a marker token to provide CSharp intellisense when we start typing the directive token.
+ AddMarkerTokenIfNecessary();
Span.ChunkGenerator = new DirectiveTokenChunkGenerator(tokenDescriptor);
Span.EditHandler = new DirectiveTokenEditHandler(Language.TokenizeString);
Output(SpanKindInternal.Code, AcceptedCharactersInternal.NonWhiteSpace);
@@ -1679,7 +1679,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup, AcceptedCharactersInternal.WhiteSpace);
}
- if (tokenDescriptor.Optional && (EndOfFile || At(CSharpSymbolType.NewLine)))
+ if (tokenDescriptor.Optional && (EndOfFile || At(CSharpTokenType.NewLine)))
{
break;
}
@@ -1700,7 +1700,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveExpectsTypeName(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length), descriptor.Directive));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
return;
}
@@ -1718,7 +1718,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break;
case DirectiveTokenKind.Member:
- if (At(CSharpSymbolType.Identifier))
+ if (At(CSharpTokenType.Identifier))
{
AcceptAndMoveNext();
}
@@ -1726,13 +1726,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveExpectsIdentifier(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length), descriptor.Directive));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
return;
}
break;
case DirectiveTokenKind.String:
- if (At(CSharpSymbolType.StringLiteral) && CurrentSymbol.Errors.Count == 0)
+ if (At(CSharpTokenType.StringLiteral) && CurrentToken.Errors.Count == 0)
{
AcceptAndMoveNext();
}
@@ -1740,7 +1740,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveExpectsQuotedStringLiteral(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length), descriptor.Directive));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
return;
}
break;
@@ -1759,13 +1759,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
case DirectiveKind.SingleLine:
Output(SpanKindInternal.None, AcceptedCharactersInternal.WhiteSpace);
- Optional(CSharpSymbolType.Semicolon);
+ Optional(CSharpTokenType.Semicolon);
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.WhiteSpace);
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
- if (At(CSharpSymbolType.NewLine))
+ if (At(CSharpTokenType.NewLine))
{
AcceptAndMoveNext();
}
@@ -1773,7 +1773,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length),
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length),
descriptor.Directive,
Resources.ErrorComponent_Newline));
}
@@ -1816,12 +1816,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
ParseDirectiveBlock(descriptor, parseChildren: (startingBraceLocation) =>
{
NextToken();
- Balance(BalancingModes.NoErrorOnFailure, CSharpSymbolType.LeftBrace, CSharpSymbolType.RightBrace, startingBraceLocation);
+ Balance(BalancingModes.NoErrorOnFailure, CSharpTokenType.LeftBrace, CSharpTokenType.RightBrace, startingBraceLocation);
Span.ChunkGenerator = new StatementChunkGenerator();
var existingEditHandler = Span.EditHandler;
Span.EditHandler = new CodeBlockEditHandler(Language.TokenizeString);
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Code);
@@ -1868,25 +1868,25 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnexpectedEOFAfterDirective(
new SourceSpan(CurrentStart, contentLength: 1 /* { */), descriptor.Directive, "{"));
}
- else if (!At(CSharpSymbolType.LeftBrace))
+ else if (!At(CSharpTokenType.LeftBrace))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length), descriptor.Directive, "{"));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive, "{"));
}
else
{
var editHandler = new AutoCompleteEditHandler(Language.TokenizeString, autoCompleteAtEndOfSpan: true);
Span.EditHandler = editHandler;
var startingBraceLocation = CurrentStart;
- Accept(CurrentSymbol);
+ Accept(CurrentToken);
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None);
parseChildren(startingBraceLocation);
Span.ChunkGenerator = SpanChunkGenerator.Null;
- if (!Optional(CSharpSymbolType.RightBrace))
+ if (!Optional(CSharpTokenType.RightBrace))
{
editHandler.AutoCompleteString = "}";
Context.ErrorSink.OnError(
@@ -1982,9 +1982,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Ex: @addTagHelper "*, Microsoft.AspNetCore.CoolLibrary"
// ^ ^
// Start End
- if (Span.Symbols.Count == 1 && (Span.Symbols[0] as CSharpSymbol)?.Type == CSharpSymbolType.StringLiteral)
+ if (Span.Tokens.Count == 1 && (Span.Tokens[0] as CSharpToken)?.Type == CSharpTokenType.StringLiteral)
{
- offset += Span.Symbols[0].Content.IndexOf(directiveText, StringComparison.Ordinal);
+ offset += Span.Tokens[0].Content.IndexOf(directiveText, StringComparison.Ordinal);
// This is safe because inside one of these directives all of the text needs to be on the
// same line.
@@ -2081,8 +2081,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Conditional("DEBUG")]
protected void AssertDirective(string directive)
{
- Debug.Assert(CurrentSymbol.Type == CSharpSymbolType.Identifier || CurrentSymbol.Type == CSharpSymbolType.Keyword);
- Debug.Assert(string.Equals(CurrentSymbol.Content, directive, StringComparison.Ordinal));
+ Debug.Assert(CurrentToken.Type == CSharpTokenType.Identifier || CurrentToken.Type == CSharpTokenType.Keyword);
+ Debug.Assert(string.Equals(CurrentToken.Content, directive, StringComparison.Ordinal));
}
private void TagHelperDirective(string keyword, Func, ISpanChunkGenerator> chunkGeneratorFactory)
@@ -2108,18 +2108,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
- var foundWhitespace = At(CSharpSymbolType.WhiteSpace);
+ var foundWhitespace = At(CSharpTokenType.WhiteSpace);
// If we found whitespace then any content placed within the whitespace MAY cause a destructive change
// to the document. We can't accept it.
var acceptedCharacters = foundWhitespace ? AcceptedCharactersInternal.None : AcceptedCharactersInternal.AnyExceptNewline;
Output(SpanKindInternal.MetaCode, acceptedCharacters);
- AcceptWhile(CSharpSymbolType.WhiteSpace);
+ AcceptWhile(CSharpTokenType.WhiteSpace);
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Markup, acceptedCharacters);
- if (EndOfFile || At(CSharpSymbolType.NewLine))
+ if (EndOfFile || At(CSharpTokenType.NewLine))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_DirectiveMustHaveValue(
@@ -2134,10 +2134,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code
// etc.
- AcceptUntil(CSharpSymbolType.NewLine);
+ AcceptUntil(CSharpTokenType.NewLine);
// Pull out the value and remove whitespaces and optional quotes
- var rawValue = string.Concat(Span.Symbols.Select(s => s.Content)).Trim();
+ var rawValue = string.Concat(Span.Tokens.Select(s => s.Content)).Trim();
var startsWithQuote = rawValue.StartsWith("\"", StringComparison.Ordinal);
var endsWithQuote = rawValue.EndsWith("\"", StringComparison.Ordinal);
@@ -2170,21 +2170,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Start = start;
}
- public Block(CSharpSymbol symbol, SourceLocation start)
- : this(GetName(symbol), start)
+ public Block(CSharpToken token, SourceLocation start)
+ : this(GetName(token), start)
{
}
public string Name { get; set; }
public SourceLocation Start { get; set; }
- private static string GetName(CSharpSymbol sym)
+ private static string GetName(CSharpToken token)
{
- if (sym.Type == CSharpSymbolType.Keyword)
+ if (token.Type == CSharpTokenType.Keyword)
{
- return CSharpLanguageCharacteristics.GetKeyword(sym.Keyword.Value);
+ return CSharpLanguageCharacteristics.GetKeyword(token.Keyword.Value);
}
- return sym.Content;
+ return token.Content;
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs
index 44ce3a89ce..70d4bfeda1 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpLanguageCharacteristics.cs
@@ -6,61 +6,61 @@ using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class CSharpLanguageCharacteristics : LanguageCharacteristics
+ internal class CSharpLanguageCharacteristics : LanguageCharacteristics
{
private static readonly CSharpLanguageCharacteristics _instance = new CSharpLanguageCharacteristics();
- private static Dictionary _symbolSamples = new Dictionary()
+ private static Dictionary _tokenSamples = new Dictionary()
{
- { CSharpSymbolType.Arrow, "->" },
- { CSharpSymbolType.Minus, "-" },
- { CSharpSymbolType.Decrement, "--" },
- { CSharpSymbolType.MinusAssign, "-=" },
- { CSharpSymbolType.NotEqual, "!=" },
- { CSharpSymbolType.Not, "!" },
- { CSharpSymbolType.Modulo, "%" },
- { CSharpSymbolType.ModuloAssign, "%=" },
- { CSharpSymbolType.AndAssign, "&=" },
- { CSharpSymbolType.And, "&" },
- { CSharpSymbolType.DoubleAnd, "&&" },
- { CSharpSymbolType.LeftParenthesis, "(" },
- { CSharpSymbolType.RightParenthesis, ")" },
- { CSharpSymbolType.Star, "*" },
- { CSharpSymbolType.MultiplyAssign, "*=" },
- { CSharpSymbolType.Comma, "," },
- { CSharpSymbolType.Dot, "." },
- { CSharpSymbolType.Slash, "/" },
- { CSharpSymbolType.DivideAssign, "/=" },
- { CSharpSymbolType.DoubleColon, "::" },
- { CSharpSymbolType.Colon, ":" },
- { CSharpSymbolType.Semicolon, ";" },
- { CSharpSymbolType.QuestionMark, "?" },
- { CSharpSymbolType.NullCoalesce, "??" },
- { CSharpSymbolType.RightBracket, "]" },
- { CSharpSymbolType.LeftBracket, "[" },
- { CSharpSymbolType.XorAssign, "^=" },
- { CSharpSymbolType.Xor, "^" },
- { CSharpSymbolType.LeftBrace, "{" },
- { CSharpSymbolType.OrAssign, "|=" },
- { CSharpSymbolType.DoubleOr, "||" },
- { CSharpSymbolType.Or, "|" },
- { CSharpSymbolType.RightBrace, "}" },
- { CSharpSymbolType.Tilde, "~" },
- { CSharpSymbolType.Plus, "+" },
- { CSharpSymbolType.PlusAssign, "+=" },
- { CSharpSymbolType.Increment, "++" },
- { CSharpSymbolType.LessThan, "<" },
- { CSharpSymbolType.LessThanEqual, "<=" },
- { CSharpSymbolType.LeftShift, "<<" },
- { CSharpSymbolType.LeftShiftAssign, "<<=" },
- { CSharpSymbolType.Assign, "=" },
- { CSharpSymbolType.Equals, "==" },
- { CSharpSymbolType.GreaterThan, ">" },
- { CSharpSymbolType.GreaterThanEqual, ">=" },
- { CSharpSymbolType.RightShift, ">>" },
- { CSharpSymbolType.RightShiftAssign, ">>=" },
- { CSharpSymbolType.Hash, "#" },
- { CSharpSymbolType.Transition, "@" },
+ { CSharpTokenType.Arrow, "->" },
+ { CSharpTokenType.Minus, "-" },
+ { CSharpTokenType.Decrement, "--" },
+ { CSharpTokenType.MinusAssign, "-=" },
+ { CSharpTokenType.NotEqual, "!=" },
+ { CSharpTokenType.Not, "!" },
+ { CSharpTokenType.Modulo, "%" },
+ { CSharpTokenType.ModuloAssign, "%=" },
+ { CSharpTokenType.AndAssign, "&=" },
+ { CSharpTokenType.And, "&" },
+ { CSharpTokenType.DoubleAnd, "&&" },
+ { CSharpTokenType.LeftParenthesis, "(" },
+ { CSharpTokenType.RightParenthesis, ")" },
+ { CSharpTokenType.Star, "*" },
+ { CSharpTokenType.MultiplyAssign, "*=" },
+ { CSharpTokenType.Comma, "," },
+ { CSharpTokenType.Dot, "." },
+ { CSharpTokenType.Slash, "/" },
+ { CSharpTokenType.DivideAssign, "/=" },
+ { CSharpTokenType.DoubleColon, "::" },
+ { CSharpTokenType.Colon, ":" },
+ { CSharpTokenType.Semicolon, ";" },
+ { CSharpTokenType.QuestionMark, "?" },
+ { CSharpTokenType.NullCoalesce, "??" },
+ { CSharpTokenType.RightBracket, "]" },
+ { CSharpTokenType.LeftBracket, "[" },
+ { CSharpTokenType.XorAssign, "^=" },
+ { CSharpTokenType.Xor, "^" },
+ { CSharpTokenType.LeftBrace, "{" },
+ { CSharpTokenType.OrAssign, "|=" },
+ { CSharpTokenType.DoubleOr, "||" },
+ { CSharpTokenType.Or, "|" },
+ { CSharpTokenType.RightBrace, "}" },
+ { CSharpTokenType.Tilde, "~" },
+ { CSharpTokenType.Plus, "+" },
+ { CSharpTokenType.PlusAssign, "+=" },
+ { CSharpTokenType.Increment, "++" },
+ { CSharpTokenType.LessThan, "<" },
+ { CSharpTokenType.LessThanEqual, "<=" },
+ { CSharpTokenType.LeftShift, "<<" },
+ { CSharpTokenType.LeftShiftAssign, "<<=" },
+ { CSharpTokenType.Assign, "=" },
+ { CSharpTokenType.Equals, "==" },
+ { CSharpTokenType.GreaterThan, ">" },
+ { CSharpTokenType.GreaterThanEqual, ">=" },
+ { CSharpTokenType.RightShift, ">>" },
+ { CSharpTokenType.RightShiftAssign, ">>=" },
+ { CSharpTokenType.Hash, "#" },
+ { CSharpTokenType.Transition, "@" },
};
protected CSharpLanguageCharacteristics()
@@ -74,96 +74,96 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source);
}
- protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList errors)
+ protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList errors)
{
- return new CSharpSymbol(content, type, errors);
+ return new CSharpToken(content, type, errors);
}
- public override string GetSample(CSharpSymbolType type)
+ public override string GetSample(CSharpTokenType type)
{
string sample;
- if (!_symbolSamples.TryGetValue(type, out sample))
+ if (!_tokenSamples.TryGetValue(type, out sample))
{
switch (type)
{
- case CSharpSymbolType.Identifier:
- return Resources.CSharpSymbol_Identifier;
- case CSharpSymbolType.Keyword:
- return Resources.CSharpSymbol_Keyword;
- case CSharpSymbolType.IntegerLiteral:
- return Resources.CSharpSymbol_IntegerLiteral;
- case CSharpSymbolType.NewLine:
- return Resources.CSharpSymbol_Newline;
- case CSharpSymbolType.WhiteSpace:
- return Resources.CSharpSymbol_Whitespace;
- case CSharpSymbolType.Comment:
- return Resources.CSharpSymbol_Comment;
- case CSharpSymbolType.RealLiteral:
- return Resources.CSharpSymbol_RealLiteral;
- case CSharpSymbolType.CharacterLiteral:
- return Resources.CSharpSymbol_CharacterLiteral;
- case CSharpSymbolType.StringLiteral:
- return Resources.CSharpSymbol_StringLiteral;
+ case CSharpTokenType.Identifier:
+ return Resources.CSharpToken_Identifier;
+ case CSharpTokenType.Keyword:
+ return Resources.CSharpToken_Keyword;
+ case CSharpTokenType.IntegerLiteral:
+ return Resources.CSharpToken_IntegerLiteral;
+ case CSharpTokenType.NewLine:
+ return Resources.CSharpToken_Newline;
+ case CSharpTokenType.WhiteSpace:
+ return Resources.CSharpToken_Whitespace;
+ case CSharpTokenType.Comment:
+ return Resources.CSharpToken_Comment;
+ case CSharpTokenType.RealLiteral:
+ return Resources.CSharpToken_RealLiteral;
+ case CSharpTokenType.CharacterLiteral:
+ return Resources.CSharpToken_CharacterLiteral;
+ case CSharpTokenType.StringLiteral:
+ return Resources.CSharpToken_StringLiteral;
default:
- return Resources.Symbol_Unknown;
+ return Resources.Token_Unknown;
}
}
return sample;
}
- public override CSharpSymbol CreateMarkerSymbol()
+ public override CSharpToken CreateMarkerToken()
{
- return new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown);
+ return new CSharpToken(string.Empty, CSharpTokenType.Unknown);
}
- public override CSharpSymbolType GetKnownSymbolType(KnownSymbolType type)
+ public override CSharpTokenType GetKnownTokenType(KnownTokenType type)
{
switch (type)
{
- case KnownSymbolType.Identifier:
- return CSharpSymbolType.Identifier;
- case KnownSymbolType.Keyword:
- return CSharpSymbolType.Keyword;
- case KnownSymbolType.NewLine:
- return CSharpSymbolType.NewLine;
- case KnownSymbolType.WhiteSpace:
- return CSharpSymbolType.WhiteSpace;
- case KnownSymbolType.Transition:
- return CSharpSymbolType.Transition;
- case KnownSymbolType.CommentStart:
- return CSharpSymbolType.RazorCommentTransition;
- case KnownSymbolType.CommentStar:
- return CSharpSymbolType.RazorCommentStar;
- case KnownSymbolType.CommentBody:
- return CSharpSymbolType.RazorComment;
+ case KnownTokenType.Identifier:
+ return CSharpTokenType.Identifier;
+ case KnownTokenType.Keyword:
+ return CSharpTokenType.Keyword;
+ case KnownTokenType.NewLine:
+ return CSharpTokenType.NewLine;
+ case KnownTokenType.WhiteSpace:
+ return CSharpTokenType.WhiteSpace;
+ case KnownTokenType.Transition:
+ return CSharpTokenType.Transition;
+ case KnownTokenType.CommentStart:
+ return CSharpTokenType.RazorCommentTransition;
+ case KnownTokenType.CommentStar:
+ return CSharpTokenType.RazorCommentStar;
+ case KnownTokenType.CommentBody:
+ return CSharpTokenType.RazorComment;
default:
- return CSharpSymbolType.Unknown;
+ return CSharpTokenType.Unknown;
}
}
- public override CSharpSymbolType FlipBracket(CSharpSymbolType bracket)
+ public override CSharpTokenType FlipBracket(CSharpTokenType bracket)
{
switch (bracket)
{
- case CSharpSymbolType.LeftBrace:
- return CSharpSymbolType.RightBrace;
- case CSharpSymbolType.LeftBracket:
- return CSharpSymbolType.RightBracket;
- case CSharpSymbolType.LeftParenthesis:
- return CSharpSymbolType.RightParenthesis;
- case CSharpSymbolType.LessThan:
- return CSharpSymbolType.GreaterThan;
- case CSharpSymbolType.RightBrace:
- return CSharpSymbolType.LeftBrace;
- case CSharpSymbolType.RightBracket:
- return CSharpSymbolType.LeftBracket;
- case CSharpSymbolType.RightParenthesis:
- return CSharpSymbolType.LeftParenthesis;
- case CSharpSymbolType.GreaterThan:
- return CSharpSymbolType.LessThan;
+ case CSharpTokenType.LeftBrace:
+ return CSharpTokenType.RightBrace;
+ case CSharpTokenType.LeftBracket:
+ return CSharpTokenType.RightBracket;
+ case CSharpTokenType.LeftParenthesis:
+ return CSharpTokenType.RightParenthesis;
+ case CSharpTokenType.LessThan:
+ return CSharpTokenType.GreaterThan;
+ case CSharpTokenType.RightBrace:
+ return CSharpTokenType.LeftBrace;
+ case CSharpTokenType.RightBracket:
+ return CSharpTokenType.LeftBracket;
+ case CSharpTokenType.RightParenthesis:
+ return CSharpTokenType.LeftParenthesis;
+ case CSharpTokenType.GreaterThan:
+ return CSharpTokenType.LessThan;
default:
Debug.Fail("FlipBracket must be called with a bracket character");
- return CSharpSymbolType.Unknown;
+ return CSharpTokenType.Unknown;
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbol.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpToken.cs
similarity index 83%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbol.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpToken.cs
index 21cd94915d..c928f0880e 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbol.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpToken.cs
@@ -6,11 +6,11 @@ using System.Collections.Generic;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class CSharpSymbol : SymbolBase
+ internal class CSharpToken : TokenBase
{
- public CSharpSymbol(
+ public CSharpToken(
string content,
- CSharpSymbolType type)
+ CSharpTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray)
{
if (content == null)
@@ -19,9 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- public CSharpSymbol(
+ public CSharpToken(
string content,
- CSharpSymbolType type,
+ CSharpTokenType type,
IReadOnlyList errors)
: base(content, type, errors)
{
@@ -35,7 +35,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj)
{
- var other = obj as CSharpSymbol;
+ var other = obj as CSharpToken;
return base.Equals(other) &&
other.Keyword == Keyword;
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbolType.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenType.cs
similarity index 97%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbolType.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenType.cs
index 059d7c48a6..10f3478d6d 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpSymbolType.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenType.cs
@@ -3,7 +3,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal enum CSharpSymbolType
+ internal enum CSharpTokenType
{
Unknown,
Identifier,
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs
index a1943070f3..f819e98a3a 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpTokenizer.cs
@@ -8,9 +8,9 @@ using System.Globalization;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class CSharpTokenizer : Tokenizer
+ internal class CSharpTokenizer : Tokenizer
{
- private Dictionary> _operatorHandlers;
+ private Dictionary> _operatorHandlers;
private static readonly Dictionary _keywords = new Dictionary(StringComparer.Ordinal)
{
@@ -100,31 +100,31 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
base.CurrentState = StartState;
- _operatorHandlers = new Dictionary>()
+ _operatorHandlers = new Dictionary>()
{
{ '-', MinusOperator },
{ '<', LessThanOperator },
{ '>', GreaterThanOperator },
- { '&', CreateTwoCharOperatorHandler(CSharpSymbolType.And, '=', CSharpSymbolType.AndAssign, '&', CSharpSymbolType.DoubleAnd) },
- { '|', CreateTwoCharOperatorHandler(CSharpSymbolType.Or, '=', CSharpSymbolType.OrAssign, '|', CSharpSymbolType.DoubleOr) },
- { '+', CreateTwoCharOperatorHandler(CSharpSymbolType.Plus, '=', CSharpSymbolType.PlusAssign, '+', CSharpSymbolType.Increment) },
- { '=', CreateTwoCharOperatorHandler(CSharpSymbolType.Assign, '=', CSharpSymbolType.Equals, '>', CSharpSymbolType.GreaterThanEqual) },
- { '!', CreateTwoCharOperatorHandler(CSharpSymbolType.Not, '=', CSharpSymbolType.NotEqual) },
- { '%', CreateTwoCharOperatorHandler(CSharpSymbolType.Modulo, '=', CSharpSymbolType.ModuloAssign) },
- { '*', CreateTwoCharOperatorHandler(CSharpSymbolType.Star, '=', CSharpSymbolType.MultiplyAssign) },
- { ':', CreateTwoCharOperatorHandler(CSharpSymbolType.Colon, ':', CSharpSymbolType.DoubleColon) },
- { '?', CreateTwoCharOperatorHandler(CSharpSymbolType.QuestionMark, '?', CSharpSymbolType.NullCoalesce) },
- { '^', CreateTwoCharOperatorHandler(CSharpSymbolType.Xor, '=', CSharpSymbolType.XorAssign) },
- { '(', () => CSharpSymbolType.LeftParenthesis },
- { ')', () => CSharpSymbolType.RightParenthesis },
- { '{', () => CSharpSymbolType.LeftBrace },
- { '}', () => CSharpSymbolType.RightBrace },
- { '[', () => CSharpSymbolType.LeftBracket },
- { ']', () => CSharpSymbolType.RightBracket },
- { ',', () => CSharpSymbolType.Comma },
- { ';', () => CSharpSymbolType.Semicolon },
- { '~', () => CSharpSymbolType.Tilde },
- { '#', () => CSharpSymbolType.Hash }
+ { '&', CreateTwoCharOperatorHandler(CSharpTokenType.And, '=', CSharpTokenType.AndAssign, '&', CSharpTokenType.DoubleAnd) },
+ { '|', CreateTwoCharOperatorHandler(CSharpTokenType.Or, '=', CSharpTokenType.OrAssign, '|', CSharpTokenType.DoubleOr) },
+ { '+', CreateTwoCharOperatorHandler(CSharpTokenType.Plus, '=', CSharpTokenType.PlusAssign, '+', CSharpTokenType.Increment) },
+ { '=', CreateTwoCharOperatorHandler(CSharpTokenType.Assign, '=', CSharpTokenType.Equals, '>', CSharpTokenType.GreaterThanEqual) },
+ { '!', CreateTwoCharOperatorHandler(CSharpTokenType.Not, '=', CSharpTokenType.NotEqual) },
+ { '%', CreateTwoCharOperatorHandler(CSharpTokenType.Modulo, '=', CSharpTokenType.ModuloAssign) },
+ { '*', CreateTwoCharOperatorHandler(CSharpTokenType.Star, '=', CSharpTokenType.MultiplyAssign) },
+ { ':', CreateTwoCharOperatorHandler(CSharpTokenType.Colon, ':', CSharpTokenType.DoubleColon) },
+ { '?', CreateTwoCharOperatorHandler(CSharpTokenType.QuestionMark, '?', CSharpTokenType.NullCoalesce) },
+ { '^', CreateTwoCharOperatorHandler(CSharpTokenType.Xor, '=', CSharpTokenType.XorAssign) },
+ { '(', () => CSharpTokenType.LeftParenthesis },
+ { ')', () => CSharpTokenType.RightParenthesis },
+ { '{', () => CSharpTokenType.LeftBrace },
+ { '}', () => CSharpTokenType.RightBrace },
+ { '[', () => CSharpTokenType.LeftBracket },
+ { ']', () => CSharpTokenType.RightBracket },
+ { ',', () => CSharpTokenType.Comma },
+ { ';', () => CSharpTokenType.Semicolon },
+ { '~', () => CSharpTokenType.Tilde },
+ { '#', () => CSharpTokenType.Hash }
};
}
@@ -132,11 +132,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState;
- public override CSharpSymbolType RazorCommentType => CSharpSymbolType.RazorComment;
+ public override CSharpTokenType RazorCommentType => CSharpTokenType.RazorComment;
- public override CSharpSymbolType RazorCommentTransitionType => CSharpSymbolType.RazorCommentTransition;
+ public override CSharpTokenType RazorCommentTransitionType => CSharpTokenType.RazorCommentTransition;
- public override CSharpSymbolType RazorCommentStarType => CSharpSymbolType.RazorCommentStar;
+ public override CSharpTokenType RazorCommentStarType => CSharpTokenType.RazorCommentStar;
protected override StateResult Dispatch()
{
@@ -160,8 +160,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return RazorCommentBody();
case CSharpTokenizerState.StarAfterRazorCommentBody:
return StarAfterRazorCommentBody();
- case CSharpTokenizerState.AtSymbolAfterRazorCommentBody:
- return AtSymbolAfterRazorCommentBody();
+ case CSharpTokenizerState.AtTokenAfterRazorCommentBody:
+ return AtTokenAfterRazorCommentBody();
default:
Debug.Fail("Invalid TokenizerState");
return default(StateResult);
@@ -169,15 +169,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Optimize memory allocation by returning constants for the most frequent cases
- protected override string GetSymbolContent(CSharpSymbolType type)
+ protected override string GetTokenContent(CSharpTokenType type)
{
- var symbolLength = Buffer.Length;
+ var tokenLength = Buffer.Length;
- if (symbolLength == 1)
+ if (tokenLength == 1)
{
switch (type)
{
- case CSharpSymbolType.IntegerLiteral:
+ case CSharpTokenType.IntegerLiteral:
switch (Buffer[0])
{
case '0':
@@ -202,13 +202,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "9";
}
break;
- case CSharpSymbolType.NewLine:
+ case CSharpTokenType.NewLine:
if (Buffer[0] == '\n')
{
return "\n";
}
break;
- case CSharpSymbolType.WhiteSpace:
+ case CSharpTokenType.WhiteSpace:
if (Buffer[0] == ' ')
{
return " ";
@@ -218,134 +218,134 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t";
}
break;
- case CSharpSymbolType.Minus:
+ case CSharpTokenType.Minus:
return "-";
- case CSharpSymbolType.Not:
+ case CSharpTokenType.Not:
return "!";
- case CSharpSymbolType.Modulo:
+ case CSharpTokenType.Modulo:
return "%";
- case CSharpSymbolType.And:
+ case CSharpTokenType.And:
return "&";
- case CSharpSymbolType.LeftParenthesis:
+ case CSharpTokenType.LeftParenthesis:
return "(";
- case CSharpSymbolType.RightParenthesis:
+ case CSharpTokenType.RightParenthesis:
return ")";
- case CSharpSymbolType.Star:
+ case CSharpTokenType.Star:
return "*";
- case CSharpSymbolType.Comma:
+ case CSharpTokenType.Comma:
return ",";
- case CSharpSymbolType.Dot:
+ case CSharpTokenType.Dot:
return ".";
- case CSharpSymbolType.Slash:
+ case CSharpTokenType.Slash:
return "/";
- case CSharpSymbolType.Colon:
+ case CSharpTokenType.Colon:
return ":";
- case CSharpSymbolType.Semicolon:
+ case CSharpTokenType.Semicolon:
return ";";
- case CSharpSymbolType.QuestionMark:
+ case CSharpTokenType.QuestionMark:
return "?";
- case CSharpSymbolType.RightBracket:
+ case CSharpTokenType.RightBracket:
return "]";
- case CSharpSymbolType.LeftBracket:
+ case CSharpTokenType.LeftBracket:
return "[";
- case CSharpSymbolType.Xor:
+ case CSharpTokenType.Xor:
return "^";
- case CSharpSymbolType.LeftBrace:
+ case CSharpTokenType.LeftBrace:
return "{";
- case CSharpSymbolType.Or:
+ case CSharpTokenType.Or:
return "|";
- case CSharpSymbolType.RightBrace:
+ case CSharpTokenType.RightBrace:
return "}";
- case CSharpSymbolType.Tilde:
+ case CSharpTokenType.Tilde:
return "~";
- case CSharpSymbolType.Plus:
+ case CSharpTokenType.Plus:
return "+";
- case CSharpSymbolType.LessThan:
+ case CSharpTokenType.LessThan:
return "<";
- case CSharpSymbolType.Assign:
+ case CSharpTokenType.Assign:
return "=";
- case CSharpSymbolType.GreaterThan:
+ case CSharpTokenType.GreaterThan:
return ">";
- case CSharpSymbolType.Hash:
+ case CSharpTokenType.Hash:
return "#";
- case CSharpSymbolType.Transition:
+ case CSharpTokenType.Transition:
return "@";
}
}
- else if (symbolLength == 2)
+ else if (tokenLength == 2)
{
switch (type)
{
- case CSharpSymbolType.NewLine:
+ case CSharpTokenType.NewLine:
return "\r\n";
- case CSharpSymbolType.Arrow:
+ case CSharpTokenType.Arrow:
return "->";
- case CSharpSymbolType.Decrement:
+ case CSharpTokenType.Decrement:
return "--";
- case CSharpSymbolType.MinusAssign:
+ case CSharpTokenType.MinusAssign:
return "-=";
- case CSharpSymbolType.NotEqual:
+ case CSharpTokenType.NotEqual:
return "!=";
- case CSharpSymbolType.ModuloAssign:
+ case CSharpTokenType.ModuloAssign:
return "%=";
- case CSharpSymbolType.AndAssign:
+ case CSharpTokenType.AndAssign:
return "&=";
- case CSharpSymbolType.DoubleAnd:
+ case CSharpTokenType.DoubleAnd:
return "&&";
- case CSharpSymbolType.MultiplyAssign:
+ case CSharpTokenType.MultiplyAssign:
return "*=";
- case CSharpSymbolType.DivideAssign:
+ case CSharpTokenType.DivideAssign:
return "/=";
- case CSharpSymbolType.DoubleColon:
+ case CSharpTokenType.DoubleColon:
return "::";
- case CSharpSymbolType.NullCoalesce:
+ case CSharpTokenType.NullCoalesce:
return "??";
- case CSharpSymbolType.XorAssign:
+ case CSharpTokenType.XorAssign:
return "^=";
- case CSharpSymbolType.OrAssign:
+ case CSharpTokenType.OrAssign:
return "|=";
- case CSharpSymbolType.DoubleOr:
+ case CSharpTokenType.DoubleOr:
return "||";
- case CSharpSymbolType.PlusAssign:
+ case CSharpTokenType.PlusAssign:
return "+=";
- case CSharpSymbolType.Increment:
+ case CSharpTokenType.Increment:
return "++";
- case CSharpSymbolType.LessThanEqual:
+ case CSharpTokenType.LessThanEqual:
return "<=";
- case CSharpSymbolType.LeftShift:
+ case CSharpTokenType.LeftShift:
return "<<";
- case CSharpSymbolType.Equals:
+ case CSharpTokenType.Equals:
return "==";
- case CSharpSymbolType.GreaterThanEqual:
+ case CSharpTokenType.GreaterThanEqual:
if (Buffer[0] == '=')
{
return "=>";
}
return ">=";
- case CSharpSymbolType.RightShift:
+ case CSharpTokenType.RightShift:
return ">>";
}
}
- else if (symbolLength == 3)
+ else if (tokenLength == 3)
{
switch (type)
{
- case CSharpSymbolType.LeftShiftAssign:
+ case CSharpTokenType.LeftShiftAssign:
return "<<=";
- case CSharpSymbolType.RightShiftAssign:
+ case CSharpTokenType.RightShiftAssign:
return ">>=";
}
}
- return base.GetSymbolContent(type);
+ return base.GetTokenContent(type);
}
- protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList errors)
+ protected override CSharpToken CreateToken(string content, CSharpTokenType type, IReadOnlyList errors)
{
- return new CSharpSymbol(content, type, errors);
+ return new CSharpToken(content, type, errors);
}
private StateResult Data()
@@ -359,13 +359,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent();
}
- return Stay(EndSymbol(CSharpSymbolType.NewLine));
+ return Stay(EndToken(CSharpTokenType.NewLine));
}
else if (ParserHelpers.IsWhitespace(CurrentCharacter))
{
// CSharp Spec §2.3.3
TakeUntil(c => !ParserHelpers.IsWhitespace(c));
- return Stay(EndSymbol(CSharpSymbolType.WhiteSpace));
+ return Stay(EndToken(CSharpTokenType.WhiteSpace));
}
else if (IsIdentifierStart(CurrentCharacter))
{
@@ -378,7 +378,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
switch (CurrentCharacter)
{
case '@':
- return AtSymbol();
+ return AtToken();
case '\'':
TakeCurrent();
return Transition(CSharpTokenizerState.QuotedCharacterLiteral);
@@ -390,7 +390,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return RealLiteral();
}
- return Stay(Single(CSharpSymbolType.Dot));
+ return Stay(Single(CSharpTokenType.Dot));
case '/':
TakeCurrent();
if (CurrentCharacter == '/')
@@ -406,18 +406,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else if (CurrentCharacter == '=')
{
TakeCurrent();
- return Stay(EndSymbol(CSharpSymbolType.DivideAssign));
+ return Stay(EndToken(CSharpTokenType.DivideAssign));
}
else
{
- return Stay(EndSymbol(CSharpSymbolType.Slash));
+ return Stay(EndToken(CSharpTokenType.Slash));
}
default:
- return Stay(EndSymbol(Operator()));
+ return Stay(EndToken(Operator()));
}
}
- private StateResult AtSymbol()
+ private StateResult AtToken()
{
TakeCurrent();
if (CurrentCharacter == '"')
@@ -429,78 +429,78 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
CSharpTokenizerState.AfterRazorCommentTransition,
- EndSymbol(CSharpSymbolType.RazorCommentTransition));
+ EndToken(CSharpTokenType.RazorCommentTransition));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return Transition(
CSharpTokenizerState.EscapedRazorCommentTransition,
- EndSymbol(CSharpSymbolType.Transition));
+ EndToken(CSharpTokenType.Transition));
}
- return Stay(EndSymbol(CSharpSymbolType.Transition));
+ return Stay(EndToken(CSharpTokenType.Transition));
}
private StateResult EscapedRazorCommentTransition()
{
TakeCurrent();
- return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Transition));
+ return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Transition));
}
- private CSharpSymbolType Operator()
+ private CSharpTokenType Operator()
{
var first = CurrentCharacter;
TakeCurrent();
- Func handler;
+ Func handler;
if (_operatorHandlers.TryGetValue(first, out handler))
{
return handler();
}
- return CSharpSymbolType.Unknown;
+ return CSharpTokenType.Unknown;
}
- private CSharpSymbolType LessThanOperator()
+ private CSharpTokenType LessThanOperator()
{
if (CurrentCharacter == '=')
{
TakeCurrent();
- return CSharpSymbolType.LessThanEqual;
+ return CSharpTokenType.LessThanEqual;
}
- return CSharpSymbolType.LessThan;
+ return CSharpTokenType.LessThan;
}
- private CSharpSymbolType GreaterThanOperator()
+ private CSharpTokenType GreaterThanOperator()
{
if (CurrentCharacter == '=')
{
TakeCurrent();
- return CSharpSymbolType.GreaterThanEqual;
+ return CSharpTokenType.GreaterThanEqual;
}
- return CSharpSymbolType.GreaterThan;
+ return CSharpTokenType.GreaterThan;
}
- private CSharpSymbolType MinusOperator()
+ private CSharpTokenType MinusOperator()
{
if (CurrentCharacter == '>')
{
TakeCurrent();
- return CSharpSymbolType.Arrow;
+ return CSharpTokenType.Arrow;
}
else if (CurrentCharacter == '-')
{
TakeCurrent();
- return CSharpSymbolType.Decrement;
+ return CSharpTokenType.Decrement;
}
else if (CurrentCharacter == '=')
{
TakeCurrent();
- return CSharpSymbolType.MinusAssign;
+ return CSharpTokenType.MinusAssign;
}
- return CSharpSymbolType.Minus;
+ return CSharpTokenType.Minus;
}
- private Func CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char second, CSharpSymbolType typeIfBoth)
+ private Func CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char second, CSharpTokenType typeIfBoth)
{
return () =>
{
@@ -513,7 +513,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
};
}
- private Func CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char option1, CSharpSymbolType typeIfOption1, char option2, CSharpSymbolType typeIfOption2)
+ private Func CreateTwoCharOperatorHandler(CSharpTokenType typeIfOnlyFirst, char option1, CSharpTokenType typeIfOption1, char option2, CSharpTokenType typeIfOption2)
{
return () =>
{
@@ -550,14 +550,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
}
- return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.StringLiteral));
+ return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.StringLiteral));
}
- private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', CSharpSymbolType.CharacterLiteral);
+ private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', CSharpTokenType.CharacterLiteral);
- private StateResult QuotedStringLiteral() => QuotedLiteral('\"', CSharpSymbolType.StringLiteral);
+ private StateResult QuotedStringLiteral() => QuotedLiteral('\"', CSharpTokenType.StringLiteral);
- private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType)
+ private StateResult QuotedLiteral(char quote, CSharpTokenType literalType)
{
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\')
@@ -581,7 +581,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent(); // No-op if at EOF
}
- return Transition(CSharpTokenizerState.Data, EndSymbol(literalType));
+ return Transition(CSharpTokenizerState.Data, EndToken(literalType));
}
// CSharp Spec §2.3.2
@@ -594,7 +594,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
- return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Comment));
+ return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment));
}
if (CurrentCharacter == '*')
{
@@ -602,7 +602,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (CurrentCharacter == '/')
{
TakeCurrent();
- return Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Comment));
+ return Transition(CSharpTokenizerState.Data, EndToken(CSharpTokenType.Comment));
}
}
return Stay();
@@ -612,7 +612,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult SingleLineComment()
{
TakeUntil(c => ParserHelpers.IsNewLine(c));
- return Stay(EndSymbol(CSharpSymbolType.Comment));
+ return Stay(EndToken(CSharpTokenType.Comment));
}
// CSharp Spec §2.4.4
@@ -632,7 +632,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeUntil(c => !IsHexDigit(c));
TakeIntegerSuffix();
- return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral));
+ return Stay(EndToken(CSharpTokenType.IntegerLiteral));
}
private StateResult DecimalLiteral()
@@ -650,7 +650,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
TakeIntegerSuffix();
- return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral));
+ return Stay(EndToken(CSharpTokenType.IntegerLiteral));
}
}
@@ -669,7 +669,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent();
}
- return Stay(EndSymbol(CSharpSymbolType.RealLiteral));
+ return Stay(EndToken(CSharpTokenType.RealLiteral));
}
// CSharp Spec §2.4.4.3
@@ -708,27 +708,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Debug.Assert(IsIdentifierStart(CurrentCharacter));
TakeCurrent();
TakeUntil(c => !IsIdentifierPart(c));
- CSharpSymbol symbol = null;
+ CSharpToken token = null;
if (HaveContent)
{
CSharpKeyword keyword;
- var type = CSharpSymbolType.Identifier;
- var symbolContent = Buffer.ToString();
- if (_keywords.TryGetValue(symbolContent, out keyword))
+ var type = CSharpTokenType.Identifier;
+ var tokenContent = Buffer.ToString();
+ if (_keywords.TryGetValue(tokenContent, out keyword))
{
- type = CSharpSymbolType.Keyword;
+ type = CSharpTokenType.Keyword;
}
- symbol = new CSharpSymbol(symbolContent, type)
+ token = new CSharpToken(tokenContent, type)
{
- Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null,
+ Keyword = type == CSharpTokenType.Keyword ? (CSharpKeyword?)keyword : null,
};
Buffer.Clear();
CurrentErrors.Clear();
}
- return Stay(symbol);
+ return Stay(token);
}
private StateResult Transition(CSharpTokenizerState state)
@@ -736,7 +736,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null);
}
- private StateResult Transition(CSharpTokenizerState state, CSharpSymbol result)
+ private StateResult Transition(CSharpTokenizerState state, CSharpToken result)
{
return Transition((int)state, result);
}
@@ -793,7 +793,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition,
RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody,
StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody,
- AtSymbolAfterRazorCommentBody = RazorCommentTokenizerState.AtSymbolAfterRazorCommentBody,
+ AtTokenAfterRazorCommentBody = RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs
index 0430388d81..d5ce71c43e 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CodeBlockEditHandler.cs
@@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal class CodeBlockEditHandler : SpanEditHandler
{
- public CodeBlockEditHandler(Func> tokenizer) : base(tokenizer)
+ public CodeBlockEditHandler(Func> tokenizer) : base(tokenizer)
{
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs
index d9507e8522..a9befaa302 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveCSharpTokenizer.cs
@@ -21,7 +21,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
_visitedFirstTokenStart = true;
}
- else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpSymbolType.NewLine)
+ else if (result.Result != null && _visitedFirstTokenStart && result.Result.Type == CSharpTokenType.NewLine)
{
_visitedFirstTokenLineEnd = true;
}
@@ -29,34 +29,34 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result;
}
- public override CSharpSymbol NextSymbol()
+ public override CSharpToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
- StartSymbol();
+ StartToken();
if (EndOfFile || (_visitedFirstTokenStart && _visitedFirstTokenLineEnd))
{
return null;
}
- var symbol = Turn();
+ var token = Turn();
// Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0);
- return symbol;
+ return token;
}
- private bool IsValidTokenType(CSharpSymbolType type)
+ private bool IsValidTokenType(CSharpTokenType type)
{
- return type != CSharpSymbolType.WhiteSpace &&
- type != CSharpSymbolType.NewLine &&
- type != CSharpSymbolType.Comment &&
- type != CSharpSymbolType.RazorComment &&
- type != CSharpSymbolType.RazorCommentStar &&
- type != CSharpSymbolType.RazorCommentTransition &&
- type != CSharpSymbolType.Transition;
+ return type != CSharpTokenType.WhiteSpace &&
+ type != CSharpTokenType.NewLine &&
+ type != CSharpTokenType.Comment &&
+ type != CSharpTokenType.RazorComment &&
+ type != CSharpTokenType.RazorCommentStar &&
+ type != CSharpTokenType.RazorCommentTransition &&
+ type != CSharpTokenType.Transition;
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs
index 5e52dd6ade..034977c758 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/DirectiveHtmlTokenizer.cs
@@ -24,33 +24,33 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return result;
}
- public override HtmlSymbol NextSymbol()
+ public override HtmlToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
- StartSymbol();
+ StartToken();
if (EndOfFile || _visitedFirstTokenStart)
{
return null;
}
- var symbol = Turn();
+ var token = Turn();
// Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0);
- return symbol;
+ return token;
}
- private bool IsValidTokenType(HtmlSymbolType type)
+ private bool IsValidTokenType(HtmlTokenType type)
{
- return type != HtmlSymbolType.WhiteSpace &&
- type != HtmlSymbolType.NewLine &&
- type != HtmlSymbolType.RazorComment &&
- type != HtmlSymbolType.RazorCommentStar &&
- type != HtmlSymbolType.RazorCommentTransition &&
- type != HtmlSymbolType.Transition;
+ return type != HtmlTokenType.WhiteSpace &&
+ type != HtmlTokenType.NewLine &&
+ type != HtmlTokenType.RazorComment &&
+ type != HtmlTokenType.RazorCommentStar &&
+ type != HtmlTokenType.RazorCommentTransition &&
+ type != HtmlTokenType.Transition;
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
index cc44a34368..096ae84dc0 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
@@ -6,7 +6,7 @@ using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class HtmlLanguageCharacteristics : LanguageCharacteristics
+ internal class HtmlLanguageCharacteristics : LanguageCharacteristics
{
private static readonly HtmlLanguageCharacteristics _instance = new HtmlLanguageCharacteristics();
@@ -19,50 +19,50 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _instance; }
}
- public override string GetSample(HtmlSymbolType type)
+ public override string GetSample(HtmlTokenType type)
{
switch (type)
{
- case HtmlSymbolType.Text:
- return Resources.HtmlSymbol_Text;
- case HtmlSymbolType.WhiteSpace:
- return Resources.HtmlSymbol_WhiteSpace;
- case HtmlSymbolType.NewLine:
- return Resources.HtmlSymbol_NewLine;
- case HtmlSymbolType.OpenAngle:
+ case HtmlTokenType.Text:
+ return Resources.HtmlToken_Text;
+ case HtmlTokenType.WhiteSpace:
+ return Resources.HtmlToken_WhiteSpace;
+ case HtmlTokenType.NewLine:
+ return Resources.HtmlToken_NewLine;
+ case HtmlTokenType.OpenAngle:
return "<";
- case HtmlSymbolType.Bang:
+ case HtmlTokenType.Bang:
return "!";
- case HtmlSymbolType.ForwardSlash:
+ case HtmlTokenType.ForwardSlash:
return "/";
- case HtmlSymbolType.QuestionMark:
+ case HtmlTokenType.QuestionMark:
return "?";
- case HtmlSymbolType.DoubleHyphen:
+ case HtmlTokenType.DoubleHyphen:
return "--";
- case HtmlSymbolType.LeftBracket:
+ case HtmlTokenType.LeftBracket:
return "[";
- case HtmlSymbolType.CloseAngle:
+ case HtmlTokenType.CloseAngle:
return ">";
- case HtmlSymbolType.RightBracket:
+ case HtmlTokenType.RightBracket:
return "]";
- case HtmlSymbolType.Equals:
+ case HtmlTokenType.Equals:
return "=";
- case HtmlSymbolType.DoubleQuote:
+ case HtmlTokenType.DoubleQuote:
return "\"";
- case HtmlSymbolType.SingleQuote:
+ case HtmlTokenType.SingleQuote:
return "'";
- case HtmlSymbolType.Transition:
+ case HtmlTokenType.Transition:
return "@";
- case HtmlSymbolType.Colon:
+ case HtmlTokenType.Colon:
return ":";
- case HtmlSymbolType.RazorComment:
- return Resources.HtmlSymbol_RazorComment;
- case HtmlSymbolType.RazorCommentStar:
+ case HtmlTokenType.RazorComment:
+ return Resources.HtmlToken_RazorComment;
+ case HtmlTokenType.RazorCommentStar:
return "*";
- case HtmlSymbolType.RazorCommentTransition:
+ case HtmlTokenType.RazorCommentTransition:
return "@";
default:
- return Resources.Symbol_Unknown;
+ return Resources.Token_Unknown;
}
}
@@ -71,57 +71,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new HtmlTokenizer(source);
}
- public override HtmlSymbolType FlipBracket(HtmlSymbolType bracket)
+ public override HtmlTokenType FlipBracket(HtmlTokenType bracket)
{
switch (bracket)
{
- case HtmlSymbolType.LeftBracket:
- return HtmlSymbolType.RightBracket;
- case HtmlSymbolType.OpenAngle:
- return HtmlSymbolType.CloseAngle;
- case HtmlSymbolType.RightBracket:
- return HtmlSymbolType.LeftBracket;
- case HtmlSymbolType.CloseAngle:
- return HtmlSymbolType.OpenAngle;
+ case HtmlTokenType.LeftBracket:
+ return HtmlTokenType.RightBracket;
+ case HtmlTokenType.OpenAngle:
+ return HtmlTokenType.CloseAngle;
+ case HtmlTokenType.RightBracket:
+ return HtmlTokenType.LeftBracket;
+ case HtmlTokenType.CloseAngle:
+ return HtmlTokenType.OpenAngle;
default:
Debug.Fail("FlipBracket must be called with a bracket character");
- return HtmlSymbolType.Unknown;
+ return HtmlTokenType.Unknown;
}
}
- public override HtmlSymbol CreateMarkerSymbol()
+ public override HtmlToken CreateMarkerToken()
{
- return new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
+ return new HtmlToken(string.Empty, HtmlTokenType.Unknown);
}
- public override HtmlSymbolType GetKnownSymbolType(KnownSymbolType type)
+ public override HtmlTokenType GetKnownTokenType(KnownTokenType type)
{
switch (type)
{
- case KnownSymbolType.CommentStart:
- return HtmlSymbolType.RazorCommentTransition;
- case KnownSymbolType.CommentStar:
- return HtmlSymbolType.RazorCommentStar;
- case KnownSymbolType.CommentBody:
- return HtmlSymbolType.RazorComment;
- case KnownSymbolType.Identifier:
- return HtmlSymbolType.Text;
- case KnownSymbolType.Keyword:
- return HtmlSymbolType.Text;
- case KnownSymbolType.NewLine:
- return HtmlSymbolType.NewLine;
- case KnownSymbolType.Transition:
- return HtmlSymbolType.Transition;
- case KnownSymbolType.WhiteSpace:
- return HtmlSymbolType.WhiteSpace;
+ case KnownTokenType.CommentStart:
+ return HtmlTokenType.RazorCommentTransition;
+ case KnownTokenType.CommentStar:
+ return HtmlTokenType.RazorCommentStar;
+ case KnownTokenType.CommentBody:
+ return HtmlTokenType.RazorComment;
+ case KnownTokenType.Identifier:
+ return HtmlTokenType.Text;
+ case KnownTokenType.Keyword:
+ return HtmlTokenType.Text;
+ case KnownTokenType.NewLine:
+ return HtmlTokenType.NewLine;
+ case KnownTokenType.Transition:
+ return HtmlTokenType.Transition;
+ case KnownTokenType.WhiteSpace:
+ return HtmlTokenType.WhiteSpace;
default:
- return HtmlSymbolType.Unknown;
+ return HtmlTokenType.Unknown;
}
}
- protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList errors)
+ protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList errors)
{
- return new HtmlSymbol(content, type, errors);
+ return new HtmlToken(content, type, errors);
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
index d038f58cc5..cbf0c1021a 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
@@ -8,16 +8,16 @@ using System.Linq;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class HtmlMarkupParser : TokenizerBackedParser
+ internal class HtmlMarkupParser : TokenizerBackedParser
{
private const string ScriptTagName = "script";
- private static readonly HtmlSymbol[] nonAllowedHtmlCommentEnding = new[] { HtmlSymbol.Hyphen, new HtmlSymbol("!", HtmlSymbolType.Bang), new HtmlSymbol("<", HtmlSymbolType.OpenAngle) };
- private static readonly HtmlSymbol[] singleHyphenArray = new[] { HtmlSymbol.Hyphen };
+ private static readonly HtmlToken[] nonAllowedHtmlCommentEnding = new[] { HtmlToken.Hyphen, new HtmlToken("!", HtmlTokenType.Bang), new HtmlToken("<", HtmlTokenType.OpenAngle) };
+ private static readonly HtmlToken[] singleHyphenArray = new[] { HtmlToken.Hyphen };
private static readonly char[] ValidAfterTypeAttributeNameCharacters = { ' ', '\t', '\r', '\n', '\f', '=' };
private SourceLocation _lastTagStart = SourceLocation.Zero;
- private HtmlSymbol _bufferedOpenAngle;
+ private HtmlToken _bufferedOpenAngle;
//From http://dev.w3.org/html5/spec/Overview.html#elements-0
private ISet _voidElements = new HashSet(StringComparer.OrdinalIgnoreCase)
@@ -59,7 +59,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return CaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; }
}
- protected override bool SymbolTypeEquals(HtmlSymbolType x, HtmlSymbolType y) => x == y;
+ protected override bool TokenTypeEquals(HtmlTokenType x, HtmlTokenType y) => x == y;
public override void BuildSpan(SpanBuilder span, SourceLocation start, string content)
{
@@ -73,30 +73,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup);
}
- protected void SkipToAndParseCode(HtmlSymbolType type)
+ protected void SkipToAndParseCode(HtmlTokenType type)
{
- SkipToAndParseCode(sym => sym.Type == type);
+ SkipToAndParseCode(token => token.Type == type);
}
- protected void SkipToAndParseCode(Func condition)
+ protected void SkipToAndParseCode(Func condition)
{
- HtmlSymbol last = null;
+ HtmlToken last = null;
var startOfLine = false;
- while (!EndOfFile && !condition(CurrentSymbol))
+ while (!EndOfFile && !condition(CurrentToken))
{
if (Context.NullGenerateWhitespaceAndNewLine)
{
Context.NullGenerateWhitespaceAndNewLine = false;
Span.ChunkGenerator = SpanChunkGenerator.Null;
- AcceptWhile(symbol => symbol.Type == HtmlSymbolType.WhiteSpace);
- if (At(HtmlSymbolType.NewLine))
+ AcceptWhile(token => token.Type == HtmlTokenType.WhiteSpace);
+ if (At(HtmlTokenType.NewLine))
{
AcceptAndMoveNext();
}
Output(SpanKindInternal.Markup);
}
- else if (At(HtmlSymbolType.NewLine))
+ else if (At(HtmlTokenType.NewLine))
{
if (last != null)
{
@@ -108,11 +108,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
last = null;
AcceptAndMoveNext();
}
- else if (At(HtmlSymbolType.Transition))
+ else if (At(HtmlTokenType.Transition))
{
- var transition = CurrentSymbol;
+ var transition = CurrentToken;
NextToken();
- if (At(HtmlSymbolType.Transition))
+ if (At(HtmlTokenType.Transition))
{
if (last != null)
{
@@ -138,7 +138,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Handle whitespace rewriting
if (last != null)
{
- if (!Context.DesignTimeMode && last.Type == HtmlSymbolType.WhiteSpace && startOfLine)
+ if (!Context.DesignTimeMode && last.Type == HtmlTokenType.WhiteSpace && startOfLine)
{
// Put the whitespace back too
startOfLine = false;
@@ -155,15 +155,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
OtherParserBlock();
}
- else if (At(HtmlSymbolType.RazorCommentTransition))
+ else if (At(HtmlTokenType.RazorCommentTransition))
{
if (last != null)
{
// Don't render the whitespace between the start of the line and the razor comment.
- if (startOfLine && last.Type == HtmlSymbolType.WhiteSpace)
+ if (startOfLine && last.Type == HtmlTokenType.WhiteSpace)
{
- AddMarkerSymbolIfNecessary();
- // Output the symbols that may have been accepted prior to the whitespace.
+ AddMarkerTokenIfNecessary();
+ // Output the tokens that may have been accepted prior to the whitespace.
Output(SpanKindInternal.Markup);
Span.ChunkGenerator = SpanChunkGenerator.Null;
@@ -173,15 +173,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
last = null;
}
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Markup);
RazorComment();
// Handle the whitespace and newline at the end of a razor comment.
if (startOfLine &&
- (At(HtmlSymbolType.NewLine) ||
- (At(HtmlSymbolType.WhiteSpace) && NextIs(HtmlSymbolType.NewLine))))
+ (At(HtmlTokenType.NewLine) ||
+ (At(HtmlTokenType.WhiteSpace) && NextIs(HtmlTokenType.NewLine))))
{
AcceptWhile(IsSpacingToken(includeNewLines: false));
AcceptAndMoveNext();
@@ -192,7 +192,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
// As long as we see whitespace, we're still at the "start" of the line
- startOfLine &= At(HtmlSymbolType.WhiteSpace);
+ startOfLine &= At(HtmlTokenType.WhiteSpace);
// If there's a last token, accept it
if (last != null)
@@ -202,7 +202,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Advance
- last = CurrentSymbol;
+ last = CurrentToken;
NextToken();
}
}
@@ -213,14 +213,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- protected static Func IsSpacingToken(bool includeNewLines)
+ protected static Func IsSpacingToken(bool includeNewLines)
{
- return sym => sym.Type == HtmlSymbolType.WhiteSpace || (includeNewLines && sym.Type == HtmlSymbolType.NewLine);
+ return token => token.Type == HtmlTokenType.WhiteSpace || (includeNewLines && token.Type == HtmlTokenType.NewLine);
}
private void OtherParserBlock()
{
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Markup);
using (PushSpanConfig())
@@ -238,12 +238,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var potentialBang = Lookahead(lookahead);
if (potentialBang != null &&
- potentialBang.Type == HtmlSymbolType.Bang)
+ potentialBang.Type == HtmlTokenType.Bang)
{
var afterBang = Lookahead(lookahead + 1);
return afterBang != null &&
- afterBang.Type == HtmlSymbolType.Text &&
+ afterBang.Type == HtmlTokenType.Text &&
!string.Equals(afterBang.Content, "DOCTYPE", StringComparison.OrdinalIgnoreCase);
}
@@ -257,7 +257,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup);
// Accept the parser escape character '!'.
- Assert(HtmlSymbolType.Bang);
+ Assert(HtmlTokenType.Bang);
AcceptAndMoveNext();
// Setup the metacode span that we will be outputing.
@@ -286,23 +286,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptWhile(IsSpacingToken(includeNewLines: true));
- if (CurrentSymbol.Type == HtmlSymbolType.OpenAngle)
+ if (CurrentToken.Type == HtmlTokenType.OpenAngle)
{
// "<" => Implicit Tag Block
- TagBlock(new Stack>());
+ TagBlock(new Stack>());
}
- else if (CurrentSymbol.Type == HtmlSymbolType.Transition)
+ else if (CurrentToken.Type == HtmlTokenType.Transition)
{
// "@" => Explicit Tag/Single Line Block OR Template
Output(SpanKindInternal.Markup);
// Definitely have a transition span
- Assert(HtmlSymbolType.Transition);
+ Assert(HtmlTokenType.Transition);
AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKindInternal.Transition);
- if (At(HtmlSymbolType.Transition))
+ if (At(HtmlTokenType.Transition))
{
Span.ChunkGenerator = SpanChunkGenerator.Null;
AcceptAndMoveNext();
@@ -314,7 +314,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_MarkupBlockMustStartWithTag(
- new SourceSpan(CurrentStart, CurrentSymbol.Content.Length)));
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length)));
}
Output(SpanKindInternal.Markup);
}
@@ -330,10 +330,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void AfterTransition()
{
// "@:" => Explicit Single Line Block
- if (CurrentSymbol.Type == HtmlSymbolType.Text && CurrentSymbol.Content.Length > 0 && CurrentSymbol.Content[0] == ':')
+ if (CurrentToken.Type == HtmlTokenType.Text && CurrentToken.Content.Length > 0 && CurrentToken.Content[0] == ':')
{
// Split the token
- Tuple split = Language.SplitSymbol(CurrentSymbol, 1, HtmlSymbolType.Colon);
+ Tuple split = Language.SplitToken(CurrentToken, 1, HtmlTokenType.Colon);
// The first part (left) is added to this span and we return a MetaCode span
Accept(split.Item1);
@@ -346,9 +346,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
NextToken();
SingleLineMarkup();
}
- else if (CurrentSymbol.Type == HtmlSymbolType.OpenAngle)
+ else if (CurrentToken.Type == HtmlTokenType.OpenAngle)
{
- TagBlock(new Stack>());
+ TagBlock(new Stack>());
}
}
@@ -359,8 +359,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var old = Context.WhiteSpaceIsSignificantToAncestorBlock;
Context.WhiteSpaceIsSignificantToAncestorBlock = true;
Span.EditHandler = new SpanEditHandler(Language.TokenizeString);
- SkipToAndParseCode(HtmlSymbolType.NewLine);
- if (!EndOfFile && CurrentSymbol.Type == HtmlSymbolType.NewLine)
+ SkipToAndParseCode(HtmlTokenType.NewLine);
+ if (!EndOfFile && CurrentToken.Type == HtmlTokenType.NewLine)
{
AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
@@ -370,13 +370,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup);
}
- private void TagBlock(Stack> tags)
+ private void TagBlock(Stack> tags)
{
// Skip Whitespace and Text
var complete = false;
do
{
- SkipToAndParseCode(HtmlSymbolType.OpenAngle);
+ SkipToAndParseCode(HtmlTokenType.OpenAngle);
// Output everything prior to the OpenAngle into a markup span
Output(SpanKindInternal.Markup);
@@ -401,8 +401,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
_bufferedOpenAngle = null;
_lastTagStart = CurrentStart;
- Assert(HtmlSymbolType.OpenAngle);
- _bufferedOpenAngle = CurrentSymbol;
+ Assert(HtmlTokenType.OpenAngle);
+ _bufferedOpenAngle = CurrentToken;
var tagStart = CurrentStart;
if (!NextToken())
{
@@ -440,18 +440,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
private bool AfterTagStart(SourceLocation tagStart,
- Stack> tags,
+ Stack> tags,
bool atSpecialTag,
IDisposable tagBlockWrapper)
{
if (!EndOfFile)
{
- switch (CurrentSymbol.Type)
+ switch (CurrentToken.Type)
{
- case HtmlSymbolType.ForwardSlash:
+ case HtmlTokenType.ForwardSlash:
// End Tag
return EndTag(tagStart, tags, tagBlockWrapper);
- case HtmlSymbolType.Bang:
+ case HtmlTokenType.Bang:
// Comment, CDATA, DOCTYPE, or a parser-escaped HTML tag.
if (atSpecialTag)
{
@@ -462,7 +462,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
goto default;
}
- case HtmlSymbolType.QuestionMark:
+ case HtmlTokenType.QuestionMark:
// XML PI
Accept(_bufferedOpenAngle);
return XmlPI();
@@ -483,15 +483,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool XmlPI()
{
// Accept "?"
- Assert(HtmlSymbolType.QuestionMark);
+ Assert(HtmlTokenType.QuestionMark);
AcceptAndMoveNext();
- return AcceptUntilAll(HtmlSymbolType.QuestionMark, HtmlSymbolType.CloseAngle);
+ return AcceptUntilAll(HtmlTokenType.QuestionMark, HtmlTokenType.CloseAngle);
}
private bool BangTag()
{
// Accept "!"
- Assert(HtmlSymbolType.Bang);
+ Assert(HtmlTokenType.Bang);
if (AcceptAndMoveNext())
{
@@ -499,17 +499,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
using (Context.Builder.StartBlock(BlockKindInternal.HtmlComment))
{
- // Accept the double-hyphen symbol at the beginning of the comment block.
+ // Accept the double-hyphen token at the beginning of the comment block.
AcceptAndMoveNext();
Output(SpanKindInternal.Markup, AcceptedCharactersInternal.None);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.WhiteSpace;
while (!EndOfFile)
{
- SkipToAndParseCode(HtmlSymbolType.DoubleHyphen);
+ SkipToAndParseCode(HtmlTokenType.DoubleHyphen);
var lastDoubleHyphen = AcceptAllButLastDoubleHyphens();
- if (At(HtmlSymbolType.CloseAngle))
+ if (At(HtmlTokenType.CloseAngle))
{
// Output the content in the comment block as a separate markup
Output(SpanKindInternal.Markup, AcceptedCharactersInternal.WhiteSpace);
@@ -527,7 +527,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
}
- else if (CurrentSymbol.Type == HtmlSymbolType.LeftBracket)
+ else if (CurrentToken.Type == HtmlTokenType.LeftBracket)
{
if (AcceptAndMoveNext())
{
@@ -537,19 +537,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
AcceptAndMoveNext();
- return AcceptUntilAll(HtmlSymbolType.CloseAngle);
+ return AcceptUntilAll(HtmlTokenType.CloseAngle);
}
}
return false;
}
- protected HtmlSymbol AcceptAllButLastDoubleHyphens()
+ protected HtmlToken AcceptAllButLastDoubleHyphens()
{
- var lastDoubleHyphen = CurrentSymbol;
+ var lastDoubleHyphen = CurrentToken;
AcceptWhile(s =>
{
- if (NextIs(HtmlSymbolType.DoubleHyphen))
+ if (NextIs(HtmlTokenType.DoubleHyphen))
{
lastDoubleHyphen = s;
return true;
@@ -560,10 +560,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
NextToken();
- if (At(HtmlSymbolType.Text) && IsHyphen(CurrentSymbol))
+ if (At(HtmlTokenType.Text) && IsHyphen(CurrentToken))
{
- // Doing this here to maintain the order of symbols
- if (!NextIs(HtmlSymbolType.CloseAngle))
+ // Doing this here to maintain the order of tokens
+ if (!NextIs(HtmlTokenType.CloseAngle))
{
Accept(lastDoubleHyphen);
lastDoubleHyphen = null;
@@ -575,9 +575,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return lastDoubleHyphen;
}
- internal static bool IsHyphen(HtmlSymbol symbol)
+ internal static bool IsHyphen(HtmlToken token)
{
- return symbol.Equals(HtmlSymbol.Hyphen);
+ return token.Equals(HtmlToken.Hyphen);
}
protected bool IsHtmlCommentAhead()
@@ -598,30 +598,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
*
* */
- if (CurrentSymbol.Type != HtmlSymbolType.DoubleHyphen)
+ if (CurrentToken.Type != HtmlTokenType.DoubleHyphen)
{
return false;
}
// Check condition 2.1
- if (NextIs(HtmlSymbolType.CloseAngle) || NextIs(next => IsHyphen(next) && NextIs(HtmlSymbolType.CloseAngle)))
+ if (NextIs(HtmlTokenType.CloseAngle) || NextIs(next => IsHyphen(next) && NextIs(HtmlTokenType.CloseAngle)))
{
return false;
}
// Check condition 2.2
var isValidComment = false;
- LookaheadUntil((symbol, prevSymbols) =>
+ LookaheadUntil((token, prevTokens) =>
{
- if (symbol.Type == HtmlSymbolType.DoubleHyphen)
+ if (token.Type == HtmlTokenType.DoubleHyphen)
{
- if (NextIs(HtmlSymbolType.CloseAngle))
+ if (NextIs(HtmlTokenType.CloseAngle))
{
// Check condition 2.3: We're at the end of a comment. Check to make sure the text ending is allowed.
- isValidComment = !IsCommentContentEndingInvalid(prevSymbols);
+ isValidComment = !IsCommentContentEndingInvalid(prevTokens);
return true;
}
- else if (NextIs(ns => IsHyphen(ns) && NextIs(HtmlSymbolType.CloseAngle)))
+ else if (NextIs(ns => IsHyphen(ns) && NextIs(HtmlTokenType.CloseAngle)))
{
// Check condition 2.3: we're at the end of a comment, which has an extra dash.
// Need to treat the dash as part of the content and check the ending.
@@ -630,17 +630,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
isValidComment = true;
return true;
}
- else if (NextIs(ns => ns.Type == HtmlSymbolType.Bang && NextIs(HtmlSymbolType.CloseAngle)))
+ else if (NextIs(ns => ns.Type == HtmlTokenType.Bang && NextIs(HtmlTokenType.CloseAngle)))
{
// This is condition 2.2.3
isValidComment = false;
return true;
}
}
- else if (symbol.Type == HtmlSymbolType.OpenAngle)
+ else if (token.Type == HtmlTokenType.OpenAngle)
{
// Checking condition 2.2.1
- if (NextIs(ns => ns.Type == HtmlSymbolType.Bang && NextIs(HtmlSymbolType.DoubleHyphen)))
+ if (NextIs(ns => ns.Type == HtmlTokenType.Bang && NextIs(HtmlTokenType.DoubleHyphen)))
{
isValidComment = false;
return true;
@@ -654,9 +654,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
///
- /// Verifies, that the sequence doesn't end with the "<!-" HtmlSymbols. Note, the first symbol is an opening bracket symbol
+ /// Verifies, that the sequence doesn't end with the "<!-" HtmlTokens. Note, the first token is an opening bracket token
///
- internal static bool IsCommentContentEndingInvalid(IEnumerable sequence)
+ internal static bool IsCommentContentEndingInvalid(IEnumerable sequence)
{
var reversedSequence = sequence.Reverse();
var index = 0;
@@ -678,13 +678,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool CData()
{
- if (CurrentSymbol.Type == HtmlSymbolType.Text && string.Equals(CurrentSymbol.Content, "cdata", StringComparison.OrdinalIgnoreCase))
+ if (CurrentToken.Type == HtmlTokenType.Text && string.Equals(CurrentToken.Content, "cdata", StringComparison.OrdinalIgnoreCase))
{
if (AcceptAndMoveNext())
{
- if (CurrentSymbol.Type == HtmlSymbolType.LeftBracket)
+ if (CurrentToken.Type == HtmlTokenType.LeftBracket)
{
- return AcceptUntilAll(HtmlSymbolType.RightBracket, HtmlSymbolType.RightBracket, HtmlSymbolType.CloseAngle);
+ return AcceptUntilAll(HtmlTokenType.RightBracket, HtmlTokenType.RightBracket, HtmlTokenType.CloseAngle);
}
}
}
@@ -693,12 +693,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
private bool EndTag(SourceLocation tagStart,
- Stack> tags,
+ Stack> tags,
IDisposable tagBlockWrapper)
{
// Accept "/" and move next
- Assert(HtmlSymbolType.ForwardSlash);
- var forwardSlash = CurrentSymbol;
+ Assert(HtmlTokenType.ForwardSlash);
+ var forwardSlash = CurrentToken;
if (!NextToken())
{
Accept(_bufferedOpenAngle);
@@ -708,22 +708,22 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
var tagName = string.Empty;
- HtmlSymbol bangSymbol = null;
+ HtmlToken bangToken = null;
- if (At(HtmlSymbolType.Bang))
+ if (At(HtmlTokenType.Bang))
{
- bangSymbol = CurrentSymbol;
+ bangToken = CurrentToken;
- var nextSymbol = Lookahead(count: 1);
+ var nextToken = Lookahead(count: 1);
- if (nextSymbol != null && nextSymbol.Type == HtmlSymbolType.Text)
+ if (nextToken != null && nextToken.Type == HtmlTokenType.Text)
{
- tagName = "!" + nextSymbol.Content;
+ tagName = "!" + nextToken.Content;
}
}
- else if (At(HtmlSymbolType.Text))
+ else if (At(HtmlTokenType.Text))
{
- tagName = CurrentSymbol.Content;
+ tagName = CurrentToken.Content;
}
var matched = RemoveTag(tags, tagName, tagStart);
@@ -741,32 +741,32 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
OptionalBangEscape();
- AcceptUntil(HtmlSymbolType.CloseAngle);
+ AcceptUntil(HtmlTokenType.CloseAngle);
// Accept the ">"
- return Optional(HtmlSymbolType.CloseAngle);
+ return Optional(HtmlTokenType.CloseAngle);
}
}
private void RecoverTextTag()
{
// We don't want to skip-to and parse because there shouldn't be anything in the body of text tags.
- AcceptUntil(HtmlSymbolType.CloseAngle, HtmlSymbolType.NewLine);
+ AcceptUntil(HtmlTokenType.CloseAngle, HtmlTokenType.NewLine);
// Include the close angle in the text tag block if it's there, otherwise just move on
- Optional(HtmlSymbolType.CloseAngle);
+ Optional(HtmlTokenType.CloseAngle);
}
- private bool EndTextTag(HtmlSymbol solidus, IDisposable tagBlockWrapper)
+ private bool EndTextTag(HtmlToken solidus, IDisposable tagBlockWrapper)
{
Accept(_bufferedOpenAngle);
Accept(solidus);
var textLocation = CurrentStart;
- Assert(HtmlSymbolType.Text);
+ Assert(HtmlTokenType.Text);
AcceptAndMoveNext();
- var seenCloseAngle = Optional(HtmlSymbolType.CloseAngle);
+ var seenCloseAngle = Optional(HtmlTokenType.CloseAngle);
if (!seenCloseAngle)
{
@@ -794,32 +794,32 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
get
{
- if (At(HtmlSymbolType.OpenAngle))
+ if (At(HtmlTokenType.OpenAngle))
{
- if (NextIs(HtmlSymbolType.Bang))
+ if (NextIs(HtmlTokenType.Bang))
{
return !IsBangEscape(lookahead: 1);
}
- return NextIs(HtmlSymbolType.QuestionMark);
+ return NextIs(HtmlTokenType.QuestionMark);
}
return false;
}
}
- private bool IsTagRecoveryStopPoint(HtmlSymbol sym)
+ private bool IsTagRecoveryStopPoint(HtmlToken token)
{
- return sym.Type == HtmlSymbolType.CloseAngle ||
- sym.Type == HtmlSymbolType.ForwardSlash ||
- sym.Type == HtmlSymbolType.OpenAngle ||
- sym.Type == HtmlSymbolType.SingleQuote ||
- sym.Type == HtmlSymbolType.DoubleQuote;
+ return token.Type == HtmlTokenType.CloseAngle ||
+ token.Type == HtmlTokenType.ForwardSlash ||
+ token.Type == HtmlTokenType.OpenAngle ||
+ token.Type == HtmlTokenType.SingleQuote ||
+ token.Type == HtmlTokenType.DoubleQuote;
}
private void TagContent()
{
- if (!At(HtmlSymbolType.WhiteSpace) && !At(HtmlSymbolType.NewLine))
+ if (!At(HtmlTokenType.WhiteSpace) && !At(HtmlTokenType.NewLine))
{
// We should be right after the tag name, so if there's no whitespace or new line, something is wrong
RecoverToEndOfTag();
@@ -836,9 +836,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool IsEndOfTag()
{
- if (At(HtmlSymbolType.ForwardSlash))
+ if (At(HtmlTokenType.ForwardSlash))
{
- if (NextIs(HtmlSymbolType.CloseAngle))
+ if (NextIs(HtmlTokenType.CloseAngle))
{
return true;
}
@@ -847,16 +847,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptAndMoveNext();
}
}
- return At(HtmlSymbolType.CloseAngle) || At(HtmlSymbolType.OpenAngle);
+ return At(HtmlTokenType.CloseAngle) || At(HtmlTokenType.OpenAngle);
}
private void BeforeAttribute()
{
// http://dev.w3.org/html5/spec/tokenization.html#before-attribute-name-state
// Capture whitespace
- var whitespace = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine);
+ var whitespace = ReadWhile(token => token.Type == HtmlTokenType.WhiteSpace || token.Type == HtmlTokenType.NewLine);
- if (At(HtmlSymbolType.Transition))
+ if (At(HtmlTokenType.Transition))
{
// Transition outside of attribute value => Switch to recovery mode
Accept(whitespace);
@@ -866,21 +866,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// http://dev.w3.org/html5/spec/tokenization.html#attribute-name-state
// Read the 'name' (i.e. read until the '=' or whitespace/newline)
- var name = Enumerable.Empty();
- var whitespaceAfterAttributeName = Enumerable.Empty();
- if (IsValidAttributeNameSymbol(CurrentSymbol))
+ var name = Enumerable.Empty();
+ var whitespaceAfterAttributeName = Enumerable.Empty();
+ if (IsValidAttributeNameToken(CurrentToken))
{
- name = ReadWhile(sym =>
- sym.Type != HtmlSymbolType.WhiteSpace &&
- sym.Type != HtmlSymbolType.NewLine &&
- sym.Type != HtmlSymbolType.Equals &&
- sym.Type != HtmlSymbolType.CloseAngle &&
- sym.Type != HtmlSymbolType.OpenAngle &&
- (sym.Type != HtmlSymbolType.ForwardSlash || !NextIs(HtmlSymbolType.CloseAngle)));
+ name = ReadWhile(token =>
+ token.Type != HtmlTokenType.WhiteSpace &&
+ token.Type != HtmlTokenType.NewLine &&
+ token.Type != HtmlTokenType.Equals &&
+ token.Type != HtmlTokenType.CloseAngle &&
+ token.Type != HtmlTokenType.OpenAngle &&
+ (token.Type != HtmlTokenType.ForwardSlash || !NextIs(HtmlTokenType.CloseAngle)));
// capture whitespace after attribute name (if any)
whitespaceAfterAttributeName = ReadWhile(
- sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine);
+ token => token.Type == HtmlTokenType.WhiteSpace || token.Type == HtmlTokenType.NewLine);
}
else
{
@@ -890,7 +890,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return;
}
- if (!At(HtmlSymbolType.Equals))
+ if (!At(HtmlTokenType.Equals))
{
// Minimized attribute
@@ -925,32 +925,32 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
private void AttributePrefix(
- IEnumerable whitespace,
- IEnumerable nameSymbols,
- IEnumerable whitespaceAfterAttributeName)
+ IEnumerable whitespace,
+ IEnumerable nameTokens,
+ IEnumerable whitespaceAfterAttributeName)
{
// First, determine if this is a 'data-' attribute (since those can't use conditional attributes)
- var name = string.Concat(nameSymbols.Select(s => s.Content));
+ var name = string.Concat(nameTokens.Select(s => s.Content));
var attributeCanBeConditional =
Context.FeatureFlags.EXPERIMENTAL_AllowConditionalDataDashAttributes ||
!name.StartsWith("data-", StringComparison.OrdinalIgnoreCase);
// Accept the whitespace and name
Accept(whitespace);
- Accept(nameSymbols);
+ Accept(nameTokens);
// Since this is not a minimized attribute, the whitespace after attribute name belongs to this attribute.
Accept(whitespaceAfterAttributeName);
- Assert(HtmlSymbolType.Equals); // We should be at "="
+ Assert(HtmlTokenType.Equals); // We should be at "="
AcceptAndMoveNext();
- var whitespaceAfterEquals = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine);
- var quote = HtmlSymbolType.Unknown;
- if (At(HtmlSymbolType.SingleQuote) || At(HtmlSymbolType.DoubleQuote))
+ var whitespaceAfterEquals = ReadWhile(token => token.Type == HtmlTokenType.WhiteSpace || token.Type == HtmlTokenType.NewLine);
+ var quote = HtmlTokenType.Unknown;
+ if (At(HtmlTokenType.SingleQuote) || At(HtmlTokenType.DoubleQuote))
{
// Found a quote, the whitespace belongs to this attribute.
Accept(whitespaceAfterEquals);
- quote = CurrentSymbol.Type;
+ quote = CurrentToken.Type;
AcceptAndMoveNext();
}
else if (whitespaceAfterEquals.Any())
@@ -961,7 +961,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// We now have the prefix: (i.e. ' foo="')
- var prefix = new LocationTagged(string.Concat(Span.Symbols.Select(s => s.Content)), Span.Start);
+ var prefix = new LocationTagged(string.Concat(Span.Tokens.Select(s => s.Content)), Span.Start);
if (attributeCanBeConditional)
{
@@ -970,10 +970,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Read the attribute value only if the value is quoted
// or if there is no whitespace between '=' and the unquoted value.
- if (quote != HtmlSymbolType.Unknown || !whitespaceAfterEquals.Any())
+ if (quote != HtmlTokenType.Unknown || !whitespaceAfterEquals.Any())
{
// Read the attribute value.
- while (!EndOfFile && !IsEndOfAttributeValue(quote, CurrentSymbol))
+ while (!EndOfFile && !IsEndOfAttributeValue(quote, CurrentToken))
{
AttributeValue(quote);
}
@@ -981,13 +981,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Capture the suffix
var suffix = new LocationTagged(string.Empty, CurrentStart);
- if (quote != HtmlSymbolType.Unknown && At(quote))
+ if (quote != HtmlTokenType.Unknown && At(quote))
{
- suffix = new LocationTagged(CurrentSymbol.Content, CurrentStart);
+ suffix = new LocationTagged(CurrentToken.Content, CurrentStart);
AcceptAndMoveNext();
}
- if (Span.Symbols.Count > 0)
+ if (Span.Tokens.Count > 0)
{
// Again, block chunk generator will render the suffix
Span.ChunkGenerator = SpanChunkGenerator.Null;
@@ -1003,18 +1003,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Output the attribute name, the equals and optional quote. Ex: foo="
Output(SpanKindInternal.Markup);
- if (quote == HtmlSymbolType.Unknown && whitespaceAfterEquals.Any())
+ if (quote == HtmlTokenType.Unknown && whitespaceAfterEquals.Any())
{
return;
}
// Not a "conditional" attribute, so just read the value
- SkipToAndParseCode(sym => IsEndOfAttributeValue(quote, sym));
+ SkipToAndParseCode(token => IsEndOfAttributeValue(quote, token));
// Output the attribute value (will include everything in-between the attribute's quotes).
Output(SpanKindInternal.Markup);
- if (quote != HtmlSymbolType.Unknown)
+ if (quote != HtmlTokenType.Unknown)
{
Optional(quote);
}
@@ -1022,14 +1022,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- private void AttributeValue(HtmlSymbolType quote)
+ private void AttributeValue(HtmlTokenType quote)
{
var prefixStart = CurrentStart;
- var prefix = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine);
+ var prefix = ReadWhile(token => token.Type == HtmlTokenType.WhiteSpace || token.Type == HtmlTokenType.NewLine);
- if (At(HtmlSymbolType.Transition))
+ if (At(HtmlTokenType.Transition))
{
- if (NextIs(HtmlSymbolType.Transition))
+ if (NextIs(HtmlTokenType.Transition))
{
// Wrapping this in a block so that the ConditionalAttributeCollapser doesn't rewrite it.
using (Context.Builder.StartBlock(BlockKindInternal.Markup))
@@ -1039,7 +1039,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Render a single "@" in place of "@@".
Span.ChunkGenerator = new LiteralAttributeChunkGenerator(
new LocationTagged(string.Concat(prefix.Select(s => s.Content)), prefixStart),
- new LocationTagged(CurrentSymbol.Content, CurrentStart));
+ new LocationTagged(CurrentToken.Content, CurrentStart));
AcceptAndMoveNext();
Output(SpanKindInternal.Markup, AcceptedCharactersInternal.None);
@@ -1074,17 +1074,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(prefix);
// Literal value
- // 'quote' should be "Unknown" if not quoted and symbols coming from the tokenizer should never have
+ // 'quote' should be "Unknown" if not quoted and tokens coming from the tokenizer should never have
// "Unknown" type.
var valueStart = CurrentStart;
- var value = ReadWhile(sym =>
+ var value = ReadWhile(token =>
// These three conditions find separators which break the attribute value into portions
- sym.Type != HtmlSymbolType.WhiteSpace &&
- sym.Type != HtmlSymbolType.NewLine &&
- sym.Type != HtmlSymbolType.Transition &&
+ token.Type != HtmlTokenType.WhiteSpace &&
+ token.Type != HtmlTokenType.NewLine &&
+ token.Type != HtmlTokenType.Transition &&
// This condition checks for the end of the attribute value (it repeats some of the checks above
// but for now that's ok)
- !IsEndOfAttributeValue(quote, sym));
+ !IsEndOfAttributeValue(quote, token));
Accept(value);
Span.ChunkGenerator = new LiteralAttributeChunkGenerator(
new LocationTagged(string.Concat(prefix.Select(s => s.Content)), prefixStart),
@@ -1093,27 +1093,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.Markup);
}
- private bool IsEndOfAttributeValue(HtmlSymbolType quote, HtmlSymbol sym)
+ private bool IsEndOfAttributeValue(HtmlTokenType quote, HtmlToken token)
{
- return EndOfFile || sym == null ||
- (quote != HtmlSymbolType.Unknown
- ? sym.Type == quote // If quoted, just wait for the quote
- : IsUnquotedEndOfAttributeValue(sym));
+ return EndOfFile || token == null ||
+ (quote != HtmlTokenType.Unknown
+ ? token.Type == quote // If quoted, just wait for the quote
+ : IsUnquotedEndOfAttributeValue(token));
}
- private bool IsUnquotedEndOfAttributeValue(HtmlSymbol sym)
+ private bool IsUnquotedEndOfAttributeValue(HtmlToken token)
{
// If unquoted, we have a larger set of terminating characters:
// http://dev.w3.org/html5/spec/tokenization.html#attribute-value-unquoted-state
// Also we need to detect "/" and ">"
- return sym.Type == HtmlSymbolType.DoubleQuote ||
- sym.Type == HtmlSymbolType.SingleQuote ||
- sym.Type == HtmlSymbolType.OpenAngle ||
- sym.Type == HtmlSymbolType.Equals ||
- (sym.Type == HtmlSymbolType.ForwardSlash && NextIs(HtmlSymbolType.CloseAngle)) ||
- sym.Type == HtmlSymbolType.CloseAngle ||
- sym.Type == HtmlSymbolType.WhiteSpace ||
- sym.Type == HtmlSymbolType.NewLine;
+ return token.Type == HtmlTokenType.DoubleQuote ||
+ token.Type == HtmlTokenType.SingleQuote ||
+ token.Type == HtmlTokenType.OpenAngle ||
+ token.Type == HtmlTokenType.Equals ||
+ (token.Type == HtmlTokenType.ForwardSlash && NextIs(HtmlTokenType.CloseAngle)) ||
+ token.Type == HtmlTokenType.CloseAngle ||
+ token.Type == HtmlTokenType.WhiteSpace ||
+ token.Type == HtmlTokenType.NewLine;
}
private void RecoverToEndOfTag()
@@ -1125,17 +1125,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (!EndOfFile)
{
EnsureCurrent();
- switch (CurrentSymbol.Type)
+ switch (CurrentToken.Type)
{
- case HtmlSymbolType.SingleQuote:
- case HtmlSymbolType.DoubleQuote:
+ case HtmlTokenType.SingleQuote:
+ case HtmlTokenType.DoubleQuote:
ParseQuoted();
break;
- case HtmlSymbolType.OpenAngle:
+ case HtmlTokenType.OpenAngle:
// Another "<" means this tag is invalid.
- case HtmlSymbolType.ForwardSlash:
+ case HtmlTokenType.ForwardSlash:
// Empty tag
- case HtmlSymbolType.CloseAngle:
+ case HtmlTokenType.CloseAngle:
// End of tag
return;
default:
@@ -1148,12 +1148,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void ParseQuoted()
{
- var type = CurrentSymbol.Type;
+ var type = CurrentToken.Type;
AcceptAndMoveNext();
ParseQuoted(type);
}
- private void ParseQuoted(HtmlSymbolType type)
+ private void ParseQuoted(HtmlTokenType type)
{
SkipToAndParseCode(type);
if (!EndOfFile)
@@ -1163,38 +1163,38 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- private bool StartTag(Stack> tags, IDisposable tagBlockWrapper)
+ private bool StartTag(Stack> tags, IDisposable tagBlockWrapper)
{
- HtmlSymbol bangSymbol = null;
- HtmlSymbol potentialTagNameSymbol;
+ HtmlToken bangToken = null;
+ HtmlToken potentialTagNameToken;
- if (At(HtmlSymbolType.Bang))
+ if (At(HtmlTokenType.Bang))
{
- bangSymbol = CurrentSymbol;
+ bangToken = CurrentToken;
- potentialTagNameSymbol = Lookahead(count: 1);
+ potentialTagNameToken = Lookahead(count: 1);
}
else
{
- potentialTagNameSymbol = CurrentSymbol;
+ potentialTagNameToken = CurrentToken;
}
- HtmlSymbol tagName;
+ HtmlToken tagName;
- if (potentialTagNameSymbol == null || potentialTagNameSymbol.Type != HtmlSymbolType.Text)
+ if (potentialTagNameToken == null || potentialTagNameToken.Type != HtmlTokenType.Text)
{
- tagName = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
+ tagName = new HtmlToken(string.Empty, HtmlTokenType.Unknown);
}
- else if (bangSymbol != null)
+ else if (bangToken != null)
{
- tagName = new HtmlSymbol("!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text);
+ tagName = new HtmlToken("!" + potentialTagNameToken.Content, HtmlTokenType.Text);
}
else
{
- tagName = potentialTagNameSymbol;
+ tagName = potentialTagNameToken;
}
- Tuple tag = Tuple.Create(tagName, _lastTagStart);
+ Tuple tag = Tuple.Create(tagName, _lastTagStart);
if (tags.Count == 0 &&
// Note tagName may contain a '!' escape character. This ensures doesn't match here.
@@ -1206,23 +1206,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(_bufferedOpenAngle);
var textLocation = CurrentStart;
- Assert(HtmlSymbolType.Text);
+ Assert(HtmlTokenType.Text);
AcceptAndMoveNext();
var bookmark = CurrentStart.AbsoluteIndex;
- IEnumerable tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
- var empty = At(HtmlSymbolType.ForwardSlash);
+ IEnumerable tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
+ var empty = At(HtmlTokenType.ForwardSlash);
if (empty)
{
Accept(tokens);
- Assert(HtmlSymbolType.ForwardSlash);
+ Assert(HtmlTokenType.ForwardSlash);
AcceptAndMoveNext();
bookmark = CurrentStart.AbsoluteIndex;
tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
}
- if (!Optional(HtmlSymbolType.CloseAngle))
+ if (!Optional(HtmlTokenType.CloseAngle))
{
Context.Source.Position = bookmark;
NextToken();
@@ -1250,24 +1250,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(_bufferedOpenAngle);
OptionalBangEscape();
- Optional(HtmlSymbolType.Text);
+ Optional(HtmlTokenType.Text);
return RestOfTag(tag, tags, tagBlockWrapper);
}
- private bool RestOfTag(Tuple tag,
- Stack> tags,
+ private bool RestOfTag(Tuple tag,
+ Stack> tags,
IDisposable tagBlockWrapper)
{
TagContent();
// We are now at a possible end of the tag
// Found '<', so we just abort this tag.
- if (At(HtmlSymbolType.OpenAngle))
+ if (At(HtmlTokenType.OpenAngle))
{
return false;
}
- var isEmpty = At(HtmlSymbolType.ForwardSlash);
+ var isEmpty = At(HtmlTokenType.ForwardSlash);
// Found a solidus, so don't accept it but DON'T push the tag to the stack
if (isEmpty)
{
@@ -1275,7 +1275,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Check for the '>' to determine if the tag is finished
- var seenClose = Optional(HtmlSymbolType.CloseAngle);
+ var seenClose = Optional(HtmlTokenType.CloseAngle);
if (!seenClose)
{
Context.ErrorSink.OnError(
@@ -1301,17 +1301,17 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var bookmark = CurrentStart.AbsoluteIndex;
// Skip whitespace
- IEnumerable whiteSpace = ReadWhile(IsSpacingToken(includeNewLines: true));
+ IEnumerable whiteSpace = ReadWhile(IsSpacingToken(includeNewLines: true));
// Open Angle
- if (At(HtmlSymbolType.OpenAngle) && NextIs(HtmlSymbolType.ForwardSlash))
+ if (At(HtmlTokenType.OpenAngle) && NextIs(HtmlTokenType.ForwardSlash))
{
- var openAngle = CurrentSymbol;
+ var openAngle = CurrentToken;
NextToken();
- Assert(HtmlSymbolType.ForwardSlash);
- var solidus = CurrentSymbol;
+ Assert(HtmlTokenType.ForwardSlash);
+ var solidus = CurrentToken;
NextToken();
- if (At(HtmlSymbolType.Text) && string.Equals(CurrentSymbol.Content, tagName, StringComparison.OrdinalIgnoreCase))
+ if (At(HtmlTokenType.Text) && string.Equals(CurrentToken.Content, tagName, StringComparison.OrdinalIgnoreCase))
{
// Accept up to here
Accept(whiteSpace);
@@ -1324,9 +1324,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptAndMoveNext();
// Accept to '>', '<' or EOF
- AcceptUntil(HtmlSymbolType.CloseAngle, HtmlSymbolType.OpenAngle);
+ AcceptUntil(HtmlTokenType.CloseAngle, HtmlTokenType.OpenAngle);
// Accept the '>' if we saw it. And if we do see it, we're complete
- var complete = Optional(HtmlSymbolType.CloseAngle);
+ var complete = Optional(HtmlTokenType.CloseAngle);
if (complete)
{
@@ -1376,18 +1376,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
while (!seenEndScript && !EndOfFile)
{
- SkipToAndParseCode(HtmlSymbolType.OpenAngle);
+ SkipToAndParseCode(HtmlTokenType.OpenAngle);
var tagStart = CurrentStart;
- if (NextIs(HtmlSymbolType.ForwardSlash))
+ if (NextIs(HtmlTokenType.ForwardSlash))
{
- var openAngle = CurrentSymbol;
+ var openAngle = CurrentToken;
NextToken(); // Skip over '<', current is '/'
- var solidus = CurrentSymbol;
+ var solidus = CurrentToken;
NextToken(); // Skip over '/', current should be text
- if (At(HtmlSymbolType.Text) &&
- string.Equals(CurrentSymbol.Content, ScriptTagName, StringComparison.OrdinalIgnoreCase))
+ if (At(HtmlTokenType.Text) &&
+ string.Equals(CurrentToken.Content, ScriptTagName, StringComparison.OrdinalIgnoreCase))
{
seenEndScript = true;
}
@@ -1398,7 +1398,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
PutBack(solidus); // Put back '/'
PutBack(openAngle); // Put back '<'
- // We just looked ahead, this NextToken will set CurrentSymbol to an open angle bracket.
+ // We just looked ahead, this NextToken will set CurrentToken to an open angle bracket.
NextToken();
}
@@ -1412,8 +1412,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AcceptAndMoveNext(); // '<'
AcceptAndMoveNext(); // '/'
- SkipToAndParseCode(HtmlSymbolType.CloseAngle);
- if (!Optional(HtmlSymbolType.CloseAngle))
+ SkipToAndParseCode(HtmlTokenType.CloseAngle);
+ if (!Optional(HtmlTokenType.CloseAngle))
{
Context.ErrorSink.OnError(
RazorDiagnosticFactory.CreateParsing_UnfinishedTag(
@@ -1444,7 +1444,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
tagBlockWrapper.Dispose();
}
- private bool AcceptUntilAll(params HtmlSymbolType[] endSequence)
+ private bool AcceptUntilAll(params HtmlTokenType[] endSequence)
{
while (!EndOfFile)
{
@@ -1459,9 +1459,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- private bool RemoveTag(Stack> tags, string tagName, SourceLocation tagStart)
+ private bool RemoveTag(Stack> tags, string tagName, SourceLocation tagStart)
{
- Tuple currentTag = null;
+ Tuple currentTag = null;
while (tags.Count > 0)
{
currentTag = tags.Pop();
@@ -1489,7 +1489,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- private void EndTagBlock(Stack> tags, bool complete)
+ private void EndTagBlock(Stack> tags, bool complete)
{
if (tags.Count > 0)
{
@@ -1517,46 +1517,46 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
if (Context.Builder.LastSpan.Kind == SpanKindInternal.Transition)
{
- var symbols = ReadWhile(
- f => (f.Type == HtmlSymbolType.WhiteSpace) || (f.Type == HtmlSymbolType.NewLine));
+ var tokens = ReadWhile(
+ f => (f.Type == HtmlTokenType.WhiteSpace) || (f.Type == HtmlTokenType.NewLine));
- // Make sure the current symbol is not markup, which can be html start tag or @:
- if (!(At(HtmlSymbolType.OpenAngle) ||
- (At(HtmlSymbolType.Transition) && Lookahead(count: 1).Content.StartsWith(":"))))
+ // Make sure the current token is not markup, which can be html start tag or @:
+ if (!(At(HtmlTokenType.OpenAngle) ||
+ (At(HtmlTokenType.Transition) && Lookahead(count: 1).Content.StartsWith(":"))))
{
// Don't accept whitespace as markup if the end text tag is followed by csharp.
shouldAcceptWhitespaceAndNewLine = false;
}
PutCurrentBack();
- PutBack(symbols);
+ PutBack(tokens);
EnsureCurrent();
}
if (shouldAcceptWhitespaceAndNewLine)
{
// Accept whitespace and a single newline if present
- AcceptWhile(HtmlSymbolType.WhiteSpace);
- Optional(HtmlSymbolType.NewLine);
+ AcceptWhile(HtmlTokenType.WhiteSpace);
+ Optional(HtmlTokenType.NewLine);
}
}
else if (Span.EditHandler.AcceptedCharacters == AcceptedCharactersInternal.Any)
{
- AcceptWhile(HtmlSymbolType.WhiteSpace);
- Optional(HtmlSymbolType.NewLine);
+ AcceptWhile(HtmlTokenType.WhiteSpace);
+ Optional(HtmlTokenType.NewLine);
}
PutCurrentBack();
if (!complete)
{
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
}
Output(SpanKindInternal.Markup);
}
- internal static bool IsValidAttributeNameSymbol(HtmlSymbol symbol)
+ internal static bool IsValidAttributeNameToken(HtmlToken token)
{
- if (symbol == null)
+ if (token == null)
{
return false;
}
@@ -1565,16 +1565,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// However, it's not all of it. For instance we don't special case control characters or allow OpenAngle.
// It also doesn't try to exclude Razor specific features such as the @ transition. This is based on the
// expectation that the parser handles such scenarios prior to falling through to name resolution.
- var symbolType = symbol.Type;
- return symbolType != HtmlSymbolType.WhiteSpace &&
- symbolType != HtmlSymbolType.NewLine &&
- symbolType != HtmlSymbolType.CloseAngle &&
- symbolType != HtmlSymbolType.OpenAngle &&
- symbolType != HtmlSymbolType.ForwardSlash &&
- symbolType != HtmlSymbolType.DoubleQuote &&
- symbolType != HtmlSymbolType.SingleQuote &&
- symbolType != HtmlSymbolType.Equals &&
- symbolType != HtmlSymbolType.Unknown;
+ var tokenType = token.Type;
+ return tokenType != HtmlTokenType.WhiteSpace &&
+ tokenType != HtmlTokenType.NewLine &&
+ tokenType != HtmlTokenType.CloseAngle &&
+ tokenType != HtmlTokenType.OpenAngle &&
+ tokenType != HtmlTokenType.ForwardSlash &&
+ tokenType != HtmlTokenType.DoubleQuote &&
+ tokenType != HtmlTokenType.SingleQuote &&
+ tokenType != HtmlTokenType.Equals &&
+ tokenType != HtmlTokenType.Unknown;
}
public void ParseDocument()
@@ -1593,10 +1593,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
NextToken();
while (!EndOfFile)
{
- SkipToAndParseCode(HtmlSymbolType.OpenAngle);
+ SkipToAndParseCode(HtmlTokenType.OpenAngle);
ScanTagInDocumentContext();
}
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Markup);
}
}
@@ -1608,14 +1608,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
///
private void ScanTagInDocumentContext()
{
- if (At(HtmlSymbolType.OpenAngle))
+ if (At(HtmlTokenType.OpenAngle))
{
- if (NextIs(HtmlSymbolType.Bang))
+ if (NextIs(HtmlTokenType.Bang))
{
// Checking to see if we meet the conditions of a special '!' tag: ';" as an HTML tag.
@@ -1670,13 +1670,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Parsing an end tag
// This section can accept things like: '
' or '' etc.
- Optional(HtmlSymbolType.ForwardSlash);
+ Optional(HtmlTokenType.ForwardSlash);
// Whitespace here is invalid (according to the spec)
OptionalBangEscape();
- Optional(HtmlSymbolType.Text);
- Optional(HtmlSymbolType.WhiteSpace);
- Optional(HtmlSymbolType.CloseAngle);
+ Optional(HtmlTokenType.Text);
+ Optional(HtmlTokenType.WhiteSpace);
+ Optional(HtmlTokenType.CloseAngle);
}
Output(SpanKindInternal.Markup);
@@ -1758,7 +1758,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
NestingSection(nestingSequences);
}
- AddMarkerSymbolIfNecessary();
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Markup);
}
}
@@ -1768,7 +1768,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
do
{
- SkipToAndParseCode(sym => sym.Type == HtmlSymbolType.OpenAngle || AtEnd(nestingSequenceComponents));
+ SkipToAndParseCode(token => token.Type == HtmlTokenType.OpenAngle || AtEnd(nestingSequenceComponents));
ScanTagInDocumentContext();
if (!EndOfFile && AtEnd(nestingSequenceComponents))
{
@@ -1785,13 +1785,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var nesting = 1;
while (nesting > 0 && !EndOfFile)
{
- SkipToAndParseCode(sym =>
- sym.Type == HtmlSymbolType.Text ||
- sym.Type == HtmlSymbolType.OpenAngle);
- if (At(HtmlSymbolType.Text))
+ SkipToAndParseCode(token =>
+ token.Type == HtmlTokenType.Text ||
+ token.Type == HtmlTokenType.OpenAngle);
+ if (At(HtmlTokenType.Text))
{
nesting += ProcessTextToken(nestingSequences, nesting);
- if (CurrentSymbol != null)
+ if (CurrentToken != null)
{
AcceptAndMoveNext();
}
@@ -1810,19 +1810,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private bool AtEnd(string[] nestingSequenceComponents)
{
EnsureCurrent();
- if (string.Equals(CurrentSymbol.Content, nestingSequenceComponents[0], Comparison))
+ if (string.Equals(CurrentToken.Content, nestingSequenceComponents[0], Comparison))
{
- var bookmark = Context.Source.Position - CurrentSymbol.Content.Length;
+ var bookmark = Context.Source.Position - CurrentToken.Content.Length;
try
{
foreach (string component in nestingSequenceComponents)
{
- if (!EndOfFile && !string.Equals(CurrentSymbol.Content, component, Comparison))
+ if (!EndOfFile && !string.Equals(CurrentToken.Content, component, Comparison))
{
return false;
}
NextToken();
- while (!EndOfFile && IsSpacingToken(includeNewLines: true)(CurrentSymbol))
+ while (!EndOfFile && IsSpacingToken(includeNewLines: true)(CurrentToken))
{
NextToken();
}
@@ -1840,7 +1840,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private int ProcessTextToken(Tuple nestingSequences, int currentNesting)
{
- for (int i = 0; i < CurrentSymbol.Content.Length; i++)
+ for (int i = 0; i < CurrentToken.Content.Length; i++)
{
var nestingDelta = HandleNestingSequence(nestingSequences.Item1, i, currentNesting, 1);
if (nestingDelta == 0)
@@ -1859,22 +1859,22 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private int HandleNestingSequence(string sequence, int position, int currentNesting, int retIfMatched)
{
if (sequence != null &&
- CurrentSymbol.Content[position] == sequence[0] &&
- position + sequence.Length <= CurrentSymbol.Content.Length)
+ CurrentToken.Content[position] == sequence[0] &&
+ position + sequence.Length <= CurrentToken.Content.Length)
{
- var possibleStart = CurrentSymbol.Content.Substring(position, sequence.Length);
+ var possibleStart = CurrentToken.Content.Substring(position, sequence.Length);
if (string.Equals(possibleStart, sequence, Comparison))
{
- // Capture the current symbol and "put it back" (really we just want to clear CurrentSymbol)
+ // Capture the current token and "put it back" (really we just want to clear CurrentToken)
var bookmark = CurrentStart;
- var sym = CurrentSymbol;
+ var token = CurrentToken;
PutCurrentBack();
- // Carve up the symbol
- Tuple pair = Language.SplitSymbol(sym, position, HtmlSymbolType.Text);
+ // Carve up the token
+ Tuple pair = Language.SplitToken(token, position, HtmlTokenType.Text);
var preSequence = pair.Item1;
Debug.Assert(pair.Item2 != null);
- pair = Language.SplitSymbol(pair.Item2, sequence.Length, HtmlSymbolType.Text);
+ pair = Language.SplitToken(pair.Item2, sequence.Length, HtmlTokenType.Text);
var sequenceToken = pair.Item1;
var postSequence = pair.Item2;
var postSequenceBookmark = bookmark.AbsoluteIndex + preSequence.Content.Length + pair.Item1.Content.Length;
@@ -1897,7 +1897,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// This isn't the end of the last nesting sequence, accept the token and keep going
Accept(sequenceToken);
- // Position at the start of the postSequence symbol, which might be null.
+ // Position at the start of the postSequence token, which might be null.
Context.Source.Position = postSequenceBookmark;
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbol.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlToken.cs
similarity index 73%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbol.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlToken.cs
index 217e293704..2d52ed14e2 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbol.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlToken.cs
@@ -6,11 +6,11 @@ using System.Collections.Generic;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class HtmlSymbol : SymbolBase
+ internal class HtmlToken : TokenBase
{
- internal static readonly HtmlSymbol Hyphen = new HtmlSymbol("-", HtmlSymbolType.Text);
+ internal static readonly HtmlToken Hyphen = new HtmlToken("-", HtmlTokenType.Text);
- public HtmlSymbol(string content, HtmlSymbolType type)
+ public HtmlToken(string content, HtmlTokenType type)
: base(content, type, RazorDiagnostic.EmptyArray)
{
if (content == null)
@@ -19,9 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- public HtmlSymbol(
+ public HtmlToken(
string content,
- HtmlSymbolType type,
+ HtmlTokenType type,
IReadOnlyList errors)
: base(content, type, errors)
{
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbolType.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenType.cs
similarity index 96%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbolType.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenType.cs
index bc90b98022..82d96d7ebc 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlSymbolType.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenType.cs
@@ -6,7 +6,7 @@ using System;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
[Flags]
- internal enum HtmlSymbolType
+ internal enum HtmlTokenType
{
Unknown,
Text, // Text which isn't one of the below
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs
index df9671d1b7..99a7152447 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlTokenizer.cs
@@ -7,7 +7,7 @@ using System.Diagnostics;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization
- internal class HtmlTokenizer : Tokenizer
+ internal class HtmlTokenizer : Tokenizer
{
private const char TransitionChar = '@';
@@ -21,24 +21,24 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private new HtmlTokenizerState? CurrentState => (HtmlTokenizerState?)base.CurrentState;
- public override HtmlSymbolType RazorCommentType
+ public override HtmlTokenType RazorCommentType
{
- get { return HtmlSymbolType.RazorComment; }
+ get { return HtmlTokenType.RazorComment; }
}
- public override HtmlSymbolType RazorCommentTransitionType
+ public override HtmlTokenType RazorCommentTransitionType
{
- get { return HtmlSymbolType.RazorCommentTransition; }
+ get { return HtmlTokenType.RazorCommentTransition; }
}
- public override HtmlSymbolType RazorCommentStarType
+ public override HtmlTokenType RazorCommentStarType
{
- get { return HtmlSymbolType.RazorCommentStar; }
+ get { return HtmlTokenType.RazorCommentStar; }
}
- protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList errors)
+ protected override HtmlToken CreateToken(string content, HtmlTokenType type, IReadOnlyList errors)
{
- return new HtmlSymbol(content, type, errors);
+ return new HtmlToken(content, type, errors);
}
protected override StateResult Dispatch()
@@ -57,8 +57,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return RazorCommentBody();
case HtmlTokenizerState.StarAfterRazorCommentBody:
return StarAfterRazorCommentBody();
- case HtmlTokenizerState.AtSymbolAfterRazorCommentBody:
- return AtSymbolAfterRazorCommentBody();
+ case HtmlTokenizerState.AtTokenAfterRazorCommentBody:
+ return AtTokenAfterRazorCommentBody();
default:
Debug.Fail("Invalid TokenizerState");
return default(StateResult);
@@ -66,35 +66,35 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Optimize memory allocation by returning constants for the most frequent cases
- protected override string GetSymbolContent(HtmlSymbolType type)
+ protected override string GetTokenContent(HtmlTokenType type)
{
- var symbolLength = Buffer.Length;
+ var tokenLength = Buffer.Length;
- if (symbolLength == 1)
+ if (tokenLength == 1)
{
switch (type)
{
- case HtmlSymbolType.OpenAngle:
+ case HtmlTokenType.OpenAngle:
return "<";
- case HtmlSymbolType.Bang:
+ case HtmlTokenType.Bang:
return "!";
- case HtmlSymbolType.ForwardSlash:
+ case HtmlTokenType.ForwardSlash:
return "/";
- case HtmlSymbolType.QuestionMark:
+ case HtmlTokenType.QuestionMark:
return "?";
- case HtmlSymbolType.LeftBracket:
+ case HtmlTokenType.LeftBracket:
return "[";
- case HtmlSymbolType.CloseAngle:
+ case HtmlTokenType.CloseAngle:
return ">";
- case HtmlSymbolType.RightBracket:
+ case HtmlTokenType.RightBracket:
return "]";
- case HtmlSymbolType.Equals:
+ case HtmlTokenType.Equals:
return "=";
- case HtmlSymbolType.DoubleQuote:
+ case HtmlTokenType.DoubleQuote:
return "\"";
- case HtmlSymbolType.SingleQuote:
+ case HtmlTokenType.SingleQuote:
return "'";
- case HtmlSymbolType.WhiteSpace:
+ case HtmlTokenType.WhiteSpace:
if (Buffer[0] == ' ')
{
return " ";
@@ -104,7 +104,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return "\t";
}
break;
- case HtmlSymbolType.NewLine:
+ case HtmlTokenType.NewLine:
if (Buffer[0] == '\n')
{
return "\n";
@@ -113,12 +113,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- if (symbolLength == 2 && type == HtmlSymbolType.NewLine)
+ if (tokenLength == 2 && type == HtmlTokenType.NewLine)
{
return "\r\n";
}
- return base.GetSymbolContent(type);
+ return base.GetTokenContent(type);
}
// http://dev.w3.org/html5/spec/Overview.html#data-state
@@ -139,21 +139,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
HtmlTokenizerState.AfterRazorCommentTransition,
- EndSymbol(HtmlSymbolType.RazorCommentTransition));
+ EndToken(HtmlTokenType.RazorCommentTransition));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return Transition(
HtmlTokenizerState.EscapedRazorCommentTransition,
- EndSymbol(HtmlSymbolType.Transition));
+ EndToken(HtmlTokenType.Transition));
}
- return Stay(EndSymbol(HtmlSymbolType.Transition));
+ return Stay(EndToken(HtmlTokenType.Transition));
}
- else if (AtSymbol())
+ else if (AtToken())
{
- return Stay(Symbol());
+ return Stay(Token());
}
else
{
@@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private StateResult EscapedRazorCommentTransition()
{
TakeCurrent();
- return Transition(HtmlTokenizerState.Data, EndSymbol(HtmlSymbolType.Transition));
+ return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Transition));
}
private StateResult Text()
@@ -172,7 +172,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var prev = '\0';
while (!EndOfFile &&
!(ParserHelpers.IsWhitespace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) &&
- !AtSymbol())
+ !AtToken())
{
prev = CurrentCharacter;
TakeCurrent();
@@ -190,56 +190,56 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Output the Text token and return to the Data state to tokenize the next character (if there is one)
- return Transition(HtmlTokenizerState.Data, EndSymbol(HtmlSymbolType.Text));
+ return Transition(HtmlTokenizerState.Data, EndToken(HtmlTokenType.Text));
}
- private HtmlSymbol Symbol()
+ private HtmlToken Token()
{
- Debug.Assert(AtSymbol());
+ Debug.Assert(AtToken());
var sym = CurrentCharacter;
TakeCurrent();
switch (sym)
{
case '<':
- return EndSymbol(HtmlSymbolType.OpenAngle);
+ return EndToken(HtmlTokenType.OpenAngle);
case '!':
- return EndSymbol(HtmlSymbolType.Bang);
+ return EndToken(HtmlTokenType.Bang);
case '/':
- return EndSymbol(HtmlSymbolType.ForwardSlash);
+ return EndToken(HtmlTokenType.ForwardSlash);
case '?':
- return EndSymbol(HtmlSymbolType.QuestionMark);
+ return EndToken(HtmlTokenType.QuestionMark);
case '[':
- return EndSymbol(HtmlSymbolType.LeftBracket);
+ return EndToken(HtmlTokenType.LeftBracket);
case '>':
- return EndSymbol(HtmlSymbolType.CloseAngle);
+ return EndToken(HtmlTokenType.CloseAngle);
case ']':
- return EndSymbol(HtmlSymbolType.RightBracket);
+ return EndToken(HtmlTokenType.RightBracket);
case '=':
- return EndSymbol(HtmlSymbolType.Equals);
+ return EndToken(HtmlTokenType.Equals);
case '"':
- return EndSymbol(HtmlSymbolType.DoubleQuote);
+ return EndToken(HtmlTokenType.DoubleQuote);
case '\'':
- return EndSymbol(HtmlSymbolType.SingleQuote);
+ return EndToken(HtmlTokenType.SingleQuote);
case '-':
Debug.Assert(CurrentCharacter == '-');
TakeCurrent();
- return EndSymbol(HtmlSymbolType.DoubleHyphen);
+ return EndToken(HtmlTokenType.DoubleHyphen);
default:
- Debug.Fail("Unexpected symbol!");
- return EndSymbol(HtmlSymbolType.Unknown);
+ Debug.Fail("Unexpected token!");
+ return EndToken(HtmlTokenType.Unknown);
}
}
- private HtmlSymbol Whitespace()
+ private HtmlToken Whitespace()
{
while (ParserHelpers.IsWhitespace(CurrentCharacter))
{
TakeCurrent();
}
- return EndSymbol(HtmlSymbolType.WhiteSpace);
+ return EndToken(HtmlTokenType.WhiteSpace);
}
- private HtmlSymbol Newline()
+ private HtmlToken Newline()
{
Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter));
// CSharp Spec §2.3.1
@@ -249,10 +249,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TakeCurrent();
}
- return EndSymbol(HtmlSymbolType.NewLine);
+ return EndToken(HtmlTokenType.NewLine);
}
- private bool AtSymbol()
+ private bool AtToken()
{
return CurrentCharacter == '<' ||
CurrentCharacter == '<' ||
@@ -274,7 +274,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Transition((int)state, result: null);
}
- private StateResult Transition(HtmlTokenizerState state, HtmlSymbol result)
+ private StateResult Transition(HtmlTokenizerState state, HtmlToken result)
{
return Transition((int)state, result);
}
@@ -289,7 +289,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition,
RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody,
StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody,
- AtSymbolAfterRazorCommentBody = RazorCommentTokenizerState.AtSymbolAfterRazorCommentBody,
+ AtTokenAfterRazorCommentBody = RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ISymbol.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/IToken.cs
similarity index 91%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/ISymbol.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/IToken.cs
index 346a390375..2f8eb1214d 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ISymbol.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/IToken.cs
@@ -3,7 +3,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal interface ISymbol
+ internal interface IToken
{
Span Parent { get; set; }
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs
index 30e5186afb..530df75dbc 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/ITokenizer.cs
@@ -5,6 +5,6 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
internal interface ITokenizer
{
- ISymbol NextSymbol();
+ IToken NextToken();
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs
index a55c2a8949..b3869f00d0 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/ImplicitExpressionEditHandler.cs
@@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private readonly ISet _keywords;
private readonly IReadOnlyCollection _readOnlyKeywords;
- public ImplicitExpressionEditHandler(Func> tokenizer, ISet keywords, bool acceptTrailingDot)
+ public ImplicitExpressionEditHandler(Func> tokenizer, ISet keywords, bool acceptTrailingDot)
: base(tokenizer)
{
_keywords = keywords ?? new HashSet();
@@ -170,44 +170,44 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- for (var i = 0; i < target.Symbols.Count; i++)
+ for (var i = 0; i < target.Tokens.Count; i++)
{
- var symbol = target.Symbols[i] as CSharpSymbol;
+ var token = target.Tokens[i] as CSharpToken;
- if (symbol == null)
+ if (token == null)
{
break;
}
- var symbolStartIndex = symbol.Start.AbsoluteIndex;
- var symbolEndIndex = symbolStartIndex + symbol.Content.Length;
+ var tokenStartIndex = token.Start.AbsoluteIndex;
+ var tokenEndIndex = tokenStartIndex + token.Content.Length;
- // We're looking for the first symbol that contains the SourceChange.
- if (symbolEndIndex > change.Span.AbsoluteIndex)
+ // We're looking for the first token that contains the SourceChange.
+ if (tokenEndIndex > change.Span.AbsoluteIndex)
{
- if (symbolEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && symbol.Type == CSharpSymbolType.Identifier)
+ if (tokenEndIndex >= change.Span.AbsoluteIndex + change.Span.Length && token.Type == CSharpTokenType.Identifier)
{
- // The symbol we're changing happens to be an identifier. Need to check if its transformed state is also one.
+ // The token we're changing happens to be an identifier. Need to check if its transformed state is also one.
// We do this transformation logic to capture the case that the new text change happens to not be an identifier;
// i.e. "5". Alone, it's numeric, within an identifier it's classified as identifier.
- var transformedContent = change.GetEditedContent(symbol.Content, change.Span.AbsoluteIndex - symbolStartIndex);
- var newSymbols = Tokenizer(transformedContent);
+ var transformedContent = change.GetEditedContent(token.Content, change.Span.AbsoluteIndex - tokenStartIndex);
+ var newTokens = Tokenizer(transformedContent);
- if (newSymbols.Count() != 1)
+ if (newTokens.Count() != 1)
{
- // The transformed content resulted in more than one symbol; we can only replace a single identifier with
+ // The transformed content resulted in more than one token; we can only replace a single identifier with
// another single identifier.
break;
}
- var newSymbol = (CSharpSymbol)newSymbols.First();
- if (newSymbol.Type == CSharpSymbolType.Identifier)
+ var newToken = (CSharpToken)newTokens.First();
+ if (newToken.Type == CSharpTokenType.Identifier)
{
return true;
}
}
- // Change is touching a non-identifier symbol or spans multiple symbols.
+ // Change is touching a non-identifier token or spans multiple tokens.
break;
}
@@ -241,7 +241,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var changeStart = change.Span.AbsoluteIndex;
var changeLength = change.Span.Length;
var changeEnd = changeStart + changeLength;
- var tokens = target.Symbols.Cast().ToArray();
+ var tokens = target.Tokens.Cast().ToArray();
if (!IsInsideParenthesis(changeStart, tokens) || !IsInsideParenthesis(changeEnd, tokens))
{
// Either the start or end of the delete does not fall inside of parenthesis, unacceptable inner deletion.
@@ -274,7 +274,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- var tokens = target.Symbols.Cast().ToArray();
+ var tokens = target.Tokens.Cast().ToArray();
if (IsInsideParenthesis(change.Span.AbsoluteIndex, tokens))
{
return true;
@@ -284,7 +284,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Internal for testing
- internal static bool IsInsideParenthesis(int position, IReadOnlyList tokens)
+ internal static bool IsInsideParenthesis(int position, IReadOnlyList tokens)
{
var balanceCount = 0;
var foundInsertionPoint = false;
@@ -322,7 +322,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Internal for testing
- internal static bool ContainsPosition(int position, CSharpSymbol currentToken)
+ internal static bool ContainsPosition(int position, CSharpToken currentToken)
{
var tokenStart = currentToken.Start.AbsoluteIndex;
if (tokenStart == position)
@@ -342,14 +342,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Internal for testing
- internal static bool TryUpdateBalanceCount(CSharpSymbol token, ref int count)
+ internal static bool TryUpdateBalanceCount(CSharpToken token, ref int count)
{
var updatedCount = count;
- if (token.Type == CSharpSymbolType.LeftParenthesis)
+ if (token.Type == CSharpTokenType.LeftParenthesis)
{
updatedCount++;
}
- else if (token.Type == CSharpSymbolType.RightParenthesis)
+ else if (token.Type == CSharpTokenType.RightParenthesis)
{
if (updatedCount == 0)
{
@@ -358,7 +358,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
updatedCount--;
}
- else if (token.Type == CSharpSymbolType.StringLiteral)
+ else if (token.Type == CSharpTokenType.StringLiteral)
{
var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '"')
@@ -370,7 +370,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
}
- else if (token.Type == CSharpSymbolType.CharacterLiteral)
+ else if (token.Type == CSharpTokenType.CharacterLiteral)
{
var content = token.Content;
if (content.Length > 0 && content[content.Length - 1] != '\'')
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownSymbolType.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownTokenType.cs
similarity index 92%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownSymbolType.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownTokenType.cs
index a1ed444ac9..ca5fad7254 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownSymbolType.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/KnownTokenType.cs
@@ -3,7 +3,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal enum KnownSymbolType
+ internal enum KnownTokenType
{
WhiteSpace,
NewLine,
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs
index 23d5f037d0..b12bf612b9 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/LanguageCharacteristics.cs
@@ -6,104 +6,104 @@ using System.Collections.Generic;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal abstract class LanguageCharacteristics
- where TSymbolType : struct
- where TTokenizer : Tokenizer
- where TSymbol : SymbolBase
+ internal abstract class LanguageCharacteristics
+ where TTokenType : struct
+ where TTokenizer : Tokenizer
+ where TToken : TokenBase
{
- public abstract string GetSample(TSymbolType type);
+ public abstract string GetSample(TTokenType type);
public abstract TTokenizer CreateTokenizer(ITextDocument source);
- public abstract TSymbolType FlipBracket(TSymbolType bracket);
- public abstract TSymbol CreateMarkerSymbol();
+ public abstract TTokenType FlipBracket(TTokenType bracket);
+ public abstract TToken CreateMarkerToken();
- public virtual IEnumerable TokenizeString(string content)
+ public virtual IEnumerable TokenizeString(string content)
{
return TokenizeString(SourceLocation.Zero, content);
}
- public virtual IEnumerable TokenizeString(SourceLocation start, string input)
+ public virtual IEnumerable TokenizeString(SourceLocation start, string input)
{
using (var reader = new SeekableTextReader(input, start.FilePath))
{
var tok = CreateTokenizer(reader);
- TSymbol sym;
- while ((sym = tok.NextSymbol()) != null)
+ TToken token;
+ while ((token = tok.NextToken()) != null)
{
- yield return sym;
+ yield return token;
}
}
}
- public virtual bool IsWhiteSpace(TSymbol symbol)
+ public virtual bool IsWhiteSpace(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.WhiteSpace);
+ return IsKnownTokenType(token, KnownTokenType.WhiteSpace);
}
- public virtual bool IsNewLine(TSymbol symbol)
+ public virtual bool IsNewLine(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.NewLine);
+ return IsKnownTokenType(token, KnownTokenType.NewLine);
}
- public virtual bool IsIdentifier(TSymbol symbol)
+ public virtual bool IsIdentifier(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.Identifier);
+ return IsKnownTokenType(token, KnownTokenType.Identifier);
}
- public virtual bool IsKeyword(TSymbol symbol)
+ public virtual bool IsKeyword(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.Keyword);
+ return IsKnownTokenType(token, KnownTokenType.Keyword);
}
- public virtual bool IsTransition(TSymbol symbol)
+ public virtual bool IsTransition(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.Transition);
+ return IsKnownTokenType(token, KnownTokenType.Transition);
}
- public virtual bool IsCommentStart(TSymbol symbol)
+ public virtual bool IsCommentStart(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.CommentStart);
+ return IsKnownTokenType(token, KnownTokenType.CommentStart);
}
- public virtual bool IsCommentStar(TSymbol symbol)
+ public virtual bool IsCommentStar(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.CommentStar);
+ return IsKnownTokenType(token, KnownTokenType.CommentStar);
}
- public virtual bool IsCommentBody(TSymbol symbol)
+ public virtual bool IsCommentBody(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.CommentBody);
+ return IsKnownTokenType(token, KnownTokenType.CommentBody);
}
- public virtual bool IsUnknown(TSymbol symbol)
+ public virtual bool IsUnknown(TToken token)
{
- return IsKnownSymbolType(symbol, KnownSymbolType.Unknown);
+ return IsKnownTokenType(token, KnownTokenType.Unknown);
}
- public virtual bool IsKnownSymbolType(TSymbol symbol, KnownSymbolType type)
+ public virtual bool IsKnownTokenType(TToken token, KnownTokenType type)
{
- return symbol != null && Equals(symbol.Type, GetKnownSymbolType(type));
+ return token != null && Equals(token.Type, GetKnownTokenType(type));
}
- public virtual Tuple SplitSymbol(TSymbol symbol, int splitAt, TSymbolType leftType)
+ public virtual Tuple SplitToken(TToken token, int splitAt, TTokenType leftType)
{
- var left = CreateSymbol(symbol.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray);
+ var left = CreateToken(token.Content.Substring(0, splitAt), leftType, RazorDiagnostic.EmptyArray);
- TSymbol right = null;
- if (splitAt < symbol.Content.Length)
+ TToken right = null;
+ if (splitAt < token.Content.Length)
{
- right = CreateSymbol(symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors);
+ right = CreateToken(token.Content.Substring(splitAt), token.Type, token.Errors);
}
return Tuple.Create(left, right);
}
- public abstract TSymbolType GetKnownSymbolType(KnownSymbolType type);
+ public abstract TTokenType GetKnownTokenType(KnownTokenType type);
- public virtual bool KnowsSymbolType(KnownSymbolType type)
+ public virtual bool KnowsTokenType(KnownTokenType type)
{
- return type == KnownSymbolType.Unknown || !Equals(GetKnownSymbolType(type), GetKnownSymbolType(KnownSymbolType.Unknown));
+ return type == KnownTokenType.Unknown || !Equals(GetKnownTokenType(type), GetKnownTokenType(KnownTokenType.Unknown));
}
- protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList errors);
+ protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList errors);
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs
index 6ce51f273f..42074050ff 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/MarkupRewriter.cs
@@ -67,7 +67,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
builder.Kind = SpanKindInternal.Markup;
builder.ChunkGenerator = new MarkupChunkGenerator();
- foreach (ISymbol sym in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content))
+ foreach (IToken sym in HtmlLanguageCharacteristics.Instance.TokenizeString(start, content))
{
builder.Accept(sym);
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs
index 9b7366fc5e..fbf9f9f2c2 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/Span.cs
@@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public ISpanChunkGenerator ChunkGenerator { get; private set; }
public SpanKindInternal Kind { get; private set; }
- public IReadOnlyList Symbols { get; private set; }
+ public IReadOnlyList Tokens { get; private set; }
// Allow test code to re-link spans
public Span Previous { get; internal set; }
@@ -42,9 +42,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var length = 0;
if (_content == null)
{
- for (var i = 0; i < Symbols.Count; i++)
+ for (var i = 0; i < Tokens.Count; i++)
{
- length += Symbols[i].Content.Length;
+ length += Tokens[i].Content.Length;
}
}
else
@@ -67,19 +67,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
if (_content == null)
{
- var symbolCount = Symbols.Count;
- if (symbolCount == 1)
+ var tokenCount = Tokens.Count;
+ if (tokenCount == 1)
{
// Perf: no StringBuilder allocation if not necessary
- _content = Symbols[0].Content;
+ _content = Tokens[0].Content;
}
else
{
var builder = new StringBuilder();
- for (var i = 0; i < symbolCount; i++)
+ for (var i = 0; i < tokenCount; i++)
{
- var symbol = Symbols[i];
- builder.Append(symbol.Content);
+ var token = Tokens[i];
+ builder.Append(token.Content);
}
_content = builder.ToString();
@@ -93,11 +93,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void ReplaceWith(SpanBuilder builder)
{
Kind = builder.Kind;
- Symbols = builder.Symbols;
+ Tokens = builder.Tokens;
- for (var i = 0; i Gen: <");
builder.Append(ChunkGenerator.ToString());
builder.Append("> {");
- builder.Append(string.Join(";", Symbols.GroupBy(sym => sym.GetType()).Select(grp => string.Concat(grp.Key.Name, ":", grp.Count()))));
+ builder.Append(string.Join(";", Tokens.GroupBy(sym => sym.GetType()).Select(grp => string.Concat(grp.Key.Name, ":", grp.Count()))));
builder.Append("}");
return builder.ToString();
}
@@ -166,7 +166,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind.Equals(other.Kind) &&
EditHandler.Equals(other.EditHandler) &&
ChunkGenerator.Equals(other.ChunkGenerator) &&
- Symbols.SequenceEqual(other.Symbols);
+ Tokens.SequenceEqual(other.Tokens);
}
public override int GetHashCode()
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs
index b9b9e2df54..6d2e647b91 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanBuilder.cs
@@ -10,7 +10,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
internal class SpanBuilder
{
private SourceLocation _start;
- private List _symbols;
+ private List _tokens;
private SourceLocationTracker _tracker;
public SpanBuilder(Span original)
@@ -20,7 +20,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
_start = original.Start;
ChunkGenerator = original.ChunkGenerator;
- _symbols = new List(original.Symbols);
+ _tokens = new List(original.Tokens);
_tracker = new SourceLocationTracker(original.Start);
}
@@ -49,16 +49,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SpanKindInternal Kind { get; set; }
- public IReadOnlyList Symbols
+ public IReadOnlyList Tokens
{
get
{
- if (_symbols == null)
+ if (_tokens == null)
{
- _symbols = new List();
+ _tokens = new List();
}
- return _symbols;
+ return _tokens;
}
}
@@ -68,10 +68,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Need to potentially allocate a new list because Span.ReplaceWith takes ownership
// of the original list.
- _symbols = null;
- _symbols = new List();
+ _tokens = null;
+ _tokens = new List();
- EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty());
+ EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty());
ChunkGenerator = SpanChunkGenerator.Null;
Start = SourceLocation.Undefined;
}
@@ -80,23 +80,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
var span = new Span(this);
- for (var i = 0; i < span.Symbols.Count; i++)
+ for (var i = 0; i < span.Tokens.Count; i++)
{
- var symbol = span.Symbols[i];
- symbol.Parent = span;
+ var token = span.Tokens[i];
+ token.Parent = span;
}
return span;
}
- public void ClearSymbols()
+ public void ClearTokens()
{
- _symbols?.Clear();
+ _tokens?.Clear();
}
- public void Accept(ISymbol symbol)
+ public void Accept(IToken token)
{
- if (symbol == null)
+ if (token == null)
{
return;
}
@@ -106,8 +106,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
throw new InvalidOperationException("SpanBuilder must have a valid location");
}
- _symbols.Add(symbol);
- _tracker.UpdateLocation(symbol.Content);
+ _tokens.Add(token);
+ _tracker.UpdateLocation(token.Content);
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs
index dd9511ce70..d5407f34dd 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanEditHandler.cs
@@ -11,12 +11,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
private static readonly int TypeHashCode = typeof(SpanEditHandler).GetHashCode();
- public SpanEditHandler(Func> tokenizer)
+ public SpanEditHandler(Func> tokenizer)
: this(tokenizer, AcceptedCharactersInternal.Any)
{
}
- public SpanEditHandler(Func> tokenizer, AcceptedCharactersInternal accepted)
+ public SpanEditHandler(Func> tokenizer, AcceptedCharactersInternal accepted)
{
AcceptedCharacters = accepted;
Tokenizer = tokenizer;
@@ -24,9 +24,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public AcceptedCharactersInternal AcceptedCharacters { get; set; }
- public Func> Tokenizer { get; set; }
+ public Func> Tokenizer { get; set; }
- public static SpanEditHandler CreateDefault(Func> tokenizer)
+ public static SpanEditHandler CreateDefault(Func> tokenizer)
{
return new SpanEditHandler(tokenizer);
}
@@ -69,7 +69,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
var newContent = change.GetEditedContent(target);
var newSpan = new SpanBuilder(target);
- newSpan.ClearSymbols();
+ newSpan.ClearTokens();
foreach (var token in Tokenizer(newContent))
{
newSpan.Accept(token);
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SyntaxTreeNode.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/SyntaxTreeNode.cs
index a20c7c318b..1b7b192db9 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SyntaxTreeNode.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/SyntaxTreeNode.cs
@@ -28,7 +28,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// The node to compare this node with
///
/// true if the provided node has all the same content and metadata, though the specific quantity and type of
- /// symbols may be different.
+ /// tokens may be different.
///
public abstract bool EquivalentTo(SyntaxTreeNode node);
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs
index d3a58ac598..f08d0579e5 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TagHelperBlockRewriter.cs
@@ -42,8 +42,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// We skip the first child "" or "/>".
// The -2 accounts for both the start and end tags. If the tag does not have a valid structure then there's
// no end tag to ignore.
- var symbolOffset = validStructure ? 2 : 1;
- var attributeChildren = tagBlock.Children.Skip(1).Take(tagBlock.Children.Count() - symbolOffset);
+ var tokenOffset = validStructure ? 2 : 1;
+ var attributeChildren = tagBlock.Children.Skip(1).Take(tagBlock.Children.Count() - tokenOffset);
var processedBoundAttributeNames = new HashSet(StringComparer.OrdinalIgnoreCase);
foreach (var child in attributeChildren)
@@ -149,8 +149,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Kind = span.Kind
};
- // Will contain symbols that represent a single attribute value:
- var htmlSymbols = span.Symbols.OfType().ToArray();
+ // Will contain tokens that represent a single attribute value:
+ var htmlTokens = span.Tokens.OfType().ToArray();
var capturedAttributeValueStart = false;
var attributeValueStartLocation = span.Start;
@@ -158,53 +158,53 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// TryParseBlock() variation of attribute parsing.
var attributeValueStyle = AttributeStructure.DoubleQuotes;
- // The symbolOffset is initialized to 0 to expect worst case: "class=". If a quote is found later on for
- // the attribute value the symbolOffset is adjusted accordingly.
- var symbolOffset = 0;
+ // The tokenOffset is initialized to 0 to expect worst case: "class=". If a quote is found later on for
+ // the attribute value the tokenOffset is adjusted accordingly.
+ var tokenOffset = 0;
string name = null;
- // Iterate down through the symbols to find the name and the start of the value.
- // We subtract the symbolOffset so we don't accept an ending quote of a span.
- for (var i = 0; i < htmlSymbols.Length - symbolOffset; i++)
+ // Iterate down through the tokens to find the name and the start of the value.
+ // We subtract the tokenOffset so we don't accept an ending quote of a span.
+ for (var i = 0; i < htmlTokens.Length - tokenOffset; i++)
{
- var symbol = htmlSymbols[i];
+ var token = htmlTokens[i];
if (afterEquals)
{
// We've captured all leading whitespace, the attribute name, and an equals with an optional
// quote/double quote. We're now at: " asp-for='|...'" or " asp-for=|..."
- // The goal here is to capture all symbols until the end of the attribute. Note this will not
- // consume an ending quote due to the symbolOffset.
+ // The goal here is to capture all tokens until the end of the attribute. Note this will not
+ // consume an ending quote due to the tokenOffset.
- // When symbols are accepted into SpanBuilders, their locations get altered to be offset by the
- // parent which is why we need to mark our start location prior to adding the symbol.
+ // When tokens are accepted into SpanBuilders, their locations get altered to be offset by the
+ // parent which is why we need to mark our start location prior to adding the token.
// This is needed to know the location of the attribute value start within the document.
if (!capturedAttributeValueStart)
{
capturedAttributeValueStart = true;
- attributeValueStartLocation = symbol.Start;
+ attributeValueStartLocation = token.Start;
}
- builder.Accept(symbol);
+ builder.Accept(token);
}
- else if (name == null && HtmlMarkupParser.IsValidAttributeNameSymbol(symbol))
+ else if (name == null && HtmlMarkupParser.IsValidAttributeNameToken(token))
{
// We've captured all leading whitespace prior to the attribute name.
// We're now at: " |asp-for='...'" or " |asp-for=..."
// The goal here is to capture the attribute name.
var nameBuilder = new StringBuilder();
- // Move the indexer past the attribute name symbols.
- for (var j = i; j < htmlSymbols.Length; j++)
+ // Move the indexer past the attribute name tokens.
+ for (var j = i; j < htmlTokens.Length; j++)
{
- var nameSymbol = htmlSymbols[j];
- if (!HtmlMarkupParser.IsValidAttributeNameSymbol(nameSymbol))
+ var nameToken = htmlTokens[j];
+ if (!HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
{
break;
}
- nameBuilder.Append(nameSymbol.Content);
+ nameBuilder.Append(nameToken.Content);
i++;
}
@@ -213,67 +213,67 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
name = nameBuilder.ToString();
attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, name);
}
- else if (symbol.Type == HtmlSymbolType.Equals)
+ else if (token.Type == HtmlTokenType.Equals)
{
// We've captured all leading whitespace and the attribute name.
// We're now at: " asp-for|='...'" or " asp-for|=..."
// The goal here is to consume the equal sign and the optional single/double-quote.
- // The coming symbols will either be a quote or value (in the case that the value is unquoted).
+ // The coming tokens will either be a quote or value (in the case that the value is unquoted).
- SourceLocation symbolStartLocation;
+ SourceLocation tokenStartLocation;
// Skip the whitespace preceding the start of the attribute value.
do
{
- i++; // Start from the symbol after '='.
- } while (i < htmlSymbols.Length &&
- (htmlSymbols[i].Type == HtmlSymbolType.WhiteSpace ||
- htmlSymbols[i].Type == HtmlSymbolType.NewLine));
+ i++; // Start from the token after '='.
+ } while (i < htmlTokens.Length &&
+ (htmlTokens[i].Type == HtmlTokenType.WhiteSpace ||
+ htmlTokens[i].Type == HtmlTokenType.NewLine));
// Check for attribute start values, aka single or double quote
- if (i < htmlSymbols.Length && IsQuote(htmlSymbols[i]))
+ if (i < htmlTokens.Length && IsQuote(htmlTokens[i]))
{
- if (htmlSymbols[i].Type == HtmlSymbolType.SingleQuote)
+ if (htmlTokens[i].Type == HtmlTokenType.SingleQuote)
{
attributeValueStyle = AttributeStructure.SingleQuotes;
}
- symbolStartLocation = htmlSymbols[i].Start;
+ tokenStartLocation = htmlTokens[i].Start;
// If there's a start quote then there must be an end quote to be valid, skip it.
- symbolOffset = 1;
+ tokenOffset = 1;
}
else
{
- // We are at the symbol after equals. Go back to equals to ensure we don't skip past that symbol.
+ // We are at the token after equals. Go back to equals to ensure we don't skip past that token.
i--;
- symbolStartLocation = symbol.Start;
+ tokenStartLocation = token.Start;
}
attributeValueStartLocation = new SourceLocation(
- symbolStartLocation.FilePath,
- symbolStartLocation.AbsoluteIndex + 1,
- symbolStartLocation.LineIndex,
- symbolStartLocation.CharacterIndex + 1);
+ tokenStartLocation.FilePath,
+ tokenStartLocation.AbsoluteIndex + 1,
+ tokenStartLocation.LineIndex,
+ tokenStartLocation.CharacterIndex + 1);
afterEquals = true;
}
- else if (symbol.Type == HtmlSymbolType.WhiteSpace)
+ else if (token.Type == HtmlTokenType.WhiteSpace)
{
// We're at the start of the attribute, this branch may be hit on the first iterations of
- // the loop since the parser separates attributes with their spaces included as symbols.
+ // the loop since the parser separates attributes with their spaces included as tokens.
// We're at: "| asp-for='...'" or "| asp-for=..."
// Note: This will not be hit even for situations like asp-for ="..." because the core Razor
// parser currently does not know how to handle attributes in that format. This will be addressed
// by https://github.com/aspnet/Razor/issues/123.
- attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, symbol.Content);
+ attributeValueStartLocation = SourceLocationTracker.Advance(attributeValueStartLocation, token.Content);
}
}
- // After all symbols have been added we need to set the builders start position so we do not indirectly
+ // After all tokens have been added we need to set the builders start position so we do not indirectly
// modify the span's start location.
builder.Start = attributeValueStartLocation;
@@ -341,14 +341,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return TryParseSpan(childSpan, descriptors, errorSink, processedBoundAttributeNames);
}
- var nameSymbols = childSpan
- .Symbols
- .OfType()
- .SkipWhile(symbol => !HtmlMarkupParser.IsValidAttributeNameSymbol(symbol)) // Skip prefix
- .TakeWhile(nameSymbol => HtmlMarkupParser.IsValidAttributeNameSymbol(nameSymbol))
- .Select(nameSymbol => nameSymbol.Content);
+ var nameTokens = childSpan
+ .Tokens
+ .OfType()
+ .SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token)) // Skip prefix
+ .TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
+ .Select(nameToken => nameToken.Content);
- var name = string.Concat(nameSymbols);
+ var name = string.Concat(nameTokens);
if (string.IsNullOrEmpty(name))
{
var location = new SourceSpan(childSpan.Start, childSpan.Length);
@@ -362,12 +362,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames);
var firstChild = builder.Children[0] as Span;
- if (firstChild != null && firstChild.Symbols[0] is HtmlSymbol)
+ if (firstChild != null && firstChild.Tokens[0] is HtmlToken)
{
- var htmlSymbol = firstChild.Symbols[firstChild.Symbols.Count - 1] as HtmlSymbol;
- switch (htmlSymbol.Type)
+ var htmlToken = firstChild.Tokens[firstChild.Tokens.Count - 1] as HtmlToken;
+ switch (htmlToken.Type)
{
- case HtmlSymbolType.Equals:
+ case HtmlTokenType.Equals:
if (builder.Children.Count == 2 &&
builder.Children[1] is Span value &&
value.Kind == SpanKindInternal.Markup)
@@ -385,10 +385,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
result.AttributeStructure = AttributeStructure.DoubleQuotes;
}
break;
- case HtmlSymbolType.DoubleQuote:
+ case HtmlTokenType.DoubleQuote:
result.AttributeStructure = AttributeStructure.DoubleQuotes;
break;
- case HtmlSymbolType.SingleQuote:
+ case HtmlTokenType.SingleQuote:
result.AttributeStructure = AttributeStructure.SingleQuotes;
break;
default:
@@ -407,12 +407,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var endSpan = (Span)endNode;
// In some malformed cases e.g.
0)
+ if (childSpan.Tokens.Count > 0)
{
- var potentialQuote = childSpan.Symbols[childSpan.Symbols.Count - 1] as HtmlSymbol;
+ var potentialQuote = childSpan.Tokens[childSpan.Tokens.Count - 1] as HtmlToken;
if (potentialQuote != null &&
- (potentialQuote.Type == HtmlSymbolType.DoubleQuote ||
- potentialQuote.Type == HtmlSymbolType.SingleQuote))
+ (potentialQuote.Type == HtmlTokenType.DoubleQuote ||
+ potentialQuote.Type == HtmlTokenType.SingleQuote))
{
childOffset = 1;
}
@@ -391,9 +391,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
else
{
var valueChildSpan = (Span)valueChild;
- for (var k = 0; k < valueChildSpan.Symbols.Count; k++)
+ for (var k = 0; k < valueChildSpan.Tokens.Count; k++)
{
- _attributeValueBuilder.Append(valueChildSpan.Symbols[k].Content);
+ _attributeValueBuilder.Append(valueChildSpan.Tokens[k].Content);
}
}
}
@@ -404,43 +404,43 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var afterEquals = false;
var atValue = false;
- var endValueMarker = childSpan.Symbols.Count;
+ var endValueMarker = childSpan.Tokens.Count;
// Entire attribute is a string
for (var j = 0; j < endValueMarker; j++)
{
- var htmlSymbol = (HtmlSymbol)childSpan.Symbols[j];
+ var htmlToken = (HtmlToken)childSpan.Tokens[j];
if (!afterEquals)
{
- afterEquals = htmlSymbol.Type == HtmlSymbolType.Equals;
+ afterEquals = htmlToken.Type == HtmlTokenType.Equals;
continue;
}
if (!atValue)
{
- atValue = htmlSymbol.Type != HtmlSymbolType.WhiteSpace &&
- htmlSymbol.Type != HtmlSymbolType.NewLine;
+ atValue = htmlToken.Type != HtmlTokenType.WhiteSpace &&
+ htmlToken.Type != HtmlTokenType.NewLine;
if (atValue)
{
- if (htmlSymbol.Type == HtmlSymbolType.DoubleQuote ||
- htmlSymbol.Type == HtmlSymbolType.SingleQuote)
+ if (htmlToken.Type == HtmlTokenType.DoubleQuote ||
+ htmlToken.Type == HtmlTokenType.SingleQuote)
{
endValueMarker--;
}
else
{
- // Current symbol is considered the value (unquoted). Add its content to the
+ // Current token is considered the value (unquoted). Add its content to the
// attribute value builder before we move past it.
- _attributeValueBuilder.Append(htmlSymbol.Content);
+ _attributeValueBuilder.Append(htmlToken.Content);
}
}
continue;
}
- _attributeValueBuilder.Append(htmlSymbol.Content);
+ _attributeValueBuilder.Append(htmlToken.Content);
}
}
@@ -642,11 +642,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// If our tag end is not a markup span it means it's some sort of code SyntaxTreeNode (not a valid format)
if (tagEnd != null && tagEnd.Kind == SpanKindInternal.Markup)
{
- var endSymbol = tagEnd.Symbols.Count > 0 ?
- tagEnd.Symbols[tagEnd.Symbols.Count - 1] as HtmlSymbol :
+ var endToken = tagEnd.Tokens.Count > 0 ?
+ tagEnd.Tokens[tagEnd.Tokens.Count - 1] as HtmlToken :
null;
- if (endSymbol != null && endSymbol.Type == HtmlSymbolType.CloseAngle)
+ if (endToken != null && endToken.Type == HtmlTokenType.CloseAngle)
{
return false;
}
@@ -793,25 +793,25 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
var childSpan = (Span)child;
- HtmlSymbol textSymbol = null;
- for (var i = 0; i < childSpan.Symbols.Count; i++)
+ HtmlToken textToken = null;
+ for (var i = 0; i < childSpan.Tokens.Count; i++)
{
- var symbol = childSpan.Symbols[i] as HtmlSymbol;
+ var token = childSpan.Tokens[i] as HtmlToken;
- if (symbol != null &&
- (symbol.Type & (HtmlSymbolType.WhiteSpace | HtmlSymbolType.Text)) == symbol.Type)
+ if (token != null &&
+ (token.Type & (HtmlTokenType.WhiteSpace | HtmlTokenType.Text)) == token.Type)
{
- textSymbol = symbol;
+ textToken = token;
break;
}
}
- if (textSymbol == null)
+ if (textToken == null)
{
return null;
}
- return textSymbol.Type == HtmlSymbolType.WhiteSpace ? null : textSymbol.Content;
+ return textToken.Type == HtmlTokenType.WhiteSpace ? null : textToken.Content;
}
private static bool IsEndTag(Block tagBlock)
@@ -820,10 +820,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var childSpan = (Span)tagBlock.Children.First();
- // We grab the symbol that could be forward slash
- var relevantSymbol = (HtmlSymbol)childSpan.Symbols[childSpan.Symbols.Count == 1 ? 0 : 1];
+ // We grab the token that could be forward slash
+ var relevantToken = (HtmlToken)childSpan.Tokens[childSpan.Tokens.Count == 1 ? 0 : 1];
- return relevantSymbol.Type == HtmlSymbolType.ForwardSlash;
+ return relevantToken.Type == HtmlTokenType.ForwardSlash;
}
internal static bool IsComment(Span span)
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SymbolBase.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenBase.cs
similarity index 83%
rename from src/Microsoft.AspNetCore.Razor.Language/Legacy/SymbolBase.cs
rename to src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenBase.cs
index c1caebb988..562a41f5bc 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/SymbolBase.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenBase.cs
@@ -8,9 +8,9 @@ using Microsoft.Extensions.Internal;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal abstract class SymbolBase : ISymbol where TType : struct
+ internal abstract class TokenBase : IToken where TType : struct
{
- protected SymbolBase(
+ protected TokenBase(
string content,
TType type,
IReadOnlyList errors)
@@ -43,15 +43,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
var tracker = new SourceLocationTracker(Parent.Start);
- for (var i = 0; i < Parent.Symbols.Count; i++)
+ for (var i = 0; i < Parent.Tokens.Count; i++)
{
- var symbol = Parent.Symbols[i];
- if (object.ReferenceEquals(this, symbol))
+ var token = Parent.Tokens[i];
+ if (object.ReferenceEquals(this, token))
{
break;
}
- tracker.UpdateLocation(symbol.Content);
+ tracker.UpdateLocation(token.Content);
}
return tracker.CurrentLocation;
@@ -60,7 +60,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override bool Equals(object obj)
{
- var other = obj as SymbolBase;
+ var other = obj as TokenBase;
return other != null &&
string.Equals(Content, other.Content, StringComparison.Ordinal) &&
Type.Equals(other.Type);
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs
index 465754f6d5..af77d2ddc6 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/Tokenizer.cs
@@ -8,9 +8,9 @@ using System.Text;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal abstract partial class Tokenizer : ITokenizer
- where TSymbolType : struct
- where TSymbol : SymbolBase
+ internal abstract partial class Tokenizer : ITokenizer
+ where TTokenType : struct
+ where TToken : TokenBase
{
protected Tokenizer(ITextDocument source)
{
@@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Source = source;
Buffer = new StringBuilder();
CurrentErrors = new List();
- StartSymbol();
+ StartToken();
}
protected List CurrentErrors { get; }
@@ -31,7 +31,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected int? CurrentState { get; set; }
- protected TSymbol CurrentSymbol { get; private set; }
+ protected TToken CurrentToken { get; private set; }
public ITextDocument Source { get; private set; }
@@ -42,9 +42,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return Source.Peek() == -1; }
}
- public abstract TSymbolType RazorCommentStarType { get; }
- public abstract TSymbolType RazorCommentType { get; }
- public abstract TSymbolType RazorCommentTransitionType { get; }
+ public abstract TTokenType RazorCommentStarType { get; }
+ public abstract TTokenType RazorCommentType { get; }
+ public abstract TTokenType RazorCommentTransitionType { get; }
protected bool HaveContent
{
@@ -64,27 +64,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SourceLocation CurrentStart { get; private set; }
- protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList errors);
+ protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList errors);
protected abstract StateResult Dispatch();
- ISymbol ITokenizer.NextSymbol()
+ IToken ITokenizer.NextToken()
{
- return NextSymbol();
+ return NextToken();
}
- public virtual TSymbol NextSymbol()
+ public virtual TToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
- StartSymbol();
+ StartToken();
if (EndOfFile)
{
return null;
}
- var symbol = Turn();
+ var token = Turn();
// Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0);
@@ -92,10 +92,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Post-Condition: Token should be non-zero length unless we're at EOF.
Debug.Assert(EndOfFile || !CurrentStart.Equals(CurrentLocation));
- return symbol;
+ return token;
}
- protected virtual TSymbol Turn()
+ protected virtual TToken Turn()
{
if (CurrentState != null)
{
@@ -105,19 +105,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var next = Dispatch();
CurrentState = next.State;
- CurrentSymbol = next.Result;
+ CurrentToken = next.Result;
}
- while (CurrentState != null && CurrentSymbol == null);
+ while (CurrentState != null && CurrentToken == null);
if (CurrentState == null)
{
- return default(TSymbol); // Terminated
+ return default(TToken); // Terminated
}
- return CurrentSymbol;
+ return CurrentToken;
}
- return default(TSymbol);
+ return default(TToken);
}
public void Reset()
@@ -149,7 +149,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to
/// should invoke the provided state.
///
- protected StateResult Transition(int state, TSymbol result)
+ protected StateResult Transition(int state, TToken result)
{
return new StateResult(state, result);
}
@@ -159,7 +159,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new StateResult((int)state, result: null);
}
- protected StateResult Transition(RazorCommentTokenizerState state, TSymbol result)
+ protected StateResult Transition(RazorCommentTokenizerState state, TToken result)
{
return new StateResult((int)state, result);
}
@@ -180,18 +180,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to
/// should re-invoke the current state.
///
- protected StateResult Stay(TSymbol result)
+ protected StateResult Stay(TToken result)
{
return new StateResult(CurrentState, result);
}
- protected TSymbol Single(TSymbolType type)
+ protected TToken Single(TTokenType type)
{
TakeCurrent();
- return EndSymbol(type);
+ return EndToken(type);
}
- protected void StartSymbol()
+ protected void StartToken()
{
Debug.Assert(Buffer.Length == 0);
Debug.Assert(CurrentErrors.Count == 0);
@@ -199,9 +199,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
CurrentStart = CurrentLocation;
}
- protected TSymbol EndSymbol(TSymbolType type)
+ protected TToken EndToken(TTokenType type)
{
- TSymbol symbol = null;
+ TToken token = null;
if (HaveContent)
{
// Perf: Don't allocate a new errors array unless necessary.
@@ -211,18 +211,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
errors[i] = CurrentErrors[i];
}
- var symbolContent = GetSymbolContent(type);
- Debug.Assert(string.Equals(symbolContent, Buffer.ToString(), StringComparison.Ordinal));
- symbol = CreateSymbol(symbolContent, type, errors);
+ var tokenContent = GetTokenContent(type);
+ Debug.Assert(string.Equals(tokenContent, Buffer.ToString(), StringComparison.Ordinal));
+ token = CreateToken(tokenContent, type, errors);
Buffer.Clear();
CurrentErrors.Clear();
}
- return symbol;
+ return token;
}
- protected virtual string GetSymbolContent(TSymbolType type)
+ protected virtual string GetTokenContent(TTokenType type)
{
return Buffer.ToString();
}
@@ -272,13 +272,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
if (CurrentCharacter != '*')
{
- // We've been moved since last time we were asked for a symbol... reset the state
+ // We've been moved since last time we were asked for a token... reset the state
return Transition(StartState);
}
AssertCurrent('*');
TakeCurrent();
- return Transition(1002, EndSymbol(RazorCommentStarType));
+ return Transition(1002, EndToken(RazorCommentStarType));
}
protected StateResult RazorCommentBody()
@@ -292,7 +292,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
RazorCommentTokenizerState.StarAfterRazorCommentBody,
- EndSymbol(RazorCommentType));
+ EndToken(RazorCommentType));
}
else
{
@@ -306,7 +306,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- return Transition(StartState, EndSymbol(RazorCommentType));
+ return Transition(StartState, EndToken(RazorCommentType));
}
protected StateResult StarAfterRazorCommentBody()
@@ -314,15 +314,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AssertCurrent('*');
TakeCurrent();
return Transition(
- RazorCommentTokenizerState.AtSymbolAfterRazorCommentBody,
- EndSymbol(RazorCommentStarType));
+ RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
+ EndToken(RazorCommentStarType));
}
- protected StateResult AtSymbolAfterRazorCommentBody()
+ protected StateResult AtTokenAfterRazorCommentBody()
{
AssertCurrent('@');
TakeCurrent();
- return Transition(StartState, EndSymbol(RazorCommentTransitionType));
+ return Transition(StartState, EndToken(RazorCommentTransitionType));
}
///
@@ -392,12 +392,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
EscapedRazorCommentTransition,
RazorCommentBody,
StarAfterRazorCommentBody,
- AtSymbolAfterRazorCommentBody,
+ AtTokenAfterRazorCommentBody,
}
protected struct StateResult
{
- public StateResult(int? state, TSymbol result)
+ public StateResult(int? state, TToken result)
{
State = state;
Result = result;
@@ -405,7 +405,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public int? State { get; }
- public TSymbol Result { get; }
+ public TToken Result { get; }
}
private static LookaheadToken BeginLookahead(ITextBuffer buffer)
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
index 01fa0b11e1..af5a08831f 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
@@ -8,20 +8,20 @@ using System.Linq;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal abstract partial class TokenizerBackedParser : ParserBase
- where TSymbolType : struct
- where TTokenizer : Tokenizer
- where TSymbol : SymbolBase
+ internal abstract partial class TokenizerBackedParser : ParserBase
+ where TTokenType : struct
+ where TTokenizer : Tokenizer
+ where TToken : TokenBase
{
- private readonly TokenizerView _tokenizer;
+ private readonly TokenizerView _tokenizer;
- protected TokenizerBackedParser(LanguageCharacteristics language, ParserContext context)
+ protected TokenizerBackedParser(LanguageCharacteristics language, ParserContext context)
: base(context)
{
Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source);
- _tokenizer = new TokenizerView(languageTokenizer);
+ _tokenizer = new TokenizerView(languageTokenizer);
Span = new SpanBuilder(CurrentLocation);
}
@@ -29,12 +29,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected Action SpanConfig { get; set; }
- protected TSymbol CurrentSymbol
+ protected TToken CurrentToken
{
get { return _tokenizer.Current; }
}
- protected TSymbol PreviousSymbol { get; private set; }
+ protected TToken PreviousToken { get; private set; }
protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
@@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _tokenizer.EndOfFile; }
}
- protected LanguageCharacteristics Language { get; }
+ protected LanguageCharacteristics Language { get; }
protected virtual void HandleEmbeddedTransition()
{
@@ -58,7 +58,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override void BuildSpan(SpanBuilder span, SourceLocation start, string content)
{
- foreach (ISymbol sym in Language.TokenizeString(start, content))
+ foreach (IToken sym in Language.TokenizeString(start, content))
{
span.Accept(sym);
}
@@ -72,7 +72,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- protected TSymbol Lookahead(int count)
+ protected TToken Lookahead(int count)
{
if (count < 0)
{
@@ -80,42 +80,42 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else if (count == 0)
{
- return CurrentSymbol;
+ return CurrentToken;
}
- // We add 1 in order to store the current symbol.
- var symbols = new TSymbol[count + 1];
- var currentSymbol = CurrentSymbol;
+ // We add 1 in order to store the current token.
+ var tokens = new TToken[count + 1];
+ var currentToken = CurrentToken;
- symbols[0] = currentSymbol;
+ tokens[0] = currentToken;
// We need to look forward "count" many times.
for (var i = 1; i <= count; i++)
{
NextToken();
- symbols[i] = CurrentSymbol;
+ tokens[i] = CurrentToken;
}
// Restore Tokenizer's location to where it was pointing before the look-ahead.
for (var i = count; i >= 0; i--)
{
- PutBack(symbols[i]);
+ PutBack(tokens[i]);
}
- // The PutBacks above will set CurrentSymbol to null. EnsureCurrent will set our CurrentSymbol to the
- // next symbol.
+ // The PutBacks above will set CurrentToken to null. EnsureCurrent will set our CurrentToken to the
+ // next token.
EnsureCurrent();
- return symbols[count];
+ return tokens[count];
}
///
/// Looks forward until the specified condition is met.
///
- /// A predicate accepting the symbol being evaluated and the list of symbols which have been looped through.
- /// true, if the condition was met. false - if the condition wasn't met and the last symbol has already been processed.
- /// The list of previous symbols is passed in the reverse order. So the last processed element will be the first one in the list.
- protected bool LookaheadUntil(Func, bool> condition)
+ /// A predicate accepting the token being evaluated and the list of tokens which have been looped through.
+ /// true, if the condition was met. false - if the condition wasn't met and the last token has already been processed.
+ /// The list of previous tokens is passed in the reverse order. So the last processed element will be the first one in the list.
+ protected bool LookaheadUntil(Func, bool> condition)
{
if (condition == null)
{
@@ -124,8 +124,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var matchFound = false;
- var symbols = new List();
- symbols.Add(CurrentSymbol);
+ var tokens = new List();
+ tokens.Add(CurrentToken);
while (true)
{
@@ -134,8 +134,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
break;
}
- symbols.Add(CurrentSymbol);
- if (condition(CurrentSymbol, symbols))
+ tokens.Add(CurrentToken);
+ if (condition(CurrentToken, tokens))
{
matchFound = true;
break;
@@ -143,13 +143,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// Restore Tokenizer's location to where it was pointing before the look-ahead.
- for (var i = symbols.Count - 1; i >= 0; i--)
+ for (var i = tokens.Count - 1; i >= 0; i--)
{
- PutBack(symbols[i]);
+ PutBack(tokens[i]);
}
- // The PutBacks above will set CurrentSymbol to null. EnsureCurrent will set our CurrentSymbol to the
- // next symbol.
+ // The PutBacks above will set CurrentToken to null. EnsureCurrent will set our CurrentToken to the
+ // next token.
EnsureCurrent();
return matchFound;
@@ -157,57 +157,57 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected internal bool NextToken()
{
- PreviousSymbol = CurrentSymbol;
+ PreviousToken = CurrentToken;
return _tokenizer.Next();
}
// Helpers
[Conditional("DEBUG")]
- internal void Assert(TSymbolType expectedType)
+ internal void Assert(TTokenType expectedType)
{
- Debug.Assert(!EndOfFile && SymbolTypeEquals(CurrentSymbol.Type, expectedType));
+ Debug.Assert(!EndOfFile && TokenTypeEquals(CurrentToken.Type, expectedType));
}
- abstract protected bool SymbolTypeEquals(TSymbolType x, TSymbolType y);
+ abstract protected bool TokenTypeEquals(TTokenType x, TTokenType y);
- protected internal void PutBack(TSymbol symbol)
+ protected internal void PutBack(TToken token)
{
- if (symbol != null)
+ if (token != null)
{
- _tokenizer.PutBack(symbol);
+ _tokenizer.PutBack(token);
}
}
///
- /// Put the specified symbols back in the input stream. The provided list MUST be in the ORDER THE SYMBOLS WERE READ. The
- /// list WILL be reversed and the Putback(TSymbol) will be called on each item.
+ /// Put the specified tokens back in the input stream. The provided list MUST be in the ORDER THE TOKENS WERE READ. The
+ /// list WILL be reversed and the Putback(TToken) will be called on each item.
///
///
- /// If a document contains symbols: a, b, c, d, e, f
+ /// If a document contains tokens: a, b, c, d, e, f
/// and AcceptWhile or AcceptUntil is used to collect until d
/// the list returned by AcceptWhile/Until will contain: a, b, c IN THAT ORDER
/// that is the correct format for providing to this method. The caller of this method would,
/// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order
///
- protected internal void PutBack(IEnumerable symbols)
+ protected internal void PutBack(IEnumerable tokens)
{
- foreach (TSymbol symbol in symbols.Reverse())
+ foreach (TToken token in tokens.Reverse())
{
- PutBack(symbol);
+ PutBack(token);
}
}
protected internal void PutCurrentBack()
{
- if (!EndOfFile && CurrentSymbol != null)
+ if (!EndOfFile && CurrentToken != null)
{
- PutBack(CurrentSymbol);
+ PutBack(CurrentToken);
}
}
protected internal bool Balance(BalancingModes mode)
{
- var left = CurrentSymbol.Type;
+ var left = CurrentToken.Type;
var right = Language.FlipBracket(left);
var start = CurrentStart;
AcceptAndMoveNext();
@@ -223,13 +223,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Balance(mode, left, right, start);
}
- protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start)
+ protected internal bool Balance(BalancingModes mode, TTokenType left, TTokenType right, SourceLocation start)
{
var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1;
if (!EndOfFile)
{
- var syms = new List();
+ var syms = new List();
do
{
if (IsAtEmbeddedTransition(
@@ -253,7 +253,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
if (nesting > 0)
{
- syms.Add(CurrentSymbol);
+ syms.Add(CurrentToken);
}
}
while (nesting > 0 && NextToken());
@@ -280,29 +280,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else
{
- // Accept all the symbols we saw
+ // Accept all the tokens we saw
Accept(syms);
}
}
return nesting == 0;
}
- protected internal bool NextIs(TSymbolType type)
+ protected internal bool NextIs(TTokenType type)
{
- return NextIs(sym => sym != null && SymbolTypeEquals(type, sym.Type));
+ return NextIs(sym => sym != null && TokenTypeEquals(type, sym.Type));
}
- protected internal bool NextIs(params TSymbolType[] types)
+ protected internal bool NextIs(params TTokenType[] types)
{
- return NextIs(sym => sym != null && types.Any(t => SymbolTypeEquals(t, sym.Type)));
+ return NextIs(sym => sym != null && types.Any(t => TokenTypeEquals(t, sym.Type)));
}
- protected internal bool NextIs(Func condition)
+ protected internal bool NextIs(Func condition)
{
- var cur = CurrentSymbol;
+ var cur = CurrentToken;
if (NextToken())
{
- var result = condition(CurrentSymbol);
+ var result = condition(CurrentToken);
PutCurrentBack();
PutBack(cur);
EnsureCurrent();
@@ -317,27 +317,27 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- protected internal bool Was(TSymbolType type)
+ protected internal bool Was(TTokenType type)
{
- return PreviousSymbol != null && SymbolTypeEquals(PreviousSymbol.Type, type);
+ return PreviousToken != null && TokenTypeEquals(PreviousToken.Type, type);
}
- protected internal bool At(TSymbolType type)
+ protected internal bool At(TTokenType type)
{
- return !EndOfFile && CurrentSymbol != null && SymbolTypeEquals(CurrentSymbol.Type, type);
+ return !EndOfFile && CurrentToken != null && TokenTypeEquals(CurrentToken.Type, type);
}
protected internal bool AcceptAndMoveNext()
{
- Accept(CurrentSymbol);
+ Accept(CurrentToken);
return NextToken();
}
- protected TSymbol AcceptSingleWhiteSpaceCharacter()
+ protected TToken AcceptSingleWhiteSpaceCharacter()
{
- if (Language.IsWhiteSpace(CurrentSymbol))
+ if (Language.IsWhiteSpace(CurrentToken))
{
- Tuple pair = Language.SplitSymbol(CurrentSymbol, 1, Language.GetKnownSymbolType(KnownSymbolType.WhiteSpace));
+ Tuple pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
Accept(pair.Item1);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
NextToken();
@@ -346,32 +346,32 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null;
}
- protected internal void Accept(IEnumerable symbols)
+ protected internal void Accept(IEnumerable tokens)
{
- foreach (TSymbol symbol in symbols)
+ foreach (TToken token in tokens)
{
- Accept(symbol);
+ Accept(token);
}
}
- protected internal void Accept(TSymbol symbol)
+ protected internal void Accept(TToken token)
{
- if (symbol != null)
+ if (token != null)
{
- foreach (var error in symbol.Errors)
+ foreach (var error in token.Errors)
{
Context.ErrorSink.OnError(error);
}
- Span.Accept(symbol);
+ Span.Accept(token);
}
}
- protected internal bool AcceptAll(params TSymbolType[] types)
+ protected internal bool AcceptAll(params TTokenType[] types)
{
- foreach (TSymbolType type in types)
+ foreach (TTokenType type in types)
{
- if (CurrentSymbol == null || !SymbolTypeEquals(CurrentSymbol.Type, type))
+ if (CurrentToken == null || !TokenTypeEquals(CurrentToken.Type, type))
{
return false;
}
@@ -380,11 +380,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true;
}
- protected internal void AddMarkerSymbolIfNecessary()
+ protected internal void AddMarkerTokenIfNecessary()
{
- if (Span.Symbols.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharactersInternal.Any)
+ if (Span.Tokens.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharactersInternal.Any)
{
- Accept(Language.CreateMarkerSymbol());
+ Accept(Language.CreateMarkerToken());
}
}
@@ -408,7 +408,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void Output()
{
- if (Span.Symbols.Count > 0)
+ if (Span.Tokens.Count > 0)
{
var nextStart = Span.End;
@@ -462,23 +462,23 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Initialize(Span);
}
- protected internal void Expected(KnownSymbolType type)
+ protected internal void Expected(KnownTokenType type)
{
- Expected(Language.GetKnownSymbolType(type));
+ Expected(Language.GetKnownTokenType(type));
}
- protected internal void Expected(params TSymbolType[] types)
+ protected internal void Expected(params TTokenType[] types)
{
- Debug.Assert(!EndOfFile && CurrentSymbol != null && types.Contains(CurrentSymbol.Type));
+ Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Type));
AcceptAndMoveNext();
}
- protected internal bool Optional(KnownSymbolType type)
+ protected internal bool Optional(KnownTokenType type)
{
- return Optional(Language.GetKnownSymbolType(type));
+ return Optional(Language.GetKnownTokenType(type));
}
- protected internal bool Optional(TSymbolType type)
+ protected internal bool Optional(TTokenType type)
{
if (At(type))
{
@@ -490,7 +490,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool EnsureCurrent()
{
- if (CurrentSymbol == null)
+ if (CurrentToken == null)
{
return NextToken();
}
@@ -498,62 +498,62 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true;
}
- protected internal void AcceptWhile(TSymbolType type)
+ protected internal void AcceptWhile(TTokenType type)
{
- AcceptWhile(sym => SymbolTypeEquals(type, sym.Type));
+ AcceptWhile(sym => TokenTypeEquals(type, sym.Type));
}
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
- protected internal void AcceptWhile(TSymbolType type1, TSymbolType type2)
+ protected internal void AcceptWhile(TTokenType type1, TTokenType type2)
{
- AcceptWhile(sym => SymbolTypeEquals(type1, sym.Type) || SymbolTypeEquals(type2, sym.Type));
+ AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type));
}
- protected internal void AcceptWhile(TSymbolType type1, TSymbolType type2, TSymbolType type3)
+ protected internal void AcceptWhile(TTokenType type1, TTokenType type2, TTokenType type3)
{
- AcceptWhile(sym => SymbolTypeEquals(type1, sym.Type) || SymbolTypeEquals(type2, sym.Type) || SymbolTypeEquals(type3, sym.Type));
+ AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type) || TokenTypeEquals(type3, sym.Type));
}
- protected internal void AcceptWhile(params TSymbolType[] types)
+ protected internal void AcceptWhile(params TTokenType[] types)
{
- AcceptWhile(sym => types.Any(expected => SymbolTypeEquals(expected, sym.Type)));
+ AcceptWhile(sym => types.Any(expected => TokenTypeEquals(expected, sym.Type)));
}
- protected internal void AcceptUntil(TSymbolType type)
+ protected internal void AcceptUntil(TTokenType type)
{
- AcceptWhile(sym => !SymbolTypeEquals(type, sym.Type));
+ AcceptWhile(sym => !TokenTypeEquals(type, sym.Type));
}
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
- protected internal void AcceptUntil(TSymbolType type1, TSymbolType type2)
+ protected internal void AcceptUntil(TTokenType type1, TTokenType type2)
{
- AcceptWhile(sym => !SymbolTypeEquals(type1, sym.Type) && !SymbolTypeEquals(type2, sym.Type));
+ AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type));
}
- protected internal void AcceptUntil(TSymbolType type1, TSymbolType type2, TSymbolType type3)
+ protected internal void AcceptUntil(TTokenType type1, TTokenType type2, TTokenType type3)
{
- AcceptWhile(sym => !SymbolTypeEquals(type1, sym.Type) && !SymbolTypeEquals(type2, sym.Type) && !SymbolTypeEquals(type3, sym.Type));
+ AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type) && !TokenTypeEquals(type3, sym.Type));
}
- protected internal void AcceptUntil(params TSymbolType[] types)
+ protected internal void AcceptUntil(params TTokenType[] types)
{
- AcceptWhile(sym => types.All(expected => !SymbolTypeEquals(expected, sym.Type)));
+ AcceptWhile(sym => types.All(expected => !TokenTypeEquals(expected, sym.Type)));
}
- protected internal void AcceptWhile(Func condition)
+ protected internal void AcceptWhile(Func condition)
{
Accept(ReadWhileLazy(condition));
}
- protected internal IEnumerable ReadWhile(Func condition)
+ protected internal IEnumerable ReadWhile(Func condition)
{
return ReadWhileLazy(condition).ToList();
}
- protected TSymbol AcceptWhiteSpaceInLines()
+ protected TToken AcceptWhiteSpaceInLines()
{
- TSymbol lastWs = null;
- while (Language.IsWhiteSpace(CurrentSymbol) || Language.IsNewLine(CurrentSymbol))
+ TToken lastWs = null;
+ while (Language.IsWhiteSpace(CurrentToken) || Language.IsNewLine(CurrentToken))
{
// Capture the previous whitespace node
if (lastWs != null)
@@ -561,14 +561,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Accept(lastWs);
}
- if (Language.IsWhiteSpace(CurrentSymbol))
+ if (Language.IsWhiteSpace(CurrentToken))
{
- lastWs = CurrentSymbol;
+ lastWs = CurrentToken;
}
- else if (Language.IsNewLine(CurrentSymbol))
+ else if (Language.IsNewLine(CurrentToken))
{
// Accept newline and reset last whitespace tracker
- Accept(CurrentSymbol);
+ Accept(CurrentToken);
lastWs = null;
}
@@ -579,18 +579,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected bool AtIdentifier(bool allowKeywords)
{
- return CurrentSymbol != null &&
- (Language.IsIdentifier(CurrentSymbol) ||
- (allowKeywords && Language.IsKeyword(CurrentSymbol)));
+ return CurrentToken != null &&
+ (Language.IsIdentifier(CurrentToken) ||
+ (allowKeywords && Language.IsKeyword(CurrentToken)));
}
// Don't open this to sub classes because it's lazy but it looks eager.
// You have to advance the Enumerable to read the next characters.
- internal IEnumerable ReadWhileLazy(Func condition)
+ internal IEnumerable ReadWhileLazy(Func condition)
{
- while (EnsureCurrent() && condition(CurrentSymbol))
+ while (EnsureCurrent() && condition(CurrentToken))
{
- yield return CurrentSymbol;
+ yield return CurrentToken;
NextToken();
}
}
@@ -620,9 +620,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected void RazorComment()
{
- if (!Language.KnowsSymbolType(KnownSymbolType.CommentStart) ||
- !Language.KnowsSymbolType(KnownSymbolType.CommentStar) ||
- !Language.KnowsSymbolType(KnownSymbolType.CommentBody))
+ if (!Language.KnowsTokenType(KnownTokenType.CommentStart) ||
+ !Language.KnowsTokenType(KnownTokenType.CommentStar) ||
+ !Language.KnowsTokenType(KnownTokenType.CommentBody))
{
throw new InvalidOperationException(Resources.Language_Does_Not_Support_RazorComment);
}
@@ -634,18 +634,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator();
var start = CurrentStart;
- Expected(KnownSymbolType.CommentStart);
+ Expected(KnownTokenType.CommentStart);
Output(SpanKindInternal.Transition, AcceptedCharactersInternal.None);
- Expected(KnownSymbolType.CommentStar);
+ Expected(KnownTokenType.CommentStar);
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None);
- Optional(KnownSymbolType.CommentBody);
- AddMarkerSymbolIfNecessary();
+ Optional(KnownTokenType.CommentBody);
+ AddMarkerTokenIfNecessary();
Output(SpanKindInternal.Comment);
var errorReported = false;
- if (!Optional(KnownSymbolType.CommentStar))
+ if (!Optional(KnownTokenType.CommentStar))
{
errorReported = true;
Context.ErrorSink.OnError(
@@ -657,7 +657,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None);
}
- if (!Optional(KnownSymbolType.CommentStart))
+ if (!Optional(KnownTokenType.CommentStart))
{
if (!errorReported)
{
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
index b85f0cd10b..2b290bda5c 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
@@ -3,10 +3,10 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class TokenizerView
- where TSymbolType : struct
- where TTokenizer : Tokenizer
- where TSymbol : SymbolBase
+ internal class TokenizerView
+ where TTokenType : struct
+ where TTokenizer : Tokenizer
+ where TToken : TokenBase
{
public TokenizerView(TTokenizer tokenizer)
{
@@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public TTokenizer Tokenizer { get; private set; }
public bool EndOfFile { get; private set; }
- public TSymbol Current { get; private set; }
+ public TToken Current { get; private set; }
public ITextDocument Source
{
@@ -24,14 +24,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public bool Next()
{
- Current = Tokenizer.NextSymbol();
+ Current = Tokenizer.NextToken();
EndOfFile = (Current == null);
return !EndOfFile;
}
- public void PutBack(TSymbol symbol)
+ public void PutBack(TToken token)
{
- Source.Position -= symbol.Content.Length;
+ Source.Position -= token.Content.Length;
Current = null;
EndOfFile = Source.Position >= Source.Length;
Tokenizer.Reset();
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/WhiteSpaceRewriter.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/WhiteSpaceRewriter.cs
index 766f57d4bf..cf71b0a024 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/WhiteSpaceRewriter.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/WhiteSpaceRewriter.cs
@@ -23,7 +23,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Add this node to the parent
var builder = new SpanBuilder(ws);
- builder.ClearSymbols();
+ builder.ClearTokens();
FillSpan(builder, ws.Start, ws.Content);
parent.Children.Add(builder.Build());
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Properties/Resources.Designer.cs b/src/Microsoft.AspNetCore.Razor.Language/Properties/Resources.Designer.cs
index a785a39bca..773fb6de9c 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Properties/Resources.Designer.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Properties/Resources.Designer.cs
@@ -139,128 +139,128 @@ namespace Microsoft.AspNetCore.Razor.Language
///
/// <<character literal>>
///
- internal static string CSharpSymbol_CharacterLiteral
+ internal static string CSharpToken_CharacterLiteral
{
- get => GetString("CSharpSymbol_CharacterLiteral");
+ get => GetString("CSharpToken_CharacterLiteral");
}
///
/// <<character literal>>
///
- internal static string FormatCSharpSymbol_CharacterLiteral()
- => GetString("CSharpSymbol_CharacterLiteral");
+ internal static string FormatCSharpToken_CharacterLiteral()
+ => GetString("CSharpToken_CharacterLiteral");
///
/// <<comment>>
///
- internal static string CSharpSymbol_Comment
+ internal static string CSharpToken_Comment
{
- get => GetString("CSharpSymbol_Comment");
+ get => GetString("CSharpToken_Comment");
}
///
/// <<comment>>
///
- internal static string FormatCSharpSymbol_Comment()
- => GetString("CSharpSymbol_Comment");
+ internal static string FormatCSharpToken_Comment()
+ => GetString("CSharpToken_Comment");
///
/// <<identifier>>
///
- internal static string CSharpSymbol_Identifier
+ internal static string CSharpToken_Identifier
{
- get => GetString("CSharpSymbol_Identifier");
+ get => GetString("CSharpToken_Identifier");
}
///
/// <<identifier>>
///
- internal static string FormatCSharpSymbol_Identifier()
- => GetString("CSharpSymbol_Identifier");
+ internal static string FormatCSharpToken_Identifier()
+ => GetString("CSharpToken_Identifier");
///
/// <<integer literal>>
///
- internal static string CSharpSymbol_IntegerLiteral
+ internal static string CSharpToken_IntegerLiteral
{
- get => GetString("CSharpSymbol_IntegerLiteral");
+ get => GetString("CSharpToken_IntegerLiteral");
}
///
/// <<integer literal>>
///
- internal static string FormatCSharpSymbol_IntegerLiteral()
- => GetString("CSharpSymbol_IntegerLiteral");
+ internal static string FormatCSharpToken_IntegerLiteral()
+ => GetString("CSharpToken_IntegerLiteral");
///
/// <<keyword>>
///
- internal static string CSharpSymbol_Keyword
+ internal static string CSharpToken_Keyword
{
- get => GetString("CSharpSymbol_Keyword");
+ get => GetString("CSharpToken_Keyword");
}
///
/// <<keyword>>
///
- internal static string FormatCSharpSymbol_Keyword()
- => GetString("CSharpSymbol_Keyword");
+ internal static string FormatCSharpToken_Keyword()
+ => GetString("CSharpToken_Keyword");
///
/// <<newline sequence>>
///
- internal static string CSharpSymbol_Newline
+ internal static string CSharpToken_Newline
{
- get => GetString("CSharpSymbol_Newline");
+ get => GetString("CSharpToken_Newline");
}
///
/// <<newline sequence>>
///
- internal static string FormatCSharpSymbol_Newline()
- => GetString("CSharpSymbol_Newline");
+ internal static string FormatCSharpToken_Newline()
+ => GetString("CSharpToken_Newline");
///
/// <<real literal>>
///
- internal static string CSharpSymbol_RealLiteral
+ internal static string CSharpToken_RealLiteral
{
- get => GetString("CSharpSymbol_RealLiteral");
+ get => GetString("CSharpToken_RealLiteral");
}
///
/// <<real literal>>
///
- internal static string FormatCSharpSymbol_RealLiteral()
- => GetString("CSharpSymbol_RealLiteral");
+ internal static string FormatCSharpToken_RealLiteral()
+ => GetString("CSharpToken_RealLiteral");
///
/// <<string literal>>
///
- internal static string CSharpSymbol_StringLiteral
+ internal static string CSharpToken_StringLiteral
{
- get => GetString("CSharpSymbol_StringLiteral");
+ get => GetString("CSharpToken_StringLiteral");
}
///
/// <<string literal>>
///
- internal static string FormatCSharpSymbol_StringLiteral()
- => GetString("CSharpSymbol_StringLiteral");
+ internal static string FormatCSharpToken_StringLiteral()
+ => GetString("CSharpToken_StringLiteral");
///
/// <<white space>>
///
- internal static string CSharpSymbol_Whitespace
+ internal static string CSharpToken_Whitespace
{
- get => GetString("CSharpSymbol_Whitespace");
+ get => GetString("CSharpToken_Whitespace");
}
///
/// <<white space>>
///
- internal static string FormatCSharpSymbol_Whitespace()
- => GetString("CSharpSymbol_Whitespace");
+ internal static string FormatCSharpToken_Whitespace()
+ => GetString("CSharpToken_Whitespace");
///
/// The document type '{0}' does not support the extension '{1}'.
@@ -489,58 +489,58 @@ namespace Microsoft.AspNetCore.Razor.Language
///
/// <<newline sequence>>
///
- internal static string HtmlSymbol_NewLine
+ internal static string HtmlToken_NewLine
{
- get => GetString("HtmlSymbol_NewLine");
+ get => GetString("HtmlToken_NewLine");
}
///
/// <<newline sequence>>
///
- internal static string FormatHtmlSymbol_NewLine()
- => GetString("HtmlSymbol_NewLine");
+ internal static string FormatHtmlToken_NewLine()
+ => GetString("HtmlToken_NewLine");
///
/// <<razor comment>>
///
- internal static string HtmlSymbol_RazorComment
+ internal static string HtmlToken_RazorComment
{
- get => GetString("HtmlSymbol_RazorComment");
+ get => GetString("HtmlToken_RazorComment");
}
///
/// <<razor comment>>
///
- internal static string FormatHtmlSymbol_RazorComment()
- => GetString("HtmlSymbol_RazorComment");
+ internal static string FormatHtmlToken_RazorComment()
+ => GetString("HtmlToken_RazorComment");
///
/// <<text>>
///
- internal static string HtmlSymbol_Text
+ internal static string HtmlToken_Text
{
- get => GetString("HtmlSymbol_Text");
+ get => GetString("HtmlToken_Text");
}
///
/// <<text>>
///
- internal static string FormatHtmlSymbol_Text()
- => GetString("HtmlSymbol_Text");
+ internal static string FormatHtmlToken_Text()
+ => GetString("HtmlToken_Text");
///
/// <<white space>>
///
- internal static string HtmlSymbol_WhiteSpace
+ internal static string HtmlToken_WhiteSpace
{
- get => GetString("HtmlSymbol_WhiteSpace");
+ get => GetString("HtmlToken_WhiteSpace");
}
///
/// <<white space>>
///
- internal static string FormatHtmlSymbol_WhiteSpace()
- => GetString("HtmlSymbol_WhiteSpace");
+ internal static string FormatHtmlToken_WhiteSpace()
+ => GetString("HtmlToken_WhiteSpace");
///
/// Specify the base class for the current document.
@@ -711,7 +711,7 @@ namespace Microsoft.AspNetCore.Razor.Language
=> GetString("KeyMustNotBeNull");
///
- /// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known symbol types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
+ /// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known token types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
///
internal static string Language_Does_Not_Support_RazorComment
{
@@ -719,7 +719,7 @@ namespace Microsoft.AspNetCore.Razor.Language
}
///
- /// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known symbol types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
+ /// Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known token types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
///
internal static string FormatLanguage_Does_Not_Support_RazorComment()
=> GetString("Language_Does_Not_Support_RazorComment");
@@ -1351,16 +1351,16 @@ namespace Microsoft.AspNetCore.Razor.Language
///
/// <<unknown>>
///
- internal static string Symbol_Unknown
+ internal static string Token_Unknown
{
- get => GetString("Symbol_Unknown");
+ get => GetString("Token_Unknown");
}
///
/// <<unknown>>
///
- internal static string FormatSymbol_Unknown()
- => GetString("Symbol_Unknown");
+ internal static string FormatToken_Unknown()
+ => GetString("Token_Unknown");
///
/// Invalid tag helper bound property '{1}' on tag helper '{0}'. Tag helpers cannot bind to HTML attributes with name '{2}' because the name contains a '{3}' character.
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Resources.resx b/src/Microsoft.AspNetCore.Razor.Language/Resources.resx
index 195a9d791b..e23e66c297 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Resources.resx
+++ b/src/Microsoft.AspNetCore.Razor.Language/Resources.resx
@@ -146,31 +146,31 @@
Invalid newline sequence '{0}'. Support newline sequences are '\r\n' and '\n'.
-
+
<<character literal>>
-
+
<<comment>>
-
+
<<identifier>>
-
+
<<integer literal>>
-
+
<<keyword>>
-
+
<<newline sequence>>
-
+
<<real literal>>
-
+
<<string literal>>
-
+
<<white space>>
@@ -221,16 +221,16 @@
Specify a C# code block.
-
+
<<newline sequence>>
-
+
<<razor comment>>
-
+
<<text>>
-
+
<<white space>>
@@ -270,7 +270,7 @@
The key must not be null.
- Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known symbol types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorComment
+ Cannot use built-in RazorComment handler, language characteristics does not define the CommentStart, CommentStar and CommentBody known token types or parser does not override TokenizerBackedParser.OutputSpanBeforeRazorCommentThe specified encoding '{0}' does not match the content's encoding '{1}'.
@@ -423,7 +423,7 @@ Instead, wrap the contents of the block in "{{}}":
@section Header { ... }In CSHTML, the @section keyword is case-sensitive and lowercase (as with all C# keywords)
-
+
<<unknown>>
diff --git a/src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netcore.json b/src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netcore.json
new file mode 100644
index 0000000000..a753f1bfc8
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netcore.json
@@ -0,0 +1,15 @@
+[
+ {
+ "TypeId": "protected enum Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer+RazorCommentTokenizerState where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase where T1 : struct",
+ "Kind": "Removal"
+ },
+ {
+ "TypeId": "protected struct Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer+StateResult where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase where T1 : struct",
+ "Kind": "Removal"
+ },
+ {
+ "TypeId": "protected class Microsoft.AspNetCore.Razor.Language.Legacy.CSharpCodeParser+Block",
+ "MemberId": "public .ctor(Microsoft.AspNetCore.Razor.Language.Legacy.CSharpSymbol symbol, Microsoft.AspNetCore.Razor.Language.SourceLocation start)",
+ "Kind": "Removal"
+ }
+]
\ No newline at end of file
diff --git a/src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netframework.json b/src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netframework.json
new file mode 100644
index 0000000000..a753f1bfc8
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Razor.Language/breakingchanges.netframework.json
@@ -0,0 +1,15 @@
+[
+ {
+ "TypeId": "protected enum Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer+RazorCommentTokenizerState where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase where T1 : struct",
+ "Kind": "Removal"
+ },
+ {
+ "TypeId": "protected struct Microsoft.AspNetCore.Razor.Language.Legacy.Tokenizer+StateResult where T0 : Microsoft.AspNetCore.Razor.Language.Legacy.SymbolBase where T1 : struct",
+ "Kind": "Removal"
+ },
+ {
+ "TypeId": "protected class Microsoft.AspNetCore.Razor.Language.Legacy.CSharpCodeParser+Block",
+ "MemberId": "public .ctor(Microsoft.AspNetCore.Razor.Language.Legacy.CSharpSymbol symbol, Microsoft.AspNetCore.Razor.Language.SourceLocation start)",
+ "Kind": "Removal"
+ }
+]
\ No newline at end of file
diff --git a/src/Microsoft.VisualStudio.Editor.Razor/BraceSmartIndenter.cs b/src/Microsoft.VisualStudio.Editor.Razor/BraceSmartIndenter.cs
index 2963fc9b0f..4de857b966 100644
--- a/src/Microsoft.VisualStudio.Editor.Razor/BraceSmartIndenter.cs
+++ b/src/Microsoft.VisualStudio.Editor.Razor/BraceSmartIndenter.cs
@@ -278,9 +278,9 @@ namespace Microsoft.VisualStudio.Editor.Razor
{
// We only support whitespace based content. Any non-whitespace content is an unkonwn to us
// in regards to indentation.
- for (var i = 0; i < owner.Symbols.Count; i++)
+ for (var i = 0; i < owner.Tokens.Count; i++)
{
- if (!string.IsNullOrWhiteSpace(owner.Symbols[i].Content))
+ if (!string.IsNullOrWhiteSpace(owner.Tokens[i].Content))
{
return true;
}
diff --git a/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs b/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
index c0ed95ad11..109ba1311b 100644
--- a/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
+++ b/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
@@ -142,9 +142,9 @@ namespace Microsoft.VisualStudio.Editor.Razor
internal static bool IsCSharpOpenCurlyBrace(SyntaxTreeNode currentChild)
{
return currentChild is Span currentSpan &&
- currentSpan.Symbols.Count == 1 &&
- currentSpan.Symbols[0] is CSharpSymbol symbol &&
- symbol.Type == CSharpSymbolType.LeftBrace;
+ currentSpan.Tokens.Count == 1 &&
+ currentSpan.Tokens[0] is CSharpToken symbol &&
+ symbol.Type == CSharpTokenType.LeftBrace;
}
}
}
diff --git a/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs b/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
index 56768829d1..6a2d017716 100644
--- a/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
+++ b/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
@@ -162,7 +162,7 @@ namespace Microsoft.VisualStudio.Editor.Razor
}
if (owner.ChunkGenerator is ExpressionChunkGenerator &&
- owner.Symbols.All(IsDirectiveCompletableSymbol) &&
+ owner.Tokens.All(IsDirectiveCompletableSymbol) &&
// Do not provide IntelliSense for explicit expressions. Explicit expressions will usually look like:
// [@] [(] [DateTime.Now] [)]
owner.Parent?.Children.Count > 1 &&
@@ -204,16 +204,16 @@ namespace Microsoft.VisualStudio.Editor.Razor
return false;
}
- private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.ISymbol symbol)
+ private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.IToken symbol)
{
- if (!(symbol is CSharpSymbol csharpSymbol))
+ if (!(symbol is CSharpToken csharpSymbol))
{
return false;
}
- return csharpSymbol.Type == CSharpSymbolType.Identifier ||
+ return csharpSymbol.Type == CSharpTokenType.Identifier ||
// Marker symbol
- csharpSymbol.Type == CSharpSymbolType.Unknown;
+ csharpSymbol.Type == CSharpTokenType.Unknown;
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
index 4b4f984ce6..d54ef401df 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
@@ -13,7 +13,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero);
- spanBuilder.Accept(new HtmlSymbol("hello", HtmlSymbolType.Text));
+ spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder()
{
@@ -29,7 +29,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var parentBlock = blockBuilder.Build();
var originalBlockLength = parentBlock.Length;
spanBuilder = new SpanBuilder(SourceLocation.Zero);
- spanBuilder.Accept(new HtmlSymbol("hi", HtmlSymbolType.Text));
+ spanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
span.ReplaceWith(spanBuilder);
// Wire up parents now so we can re-trigger ChildChanged to cause cache refresh.
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
index 1ab365c98a..e8eeb8c586 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
@@ -8,13 +8,13 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public class CSharpLanguageCharacteristicsTest
{
[Fact]
- public void GetSample_RightShiftAssign_ReturnsCorrectSymbol()
+ public void GetSample_RightShiftAssign_ReturnsCorrectToken()
{
// Arrange & Act
- var symbol = CSharpLanguageCharacteristics.Instance.GetSample(CSharpSymbolType.RightShiftAssign);
+ var token = CSharpLanguageCharacteristics.Instance.GetSample(CSharpTokenType.RightShiftAssign);
// Assert
- Assert.Equal(">>=", symbol);
+ Assert.Equal(">>=", token);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
index 2ff59a679b..64dafafa16 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
@@ -7,16 +7,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
{
- private new CSharpSymbol IgnoreRemaining => (CSharpSymbol)base.IgnoreRemaining;
+ private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer(
"@* Foo * Bar * Baz *",
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol(" Foo * Bar * Baz *", CSharpSymbolType.RazorComment));
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken(" Foo * Bar * Baz *", CSharpTokenType.RazorComment));
}
[Fact]
@@ -24,11 +24,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo * Bar * Baz *@",
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol(" Foo * Bar * Baz ", CSharpSymbolType.RazorComment),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition));
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken(" Foo * Bar * Baz ", CSharpTokenType.RazorComment),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
}
[Fact]
@@ -36,59 +36,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo Bar Baz *@",
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol(" Foo Bar Baz ", CSharpSymbolType.RazorComment),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition));
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken(" Foo Bar Baz ", CSharpTokenType.RazorComment),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{
- TestTokenizer("// Foo Bar Baz", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment));
+ TestTokenizer("// Foo Bar Baz", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment));
}
[Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline()
{
- TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining);
+ TestTokenizer("// Foo Bar Baz\na", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment), IgnoreRemaining);
}
[Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{
- TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol("// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining);
+ TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpToken("// Foo/*Bar*/ Baz", CSharpTokenType.Comment), IgnoreRemaining);
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{
- TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment));
+ TestTokenizer("/* Foo\nBar\nBaz */", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment));
}
[Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{
- TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment), IgnoreRemaining);
}
[Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{
- TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol("/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo\nBar\nBaz", new CSharpToken("/* Foo\nBar\nBaz", CSharpTokenType.Comment), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{
- TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol("/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpToken("/* Foo/*\nBar\nBaz*/", CSharpTokenType.Comment), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{
- TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol("/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpToken("/* Foo\nBar\nBaz* */", CSharpTokenType.Comment), IgnoreRemaining);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
index fe17606261..b24c976dc9 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
@@ -10,73 +10,73 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact]
public void Simple_Identifier_Is_Recognized()
{
- TestTokenizer("foo", new CSharpSymbol("foo", CSharpSymbolType.Identifier));
+ TestTokenizer("foo", new CSharpToken("foo", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized()
{
- TestTokenizer("_foo", new CSharpSymbol("_foo", CSharpSymbolType.Identifier));
+ TestTokenizer("_foo", new CSharpToken("_foo", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Digits()
{
- TestTokenizer("foo4", new CSharpSymbol("foo4", CSharpSymbolType.Identifier));
+ TestTokenizer("foo4", new CSharpToken("foo4", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Titlecase_Letter()
{
- TestTokenizer("ῼfoo", new CSharpSymbol("ῼfoo", CSharpSymbolType.Identifier));
+ TestTokenizer("ῼfoo", new CSharpToken("ῼfoo", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Letter_Modifier()
{
- TestTokenizer("ᵊfoo", new CSharpSymbol("ᵊfoo", CSharpSymbolType.Identifier));
+ TestTokenizer("ᵊfoo", new CSharpToken("ᵊfoo", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Other_Letter()
{
- TestTokenizer("ƻfoo", new CSharpSymbol("ƻfoo", CSharpSymbolType.Identifier));
+ TestTokenizer("ƻfoo", new CSharpToken("ƻfoo", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Number_Letter()
{
- TestTokenizer("Ⅽool", new CSharpSymbol("Ⅽool", CSharpSymbolType.Identifier));
+ TestTokenizer("Ⅽool", new CSharpToken("Ⅽool", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark()
{
- TestTokenizer("foo\u0300", new CSharpSymbol("foo\u0300", CSharpSymbolType.Identifier));
+ TestTokenizer("foo\u0300", new CSharpToken("foo\u0300", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark()
{
- TestTokenizer("fooः", new CSharpSymbol("fooः", CSharpSymbolType.Identifier));
+ TestTokenizer("fooः", new CSharpToken("fooः", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_English_Digit()
{
- TestTokenizer("foo١", new CSharpSymbol("foo١", CSharpSymbolType.Identifier));
+ TestTokenizer("foo١", new CSharpToken("foo١", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Connector_Punctuation()
{
- TestTokenizer("foo‿bar", new CSharpSymbol("foo‿bar", CSharpSymbolType.Identifier));
+ TestTokenizer("foo‿bar", new CSharpToken("foo‿bar", CSharpTokenType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Format_Character()
{
- TestTokenizer("foobar", new CSharpSymbol("foobar", CSharpSymbolType.Identifier));
+ TestTokenizer("foobar", new CSharpToken("foobar", CSharpTokenType.Identifier));
}
[Fact]
@@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType)
{
- TestTokenizer(keyword, new CSharpSymbol(keyword, CSharpSymbolType.Keyword) { Keyword = keywordType });
+ TestTokenizer(keyword, new CSharpToken(keyword, CSharpTokenType.Keyword) { Keyword = keywordType });
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
index 0d614bf84a..a10aea8ba9 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
@@ -8,280 +8,280 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase
{
- private new CSharpSymbol IgnoreRemaining => (CSharpSymbol)base.IgnoreRemaining;
+ private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
[Fact]
public void Simple_Integer_Literal_Is_Recognized()
{
- TestSingleToken("01189998819991197253", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("01189998819991197253", CSharpTokenType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized()
{
- TestSingleToken("42U", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("42u", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("42U", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42u", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42L", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("42l", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("42L", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42l", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42UL", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("42Ul", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("42UL", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42Ul", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42uL", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("42ul", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("42uL", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42ul", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42LU", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("42Lu", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("42LU", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42Lu", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42lU", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("42lu", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("42lU", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42lu", CSharpTokenType.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{
- TestTokenizer("42a", new CSharpSymbol("42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
+ TestTokenizer("42a", new CSharpToken("42", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Simple_Hex_Literal_Is_Recognized()
{
- TestSingleToken("0x0123456789ABCDEF", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0x0123456789ABCDEF", CSharpTokenType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal()
{
- TestSingleToken("0xDEADBEEFU", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFu", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFU", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFu", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFL", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFl", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFL", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFl", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFUL", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFUl", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFUL", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFUl", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFuL", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFul", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFuL", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFul", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFLU", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFLu", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFLU", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFLu", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFlU", CSharpSymbolType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFlu", CSharpSymbolType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFlU", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFlu", CSharpTokenType.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{
- TestTokenizer("0xDEADBEEFz", new CSharpSymbol("0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
+ TestTokenizer("0xDEADBEEFz", new CSharpToken("0xDEADBEEF", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{
- TestTokenizer("3.a", new CSharpSymbol("3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
+ TestTokenizer("3.a", new CSharpToken("3", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Simple_Real_Literal_Is_Recognized()
{
- TestTokenizer("3.14159", new CSharpSymbol("3.14159", CSharpSymbolType.RealLiteral));
+ TestTokenizer("3.14159", new CSharpToken("3.14159", CSharpTokenType.RealLiteral));
}
[Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{
- TestTokenizer(".14159", new CSharpSymbol(".14159", CSharpSymbolType.RealLiteral));
+ TestTokenizer(".14159", new CSharpToken(".14159", CSharpTokenType.RealLiteral));
}
[Fact]
public void Integer_With_Real_Type_Suffix_Is_Recognized()
{
- TestSingleToken("42F", CSharpSymbolType.RealLiteral);
- TestSingleToken("42f", CSharpSymbolType.RealLiteral);
- TestSingleToken("42D", CSharpSymbolType.RealLiteral);
- TestSingleToken("42d", CSharpSymbolType.RealLiteral);
- TestSingleToken("42M", CSharpSymbolType.RealLiteral);
- TestSingleToken("42m", CSharpSymbolType.RealLiteral);
+ TestSingleToken("42F", CSharpTokenType.RealLiteral);
+ TestSingleToken("42f", CSharpTokenType.RealLiteral);
+ TestSingleToken("42D", CSharpTokenType.RealLiteral);
+ TestSingleToken("42d", CSharpTokenType.RealLiteral);
+ TestSingleToken("42M", CSharpTokenType.RealLiteral);
+ TestSingleToken("42m", CSharpTokenType.RealLiteral);
}
[Fact]
public void Integer_With_Exponent_Is_Recognized()
{
- TestSingleToken("1e10", CSharpSymbolType.RealLiteral);
- TestSingleToken("1E10", CSharpSymbolType.RealLiteral);
- TestSingleToken("1e+10", CSharpSymbolType.RealLiteral);
- TestSingleToken("1E+10", CSharpSymbolType.RealLiteral);
- TestSingleToken("1e-10", CSharpSymbolType.RealLiteral);
- TestSingleToken("1E-10", CSharpSymbolType.RealLiteral);
+ TestSingleToken("1e10", CSharpTokenType.RealLiteral);
+ TestSingleToken("1E10", CSharpTokenType.RealLiteral);
+ TestSingleToken("1e+10", CSharpTokenType.RealLiteral);
+ TestSingleToken("1E+10", CSharpTokenType.RealLiteral);
+ TestSingleToken("1e-10", CSharpTokenType.RealLiteral);
+ TestSingleToken("1E-10", CSharpTokenType.RealLiteral);
}
[Fact]
public void Real_Number_With_Type_Suffix_Is_Recognized()
{
- TestSingleToken("3.14F", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14f", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14D", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14d", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14M", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14m", CSharpSymbolType.RealLiteral);
+ TestSingleToken("3.14F", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14f", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14D", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14d", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14M", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14m", CSharpTokenType.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_Is_Recognized()
{
- TestSingleToken("3.14E10", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14e10", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14E+10", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14e+10", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14E-10", CSharpSymbolType.RealLiteral);
- TestSingleToken("3.14e-10", CSharpSymbolType.RealLiteral);
+ TestSingleToken("3.14E10", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14e10", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14E+10", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14e+10", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14E-10", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14e-10", CSharpTokenType.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized()
{
- TestSingleToken("3.14E+10F", CSharpSymbolType.RealLiteral);
+ TestSingleToken("3.14E+10F", CSharpTokenType.RealLiteral);
}
[Fact]
public void Single_Character_Literal_Is_Recognized()
{
- TestSingleToken("'f'", CSharpSymbolType.CharacterLiteral);
+ TestSingleToken("'f'", CSharpTokenType.CharacterLiteral);
}
[Fact]
public void Multi_Character_Literal_Is_Recognized()
{
- TestSingleToken("'foo'", CSharpSymbolType.CharacterLiteral);
+ TestSingleToken("'foo'", CSharpTokenType.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
- TestSingleToken("'foo bar", CSharpSymbolType.CharacterLiteral);
+ TestSingleToken("'foo bar", CSharpTokenType.CharacterLiteral);
}
[Fact]
public void Character_Literal_Not_Terminated_By_Escaped_Quote()
{
- TestSingleToken("'foo\\'bar'", CSharpSymbolType.CharacterLiteral);
+ TestSingleToken("'foo\\'bar'", CSharpTokenType.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
- TestTokenizer("'foo\n", new CSharpSymbol("'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\n", new CSharpToken("'foo", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("'foo\\\n", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\\n", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer("'foo\\\nflarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\\nflarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\" + Environment.NewLine, new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Allows_Escaped_Escape()
{
- TestTokenizer("'foo\\\\'blah", new CSharpSymbol("'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\\\'blah", new CSharpToken("'foo\\\\'", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Is_Recognized()
{
- TestSingleToken("\"foo\"", CSharpSymbolType.StringLiteral);
+ TestSingleToken("\"foo\"", CSharpTokenType.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
- TestSingleToken("\"foo bar", CSharpSymbolType.StringLiteral);
+ TestSingleToken("\"foo bar", CSharpTokenType.StringLiteral);
}
[Fact]
public void String_Literal_Not_Terminated_By_Escaped_Quote()
{
- TestSingleToken("\"foo\\\"bar\"", CSharpSymbolType.StringLiteral);
+ TestSingleToken("\"foo\\\"bar\"", CSharpTokenType.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
- TestTokenizer("\"foo\n", new CSharpSymbol("\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\n", new CSharpToken("\"foo", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("\"foo\\\n", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\\n", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer("\"foo\\\nflarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\\nflarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Allows_Escaped_Escape()
{
- TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol("\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\\\\"blah", new CSharpToken("\"foo\\\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Can_Contain_Newlines()
{
- TestSingleToken("@\"foo\nbar\nbaz\"", CSharpSymbolType.StringLiteral);
+ TestSingleToken("@\"foo\nbar\nbaz\"", CSharpTokenType.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote()
{
- TestSingleToken("@\"foo\"\"bar\"", CSharpSymbolType.StringLiteral);
+ TestSingleToken("@\"foo\"\"bar\"", CSharpTokenType.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{
- TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol("@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("@\"foo\\\"bar\"", new CSharpToken("@\"foo\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_EOF()
{
- TestSingleToken("@\"foo", CSharpSymbolType.StringLiteral);
+ TestSingleToken("@\"foo", CSharpTokenType.StringLiteral);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
index 67000e950b..0b600ea9bc 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
@@ -10,287 +10,287 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact]
public void LeftBrace_Is_Recognized()
{
- TestSingleToken("{", CSharpSymbolType.LeftBrace);
+ TestSingleToken("{", CSharpTokenType.LeftBrace);
}
[Fact]
public void Plus_Is_Recognized()
{
- TestSingleToken("+", CSharpSymbolType.Plus);
+ TestSingleToken("+", CSharpTokenType.Plus);
}
[Fact]
public void Assign_Is_Recognized()
{
- TestSingleToken("=", CSharpSymbolType.Assign);
+ TestSingleToken("=", CSharpTokenType.Assign);
}
[Fact]
public void Arrow_Is_Recognized()
{
- TestSingleToken("->", CSharpSymbolType.Arrow);
+ TestSingleToken("->", CSharpTokenType.Arrow);
}
[Fact]
public void AndAssign_Is_Recognized()
{
- TestSingleToken("&=", CSharpSymbolType.AndAssign);
+ TestSingleToken("&=", CSharpTokenType.AndAssign);
}
[Fact]
public void RightBrace_Is_Recognized()
{
- TestSingleToken("}", CSharpSymbolType.RightBrace);
+ TestSingleToken("}", CSharpTokenType.RightBrace);
}
[Fact]
public void Minus_Is_Recognized()
{
- TestSingleToken("-", CSharpSymbolType.Minus);
+ TestSingleToken("-", CSharpTokenType.Minus);
}
[Fact]
public void LessThan_Is_Recognized()
{
- TestSingleToken("<", CSharpSymbolType.LessThan);
+ TestSingleToken("<", CSharpTokenType.LessThan);
}
[Fact]
public void Equals_Is_Recognized()
{
- TestSingleToken("==", CSharpSymbolType.Equals);
+ TestSingleToken("==", CSharpTokenType.Equals);
}
[Fact]
public void OrAssign_Is_Recognized()
{
- TestSingleToken("|=", CSharpSymbolType.OrAssign);
+ TestSingleToken("|=", CSharpTokenType.OrAssign);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
- TestSingleToken("[", CSharpSymbolType.LeftBracket);
+ TestSingleToken("[", CSharpTokenType.LeftBracket);
}
[Fact]
public void Star_Is_Recognized()
{
- TestSingleToken("*", CSharpSymbolType.Star);
+ TestSingleToken("*", CSharpTokenType.Star);
}
[Fact]
public void GreaterThan_Is_Recognized()
{
- TestSingleToken(">", CSharpSymbolType.GreaterThan);
+ TestSingleToken(">", CSharpTokenType.GreaterThan);
}
[Fact]
public void NotEqual_Is_Recognized()
{
- TestSingleToken("!=", CSharpSymbolType.NotEqual);
+ TestSingleToken("!=", CSharpTokenType.NotEqual);
}
[Fact]
public void XorAssign_Is_Recognized()
{
- TestSingleToken("^=", CSharpSymbolType.XorAssign);
+ TestSingleToken("^=", CSharpTokenType.XorAssign);
}
[Fact]
public void RightBracket_Is_Recognized()
{
- TestSingleToken("]", CSharpSymbolType.RightBracket);
+ TestSingleToken("]", CSharpTokenType.RightBracket);
}
[Fact]
public void Slash_Is_Recognized()
{
- TestSingleToken("/", CSharpSymbolType.Slash);
+ TestSingleToken("/", CSharpTokenType.Slash);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
- TestSingleToken("?", CSharpSymbolType.QuestionMark);
+ TestSingleToken("?", CSharpTokenType.QuestionMark);
}
[Fact]
public void LessThanEqual_Is_Recognized()
{
- TestSingleToken("<=", CSharpSymbolType.LessThanEqual);
+ TestSingleToken("<=", CSharpTokenType.LessThanEqual);
}
[Fact]
public void LeftShift_Is_Not_Specially_Recognized()
{
TestTokenizer("<<",
- new CSharpSymbol("<", CSharpSymbolType.LessThan),
- new CSharpSymbol("<", CSharpSymbolType.LessThan));
+ new CSharpToken("<", CSharpTokenType.LessThan),
+ new CSharpToken("<", CSharpTokenType.LessThan));
}
[Fact]
public void LeftParen_Is_Recognized()
{
- TestSingleToken("(", CSharpSymbolType.LeftParenthesis);
+ TestSingleToken("(", CSharpTokenType.LeftParenthesis);
}
[Fact]
public void Modulo_Is_Recognized()
{
- TestSingleToken("%", CSharpSymbolType.Modulo);
+ TestSingleToken("%", CSharpTokenType.Modulo);
}
[Fact]
public void NullCoalesce_Is_Recognized()
{
- TestSingleToken("??", CSharpSymbolType.NullCoalesce);
+ TestSingleToken("??", CSharpTokenType.NullCoalesce);
}
[Fact]
public void GreaterThanEqual_Is_Recognized()
{
- TestSingleToken(">=", CSharpSymbolType.GreaterThanEqual);
+ TestSingleToken(">=", CSharpTokenType.GreaterThanEqual);
}
[Fact]
public void EqualGreaterThan_Is_Recognized()
{
- TestSingleToken("=>", CSharpSymbolType.GreaterThanEqual);
+ TestSingleToken("=>", CSharpTokenType.GreaterThanEqual);
}
[Fact]
public void RightParen_Is_Recognized()
{
- TestSingleToken(")", CSharpSymbolType.RightParenthesis);
+ TestSingleToken(")", CSharpTokenType.RightParenthesis);
}
[Fact]
public void And_Is_Recognized()
{
- TestSingleToken("&", CSharpSymbolType.And);
+ TestSingleToken("&", CSharpTokenType.And);
}
[Fact]
public void DoubleColon_Is_Recognized()
{
- TestSingleToken("::", CSharpSymbolType.DoubleColon);
+ TestSingleToken("::", CSharpTokenType.DoubleColon);
}
[Fact]
public void PlusAssign_Is_Recognized()
{
- TestSingleToken("+=", CSharpSymbolType.PlusAssign);
+ TestSingleToken("+=", CSharpTokenType.PlusAssign);
}
[Fact]
public void Semicolon_Is_Recognized()
{
- TestSingleToken(";", CSharpSymbolType.Semicolon);
+ TestSingleToken(";", CSharpTokenType.Semicolon);
}
[Fact]
public void Tilde_Is_Recognized()
{
- TestSingleToken("~", CSharpSymbolType.Tilde);
+ TestSingleToken("~", CSharpTokenType.Tilde);
}
[Fact]
public void DoubleOr_Is_Recognized()
{
- TestSingleToken("||", CSharpSymbolType.DoubleOr);
+ TestSingleToken("||", CSharpTokenType.DoubleOr);
}
[Fact]
public void ModuloAssign_Is_Recognized()
{
- TestSingleToken("%=", CSharpSymbolType.ModuloAssign);
+ TestSingleToken("%=", CSharpTokenType.ModuloAssign);
}
[Fact]
public void Colon_Is_Recognized()
{
- TestSingleToken(":", CSharpSymbolType.Colon);
+ TestSingleToken(":", CSharpTokenType.Colon);
}
[Fact]
public void Not_Is_Recognized()
{
- TestSingleToken("!", CSharpSymbolType.Not);
+ TestSingleToken("!", CSharpTokenType.Not);
}
[Fact]
public void DoubleAnd_Is_Recognized()
{
- TestSingleToken("&&", CSharpSymbolType.DoubleAnd);
+ TestSingleToken("&&", CSharpTokenType.DoubleAnd);
}
[Fact]
public void DivideAssign_Is_Recognized()
{
- TestSingleToken("/=", CSharpSymbolType.DivideAssign);
+ TestSingleToken("/=", CSharpTokenType.DivideAssign);
}
[Fact]
public void Comma_Is_Recognized()
{
- TestSingleToken(",", CSharpSymbolType.Comma);
+ TestSingleToken(",", CSharpTokenType.Comma);
}
[Fact]
public void Xor_Is_Recognized()
{
- TestSingleToken("^", CSharpSymbolType.Xor);
+ TestSingleToken("^", CSharpTokenType.Xor);
}
[Fact]
public void Decrement_Is_Recognized()
{
- TestSingleToken("--", CSharpSymbolType.Decrement);
+ TestSingleToken("--", CSharpTokenType.Decrement);
}
[Fact]
public void MultiplyAssign_Is_Recognized()
{
- TestSingleToken("*=", CSharpSymbolType.MultiplyAssign);
+ TestSingleToken("*=", CSharpTokenType.MultiplyAssign);
}
[Fact]
public void Dot_Is_Recognized()
{
- TestSingleToken(".", CSharpSymbolType.Dot);
+ TestSingleToken(".", CSharpTokenType.Dot);
}
[Fact]
public void Or_Is_Recognized()
{
- TestSingleToken("|", CSharpSymbolType.Or);
+ TestSingleToken("|", CSharpTokenType.Or);
}
[Fact]
public void Increment_Is_Recognized()
{
- TestSingleToken("++", CSharpSymbolType.Increment);
+ TestSingleToken("++", CSharpTokenType.Increment);
}
[Fact]
public void MinusAssign_Is_Recognized()
{
- TestSingleToken("-=", CSharpSymbolType.MinusAssign);
+ TestSingleToken("-=", CSharpTokenType.MinusAssign);
}
[Fact]
public void RightShift_Is_Not_Specially_Recognized()
{
TestTokenizer(">>",
- new CSharpSymbol(">", CSharpSymbolType.GreaterThan),
- new CSharpSymbol(">", CSharpSymbolType.GreaterThan));
+ new CSharpToken(">", CSharpTokenType.GreaterThan),
+ new CSharpToken(">", CSharpTokenType.GreaterThan));
}
[Fact]
public void Hash_Is_Recognized()
{
- TestSingleToken("#", CSharpSymbolType.Hash);
+ TestSingleToken("#", CSharpTokenType.Hash);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
index c4dabc0399..cf98382ef4 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
@@ -7,7 +7,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerTest : CSharpTokenizerTestBase
{
- private new CSharpSymbol IgnoreRemaining => (CSharpSymbol)base.IgnoreRemaining;
+ private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
@@ -20,8 +20,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\ra",
- new CSharpSymbol("\r", CSharpSymbolType.NewLine),
- new CSharpSymbol("\r", CSharpSymbolType.NewLine),
+ new CSharpToken("\r", CSharpTokenType.NewLine),
+ new CSharpToken("\r", CSharpTokenType.NewLine),
IgnoreRemaining);
}
@@ -30,8 +30,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\n\na",
- new CSharpSymbol("\n", CSharpSymbolType.NewLine),
- new CSharpSymbol("\n", CSharpSymbolType.NewLine),
+ new CSharpToken("\n", CSharpTokenType.NewLine),
+ new CSharpToken("\n", CSharpTokenType.NewLine),
IgnoreRemaining);
}
@@ -41,8 +41,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// NEL: Unicode "Next Line" U+0085
TestTokenizer(
"\u0085\u0085a",
- new CSharpSymbol("\u0085", CSharpSymbolType.NewLine),
- new CSharpSymbol("\u0085", CSharpSymbolType.NewLine),
+ new CSharpToken("\u0085", CSharpTokenType.NewLine),
+ new CSharpToken("\u0085", CSharpTokenType.NewLine),
IgnoreRemaining);
}
@@ -52,8 +52,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Line Separator" U+2028
TestTokenizer(
"\u2028\u2028a",
- new CSharpSymbol("\u2028", CSharpSymbolType.NewLine),
- new CSharpSymbol("\u2028", CSharpSymbolType.NewLine),
+ new CSharpToken("\u2028", CSharpTokenType.NewLine),
+ new CSharpToken("\u2028", CSharpTokenType.NewLine),
IgnoreRemaining);
}
@@ -63,8 +63,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Paragraph Separator" U+2029
TestTokenizer(
"\u2029\u2029a",
- new CSharpSymbol("\u2029", CSharpSymbolType.NewLine),
- new CSharpSymbol("\u2029", CSharpSymbolType.NewLine),
+ new CSharpToken("\u2029", CSharpTokenType.NewLine),
+ new CSharpToken("\u2029", CSharpTokenType.NewLine),
IgnoreRemaining);
}
@@ -73,8 +73,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n\r\na",
- new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
- new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
+ new CSharpToken("\r\n", CSharpTokenType.NewLine),
+ new CSharpToken("\r\n", CSharpTokenType.NewLine),
IgnoreRemaining);
}
@@ -83,15 +83,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
" \f\t\u000B \n ",
- new CSharpSymbol(" \f\t\u000B ", CSharpSymbolType.WhiteSpace),
- new CSharpSymbol("\n", CSharpSymbolType.NewLine),
- new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace));
+ new CSharpToken(" \f\t\u000B ", CSharpTokenType.WhiteSpace),
+ new CSharpToken("\n", CSharpTokenType.NewLine),
+ new CSharpToken(" ", CSharpTokenType.WhiteSpace));
}
[Fact]
public void Transition_Is_Recognized()
{
- TestSingleToken("@", CSharpSymbolType.Transition);
+ TestSingleToken("@", CSharpTokenType.Transition);
}
[Fact]
@@ -99,8 +99,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@(",
- new CSharpSymbol("@", CSharpSymbolType.Transition),
- new CSharpSymbol("(", CSharpSymbolType.LeftParenthesis));
+ new CSharpToken("@", CSharpTokenType.Transition),
+ new CSharpToken("(", CSharpTokenType.LeftParenthesis));
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
index a22d0d5ef1..6869138c1e 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
@@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{
- private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown);
+ private static CSharpToken _ignoreRemaining = new CSharpToken(string.Empty, CSharpTokenType.Unknown);
internal override object IgnoreRemaining
{
@@ -17,14 +17,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source);
}
- internal void TestSingleToken(string text, CSharpSymbolType expectedSymbolType)
+ internal void TestSingleToken(string text, CSharpTokenType expectedTokenType)
{
- TestTokenizer(text, new CSharpSymbol(text, expectedSymbolType));
+ TestTokenizer(text, new CSharpToken(text, expectedTokenType));
}
- internal void TestTokenizer(string input, params CSharpSymbol[] expectedSymbols)
+ internal void TestTokenizer(string input, params CSharpToken[] expectedTokens)
{
- base.TestTokenizer(input, expectedSymbols);
+ base.TestTokenizer(input, expectedTokens);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
index dcbbd2b1ef..7a1e0c070a 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
@@ -12,12 +12,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n @something \r\n @this is ignored",
- new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
- new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
- new CSharpSymbol("@", CSharpSymbolType.Transition),
- new CSharpSymbol("something", CSharpSymbolType.Identifier),
- new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
- new CSharpSymbol("\r\n", CSharpSymbolType.NewLine));
+ new CSharpToken("\r\n", CSharpTokenType.NewLine),
+ new CSharpToken(" ", CSharpTokenType.WhiteSpace),
+ new CSharpToken("@", CSharpTokenType.Transition),
+ new CSharpToken("something", CSharpTokenType.Identifier),
+ new CSharpToken(" ", CSharpTokenType.WhiteSpace),
+ new CSharpToken("\r\n", CSharpTokenType.NewLine));
}
[Fact]
@@ -25,18 +25,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@*included*@\r\n @something \"value\"\r\n @this is ignored",
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol("included", CSharpSymbolType.RazorComment),
- new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
- new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
- new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
- new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
- new CSharpSymbol("@", CSharpSymbolType.Transition),
- new CSharpSymbol("something", CSharpSymbolType.Identifier),
- new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace),
- new CSharpSymbol("\"value\"", CSharpSymbolType.StringLiteral),
- new CSharpSymbol("\r\n", CSharpSymbolType.NewLine));
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken("included", CSharpTokenType.RazorComment),
+ new CSharpToken("*", CSharpTokenType.RazorCommentStar),
+ new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
+ new CSharpToken("\r\n", CSharpTokenType.NewLine),
+ new CSharpToken(" ", CSharpTokenType.WhiteSpace),
+ new CSharpToken("@", CSharpTokenType.Transition),
+ new CSharpToken("something", CSharpTokenType.Identifier),
+ new CSharpToken(" ", CSharpTokenType.WhiteSpace),
+ new CSharpToken("\"value\"", CSharpTokenType.StringLiteral),
+ new CSharpToken("\r\n", CSharpTokenType.NewLine));
}
internal override object CreateTokenizer(ITextDocument source)
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
index b75874b681..cffd0a0d0d 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
@@ -12,9 +12,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n
Ignored
",
- new HtmlSymbol("\r\n", HtmlSymbolType.NewLine),
- new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace),
- new HtmlSymbol("<", HtmlSymbolType.OpenAngle));
+ new HtmlToken("\r\n", HtmlTokenType.NewLine),
+ new HtmlToken(" ", HtmlTokenType.WhiteSpace),
+ new HtmlToken("<", HtmlTokenType.OpenAngle));
}
[Fact]
@@ -22,15 +22,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n @*included*@
Ignored
",
- new HtmlSymbol("\r\n", HtmlSymbolType.NewLine),
- new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace),
- new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
- new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
- new HtmlSymbol("included", HtmlSymbolType.RazorComment),
- new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
- new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
- new HtmlSymbol(" ", HtmlSymbolType.WhiteSpace),
- new HtmlSymbol("<", HtmlSymbolType.OpenAngle));
+ new HtmlToken("\r\n", HtmlTokenType.NewLine),
+ new HtmlToken(" ", HtmlTokenType.WhiteSpace),
+ new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
+ new HtmlToken("*", HtmlTokenType.RazorCommentStar),
+ new HtmlToken("included", HtmlTokenType.RazorComment),
+ new HtmlToken("*", HtmlTokenType.RazorCommentStar),
+ new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
+ new HtmlToken(" ", HtmlTokenType.WhiteSpace),
+ new HtmlToken("<", HtmlTokenType.OpenAngle));
}
internal override object CreateTokenizer(ITextDocument source)
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
index cc12a33914..839eb92353 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
@@ -8,69 +8,69 @@ namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
{
public class HtmlMarkupParserTests
{
- private static readonly HtmlSymbol doubleHyphenSymbol = new HtmlSymbol("--", HtmlSymbolType.DoubleHyphen);
+ private static readonly HtmlToken doubleHyphenToken = new HtmlToken("--", HtmlTokenType.DoubleHyphen);
- public static IEnumerable