()
- .SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token)) // Skip prefix
- .TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken))
+ .SkipWhile(token => !HtmlMarkupParser.IsValidAttributeNameToken(token.Green)) // Skip prefix
+ .TakeWhile(nameToken => HtmlMarkupParser.IsValidAttributeNameToken(nameToken.Green))
.Select(nameToken => nameToken.Content);
var name = string.Concat(nameTokens);
@@ -362,12 +362,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var result = CreateTryParseResult(name, descriptors, processedBoundAttributeNames);
var firstChild = builder.Children[0] as Span;
- if (firstChild != null && firstChild.Tokens[0] is HtmlToken)
+ if (firstChild != null)
{
- var htmlToken = firstChild.Tokens[firstChild.Tokens.Count - 1] as HtmlToken;
- switch (htmlToken.Type)
+ var token = firstChild.Tokens[firstChild.Tokens.Count - 1];
+ switch (token.Kind)
{
- case HtmlTokenType.Equals:
+ case SyntaxKind.Equals:
if (builder.Children.Count == 2 &&
builder.Children[1] is Span value &&
value.Kind == SpanKindInternal.Markup)
@@ -385,10 +385,10 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
result.AttributeStructure = AttributeStructure.DoubleQuotes;
}
break;
- case HtmlTokenType.DoubleQuote:
+ case SyntaxKind.DoubleQuote:
result.AttributeStructure = AttributeStructure.DoubleQuotes;
break;
- case HtmlTokenType.SingleQuote:
+ case SyntaxKind.SingleQuote:
result.AttributeStructure = AttributeStructure.SingleQuotes;
break;
default:
@@ -408,8 +408,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// In some malformed cases e.g. : ITokenizer
- where TTokenType : struct
- where TToken : TokenBase
+ internal abstract class Tokenizer : ITokenizer
{
protected Tokenizer(ITextDocument source)
{
@@ -31,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected int? CurrentState { get; set; }
- protected TToken CurrentToken { get; private set; }
+ protected SyntaxToken CurrenSyntaxToken { get; private set; }
public ITextDocument Source { get; private set; }
@@ -42,9 +41,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return Source.Peek() == -1; }
}
- public abstract TTokenType RazorCommentStarType { get; }
- public abstract TTokenType RazorCommentType { get; }
- public abstract TTokenType RazorCommentTransitionType { get; }
+ public abstract SyntaxKind RazorCommentStarKind { get; }
+ public abstract SyntaxKind RazorCommentKind { get; }
+ public abstract SyntaxKind RazorCommentTransitionKind { get; }
protected bool HaveContent
{
@@ -64,16 +63,16 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public SourceLocation CurrentStart { get; private set; }
- protected abstract TToken CreateToken(string content, TTokenType type, IReadOnlyList errors);
+ protected abstract SyntaxToken CreateToken(string content, SyntaxKind type, IReadOnlyList errors);
protected abstract StateResult Dispatch();
- IToken ITokenizer.NextToken()
+ SyntaxToken ITokenizer.NextToken()
{
return NextToken();
}
- public virtual TToken NextToken()
+ public virtual SyntaxToken NextToken()
{
// Post-Condition: Buffer should be empty at the start of Next()
Debug.Assert(Buffer.Length == 0);
@@ -95,7 +94,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token;
}
- protected virtual TToken Turn()
+ protected virtual SyntaxToken Turn()
{
if (CurrentState != null)
{
@@ -105,19 +104,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var next = Dispatch();
CurrentState = next.State;
- CurrentToken = next.Result;
+ CurrenSyntaxToken = next.Result;
}
- while (CurrentState != null && CurrentToken == null);
+ while (CurrentState != null && CurrenSyntaxToken == null);
if (CurrentState == null)
{
- return default(TToken); // Terminated
+ return default(SyntaxToken); // Terminated
}
- return CurrentToken;
+ return CurrenSyntaxToken;
}
- return default(TToken);
+ return default(SyntaxToken);
}
public void Reset()
@@ -149,7 +148,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to
/// should invoke the provided state.
///
- protected StateResult Transition(int state, TToken result)
+ protected StateResult Transition(int state, SyntaxToken result)
{
return new StateResult(state, result);
}
@@ -159,7 +158,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new StateResult((int)state, result: null);
}
- protected StateResult Transition(RazorCommentTokenizerState state, TToken result)
+ protected StateResult Transition(RazorCommentTokenizerState state, SyntaxToken result)
{
return new StateResult((int)state, result);
}
@@ -180,12 +179,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// Returns a result containing the specified output and indicating that the next call to
/// should re-invoke the current state.
///
- protected StateResult Stay(TToken result)
+ protected StateResult Stay(SyntaxToken result)
{
return new StateResult(CurrentState, result);
}
- protected TToken Single(TTokenType type)
+ protected SyntaxToken Single(SyntaxKind type)
{
TakeCurrent();
return EndToken(type);
@@ -199,9 +198,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
CurrentStart = CurrentLocation;
}
- protected TToken EndToken(TTokenType type)
+ protected SyntaxToken EndToken(SyntaxKind type)
{
- TToken token = null;
+ SyntaxToken token = null;
if (HaveContent)
{
// Perf: Don't allocate a new errors array unless necessary.
@@ -222,7 +221,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return token;
}
- protected virtual string GetTokenContent(TTokenType type)
+ protected virtual string GetTokenContent(SyntaxKind type)
{
return Buffer.ToString();
}
@@ -278,7 +277,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
AssertCurrent('*');
TakeCurrent();
- return Transition(1002, EndToken(RazorCommentStarType));
+ return Transition(1002, EndToken(RazorCommentStarKind));
}
protected StateResult RazorCommentBody()
@@ -292,7 +291,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
return Transition(
RazorCommentTokenizerState.StarAfterRazorCommentBody,
- EndToken(RazorCommentType));
+ EndToken(RazorCommentKind));
}
else
{
@@ -306,7 +305,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- return Transition(StartState, EndToken(RazorCommentType));
+ return Transition(StartState, EndToken(RazorCommentKind));
}
protected StateResult StarAfterRazorCommentBody()
@@ -315,14 +314,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
TakeCurrent();
return Transition(
RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
- EndToken(RazorCommentStarType));
+ EndToken(RazorCommentStarKind));
}
protected StateResult AtTokenAfterRazorCommentBody()
{
AssertCurrent('@');
TakeCurrent();
- return Transition(StartState, EndToken(RazorCommentTransitionType));
+ return Transition(StartState, EndToken(RazorCommentTransitionKind));
}
///
@@ -397,7 +396,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected struct StateResult
{
- public StateResult(int? state, TToken result)
+ public StateResult(int? state, SyntaxToken result)
{
State = state;
Result = result;
@@ -405,7 +404,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public int? State { get; }
- public TToken Result { get; }
+ public SyntaxToken Result { get; }
}
private static LookaheadToken BeginLookahead(ITextBuffer buffer)
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
index f7b054ecff..5168dbe79c 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerBackedParser.cs
@@ -9,20 +9,18 @@ using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal abstract partial class TokenizerBackedParser : ParserBase
- where TTokenType : struct
- where TTokenizer : Tokenizer
- where TToken : TokenBase
+ internal abstract class TokenizerBackedParser : ParserBase
+ where TTokenizer : Tokenizer
{
- private readonly TokenizerView _tokenizer;
+ private readonly TokenizerView _tokenizer;
- protected TokenizerBackedParser(LanguageCharacteristics language, ParserContext context)
+ protected TokenizerBackedParser(LanguageCharacteristics language, ParserContext context)
: base(context)
{
Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source);
- _tokenizer = new TokenizerView(languageTokenizer);
+ _tokenizer = new TokenizerView(languageTokenizer);
Span = new SpanBuilder(CurrentLocation);
}
@@ -32,14 +30,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected Action SpanConfig { get; set; }
- protected TToken CurrentToken
+ protected SyntaxToken CurrentToken
{
get { return _tokenizer.Current; }
}
- protected SyntaxToken CurrentSyntaxToken => CurrentToken?.SyntaxToken;
-
- protected TToken PreviousToken { get; private set; }
+ protected SyntaxToken PreviousToken { get; private set; }
protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
@@ -50,7 +46,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
get { return _tokenizer.EndOfFile; }
}
- protected LanguageCharacteristics Language { get; }
+ protected LanguageCharacteristics Language { get; }
protected virtual void HandleEmbeddedTransition()
{
@@ -63,21 +59,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public override void BuildSpan(SpanBuilder span, SourceLocation start, string content)
{
- foreach (IToken sym in Language.TokenizeString(start, content))
+ foreach (var token in Language.TokenizeString(start, content))
{
- span.Accept(sym);
+ span.Accept(token);
}
}
protected void Initialize(SpanBuilder span)
{
- if (SpanConfig != null)
- {
- SpanConfig(span);
- }
+ SpanConfig?.Invoke(span);
}
- protected TToken Lookahead(int count)
+ protected SyntaxToken Lookahead(int count)
{
if (count < 0)
{
@@ -89,7 +82,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
// We add 1 in order to store the current token.
- var tokens = new TToken[count + 1];
+ var tokens = new SyntaxToken[count + 1];
var currentToken = CurrentToken;
tokens[0] = currentToken;
@@ -120,7 +113,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// A predicate accepting the token being evaluated and the list of tokens which have been looped through.
/// true, if the condition was met. false - if the condition wasn't met and the last token has already been processed.
/// The list of previous tokens is passed in the reverse order. So the last processed element will be the first one in the list.
- protected bool LookaheadUntil(Func, bool> condition)
+ protected bool LookaheadUntil(Func, bool> condition)
{
if (condition == null)
{
@@ -129,7 +122,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var matchFound = false;
- var tokens = new List();
+ var tokens = new List();
tokens.Add(CurrentToken);
while (true)
@@ -168,14 +161,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Helpers
[Conditional("DEBUG")]
- internal void Assert(TTokenType expectedType)
+ internal void Assert(SyntaxKind expectedType)
{
- Debug.Assert(!EndOfFile && TokenTypeEquals(CurrentToken.Type, expectedType));
+ Debug.Assert(!EndOfFile && TokenKindEquals(CurrentToken.Kind, expectedType));
}
- abstract protected bool TokenTypeEquals(TTokenType x, TTokenType y);
+ protected abstract bool TokenKindEquals(SyntaxKind x, SyntaxKind y);
- protected internal void PutBack(TToken token)
+ protected internal void PutBack(SyntaxToken token)
{
if (token != null)
{
@@ -185,7 +178,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
///
/// Put the specified tokens back in the input stream. The provided list MUST be in the ORDER THE TOKENS WERE READ. The
- /// list WILL be reversed and the Putback(TToken) will be called on each item.
+ /// list WILL be reversed and the Putback(SyntaxToken) will be called on each item.
///
///
/// If a document contains tokens: a, b, c, d, e, f
@@ -194,9 +187,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
/// that is the correct format for providing to this method. The caller of this method would,
/// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order
///
- protected internal void PutBack(IEnumerable tokens)
+ protected internal void PutBack(IEnumerable tokens)
{
- foreach (TToken token in tokens.Reverse())
+ foreach (var token in tokens.Reverse())
{
PutBack(token);
}
@@ -212,7 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected internal bool Balance(BalancingModes mode)
{
- var left = CurrentToken.Type;
+ var left = CurrentToken.Kind;
var right = Language.FlipBracket(left);
var start = CurrentStart;
AcceptAndMoveNext();
@@ -228,21 +221,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Balance(mode, left, right, start);
}
- protected internal bool Balance(BalancingModes mode, TTokenType left, TTokenType right, SourceLocation start)
+ protected internal bool Balance(BalancingModes mode, SyntaxKind left, SyntaxKind right, SourceLocation start)
{
var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1;
if (!EndOfFile)
{
- var syms = new List();
+ var tokens = new List();
do
{
if (IsAtEmbeddedTransition(
(mode & BalancingModes.AllowCommentsAndTemplates) == BalancingModes.AllowCommentsAndTemplates,
(mode & BalancingModes.AllowEmbeddedTransitions) == BalancingModes.AllowEmbeddedTransitions))
{
- Accept(syms);
- syms.Clear();
+ Accept(tokens);
+ tokens.Clear();
HandleEmbeddedTransition();
// Reset backtracking since we've already outputted some spans.
@@ -258,7 +251,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
if (nesting > 0)
{
- syms.Add(CurrentToken);
+ tokens.Add(CurrentToken);
}
}
while (nesting > 0 && NextToken());
@@ -280,29 +273,29 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
else
{
- Accept(syms);
+ Accept(tokens);
}
}
else
{
// Accept all the tokens we saw
- Accept(syms);
+ Accept(tokens);
}
}
return nesting == 0;
}
- protected internal bool NextIs(TTokenType type)
+ protected internal bool NextIs(SyntaxKind type)
{
- return NextIs(sym => sym != null && TokenTypeEquals(type, sym.Type));
+ return NextIs(token => token != null && TokenKindEquals(type, token.Kind));
}
- protected internal bool NextIs(params TTokenType[] types)
+ protected internal bool NextIs(params SyntaxKind[] types)
{
- return NextIs(sym => sym != null && types.Any(t => TokenTypeEquals(t, sym.Type)));
+ return NextIs(token => token != null && types.Any(t => TokenKindEquals(t, token.Kind)));
}
- protected internal bool NextIs(Func condition)
+ protected internal bool NextIs(Func condition)
{
var cur = CurrentToken;
if (NextToken())
@@ -322,14 +315,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return false;
}
- protected internal bool Was(TTokenType type)
+ protected internal bool Was(SyntaxKind type)
{
- return PreviousToken != null && TokenTypeEquals(PreviousToken.Type, type);
+ return PreviousToken != null && TokenKindEquals(PreviousToken.Kind, type);
}
- protected internal bool At(TTokenType type)
+ protected internal bool At(SyntaxKind type)
{
- return !EndOfFile && CurrentToken != null && TokenTypeEquals(CurrentToken.Type, type);
+ return !EndOfFile && CurrentToken != null && TokenKindEquals(CurrentToken.Kind, type);
}
protected internal bool AcceptAndMoveNext()
@@ -338,11 +331,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return NextToken();
}
- protected TToken AcceptSingleWhiteSpaceCharacter()
+ protected SyntaxToken AcceptSingleWhiteSpaceCharacter()
{
if (Language.IsWhiteSpace(CurrentToken))
{
- Tuple pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
+ var pair = Language.SplitToken(CurrentToken, 1, Language.GetKnownTokenType(KnownTokenType.WhiteSpace));
Accept(pair.Item1);
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
NextToken();
@@ -351,19 +344,19 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return null;
}
- protected internal void Accept(IEnumerable tokens)
+ protected internal void Accept(IEnumerable tokens)
{
- foreach (TToken token in tokens)
+ foreach (var token in tokens)
{
Accept(token);
}
}
- protected internal void Accept(TToken token)
+ protected internal void Accept(SyntaxToken token)
{
if (token != null)
{
- foreach (var error in token.Errors)
+ foreach (var error in token.GetDiagnostics())
{
Context.ErrorSink.OnError(error);
}
@@ -372,11 +365,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
}
}
- protected internal bool AcceptAll(params TTokenType[] types)
+ protected internal bool AcceptAll(params SyntaxKind[] kinds)
{
- foreach (TTokenType type in types)
+ foreach (var kind in kinds)
{
- if (CurrentToken == null || !TokenTypeEquals(CurrentToken.Type, type))
+ if (CurrentToken == null || !TokenKindEquals(CurrentToken.Kind, kind))
{
return false;
}
@@ -442,7 +435,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected IDisposable PushSpanConfig(Action> newConfig)
{
- Action old = SpanConfig;
+ var old = SpanConfig;
ConfigureSpan(newConfig);
return new DisposableAction(() => SpanConfig = old);
}
@@ -455,7 +448,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
protected void ConfigureSpan(Action> config)
{
- Action prev = SpanConfig;
+ var prev = SpanConfig;
if (config == null)
{
SpanConfig = null;
@@ -472,9 +465,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
Expected(Language.GetKnownTokenType(type));
}
- protected internal void Expected(params TTokenType[] types)
+ protected internal void Expected(params SyntaxKind[] types)
{
- Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Type));
+ Debug.Assert(!EndOfFile && CurrentToken != null && types.Contains(CurrentToken.Kind));
AcceptAndMoveNext();
}
@@ -483,7 +476,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return Optional(Language.GetKnownTokenType(type));
}
- protected internal bool Optional(TTokenType type)
+ protected internal bool Optional(SyntaxKind type)
{
if (At(type))
{
@@ -503,61 +496,61 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return true;
}
- protected internal void AcceptWhile(TTokenType type)
+ protected internal void AcceptWhile(SyntaxKind type)
{
- AcceptWhile(sym => TokenTypeEquals(type, sym.Type));
+ AcceptWhile(token => TokenKindEquals(type, token.Kind));
}
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
- protected internal void AcceptWhile(TTokenType type1, TTokenType type2)
+ protected internal void AcceptWhile(SyntaxKind type1, SyntaxKind type2)
{
- AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type));
+ AcceptWhile(token => TokenKindEquals(type1, token.Kind) || TokenKindEquals(type2, token.Kind));
}
- protected internal void AcceptWhile(TTokenType type1, TTokenType type2, TTokenType type3)
+ protected internal void AcceptWhile(SyntaxKind type1, SyntaxKind type2, SyntaxKind type3)
{
- AcceptWhile(sym => TokenTypeEquals(type1, sym.Type) || TokenTypeEquals(type2, sym.Type) || TokenTypeEquals(type3, sym.Type));
+ AcceptWhile(token => TokenKindEquals(type1, token.Kind) || TokenKindEquals(type2, token.Kind) || TokenKindEquals(type3, token.Kind));
}
- protected internal void AcceptWhile(params TTokenType[] types)
+ protected internal void AcceptWhile(params SyntaxKind[] types)
{
- AcceptWhile(sym => types.Any(expected => TokenTypeEquals(expected, sym.Type)));
+ AcceptWhile(token => types.Any(expected => TokenKindEquals(expected, token.Kind)));
}
- protected internal void AcceptUntil(TTokenType type)
+ protected internal void AcceptUntil(SyntaxKind type)
{
- AcceptWhile(sym => !TokenTypeEquals(type, sym.Type));
+ AcceptWhile(token => !TokenKindEquals(type, token.Kind));
}
// We want to avoid array allocations and enumeration where possible, so we use the same technique as string.Format
- protected internal void AcceptUntil(TTokenType type1, TTokenType type2)
+ protected internal void AcceptUntil(SyntaxKind type1, SyntaxKind type2)
{
- AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type));
+ AcceptWhile(token => !TokenKindEquals(type1, token.Kind) && !TokenKindEquals(type2, token.Kind));
}
- protected internal void AcceptUntil(TTokenType type1, TTokenType type2, TTokenType type3)
+ protected internal void AcceptUntil(SyntaxKind type1, SyntaxKind type2, SyntaxKind type3)
{
- AcceptWhile(sym => !TokenTypeEquals(type1, sym.Type) && !TokenTypeEquals(type2, sym.Type) && !TokenTypeEquals(type3, sym.Type));
+ AcceptWhile(token => !TokenKindEquals(type1, token.Kind) && !TokenKindEquals(type2, token.Kind) && !TokenKindEquals(type3, token.Kind));
}
- protected internal void AcceptUntil(params TTokenType[] types)
+ protected internal void AcceptUntil(params SyntaxKind[] types)
{
- AcceptWhile(sym => types.All(expected => !TokenTypeEquals(expected, sym.Type)));
+ AcceptWhile(token => types.All(expected => !TokenKindEquals(expected, token.Kind)));
}
- protected internal void AcceptWhile(Func condition)
+ protected internal void AcceptWhile(Func condition)
{
Accept(ReadWhileLazy(condition));
}
- protected internal IEnumerable ReadWhile(Func condition)
+ protected internal IEnumerable ReadWhile(Func condition)
{
return ReadWhileLazy(condition).ToList();
}
- protected TToken AcceptWhiteSpaceInLines()
+ protected SyntaxToken AcceptWhiteSpaceInLines()
{
- TToken lastWs = null;
+ SyntaxToken lastWs = null;
while (Language.IsWhiteSpace(CurrentToken) || Language.IsNewLine(CurrentToken))
{
// Capture the previous whitespace node
@@ -591,7 +584,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Don't open this to sub classes because it's lazy but it looks eager.
// You have to advance the Enumerable to read the next characters.
- internal IEnumerable ReadWhileLazy(Func condition)
+ internal IEnumerable ReadWhileLazy(Func condition)
{
while (EnsureCurrent() && condition(CurrentToken))
{
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
index 2b290bda5c..ef481af1ac 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/TokenizerView.cs
@@ -1,12 +1,12 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
+
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class TokenizerView
- where TTokenType : struct
- where TTokenizer : Tokenizer
- where TToken : TokenBase
+ internal class TokenizerView
+ where TTokenizer : Tokenizer
{
public TokenizerView(TTokenizer tokenizer)
{
@@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public TTokenizer Tokenizer { get; private set; }
public bool EndOfFile { get; private set; }
- public TToken Current { get; private set; }
+ public SyntaxToken Current { get; private set; }
public ITextDocument Source
{
@@ -29,7 +29,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return !EndOfFile;
}
- public void PutBack(TToken token)
+ public void PutBack(SyntaxToken token)
{
Source.Position -= token.Content.Length;
Current = null;
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/GreenNode.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/GreenNode.cs
index e90374ad2b..e14cbc0594 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/GreenNode.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/GreenNode.cs
@@ -377,6 +377,70 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
}
#endregion
+ #region Equivalence
+ public virtual bool IsEquivalentTo(GreenNode other)
+ {
+ if (this == other)
+ {
+ return true;
+ }
+
+ if (other == null)
+ {
+ return false;
+ }
+
+ return EquivalentToInternal(this, other);
+ }
+
+ private static bool EquivalentToInternal(GreenNode node1, GreenNode node2)
+ {
+ if (node1.Kind != node2.Kind)
+ {
+ // A single-element list is usually represented as just a single node,
+ // but can be represented as a List node with one child. Move to that
+ // child if necessary.
+ if (node1.IsList && node1.SlotCount == 1)
+ {
+ node1 = node1.GetSlot(0);
+ }
+
+ if (node2.IsList && node2.SlotCount == 1)
+ {
+ node2 = node2.GetSlot(0);
+ }
+
+ if (node1.Kind != node2.Kind)
+ {
+ return false;
+ }
+ }
+
+ if (node1.FullWidth != node2.FullWidth)
+ {
+ return false;
+ }
+
+ var n = node1.SlotCount;
+ if (n != node2.SlotCount)
+ {
+ return false;
+ }
+
+ for (var i = 0; i < n; i++)
+ {
+ var node1Child = node1.GetSlot(i);
+ var node2Child = node2.GetSlot(i);
+ if (node1Child != null && node2Child != null && !node1Child.IsEquivalentTo(node2Child))
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ #endregion
+
#region Factories
public virtual GreenNode CreateList(IEnumerable nodes, bool alwaysCreateListNode = false)
{
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/HtmlTextTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/HtmlTextTokenSyntax.cs
deleted file mode 100644
index 9130b5c5d4..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/HtmlTextTokenSyntax.cs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax
-{
- internal class HtmlTextTokenSyntax : SyntaxToken
- {
- internal HtmlTextTokenSyntax(GreenNode green, SyntaxNode parent, int position)
- : base(green, parent, position)
- {
- }
-
- internal new InternalSyntax.HtmlTextTokenSyntax Green => (InternalSyntax.HtmlTextTokenSyntax)base.Green;
-
- public string Value => Text;
-
- internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.HtmlTextTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as HtmlTextTokenSyntax;
- }
-
- internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.HtmlTextTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as HtmlTextTokenSyntax;
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/HtmlTextTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/HtmlTextTokenSyntax.cs
deleted file mode 100644
index 68d6eecd97..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/HtmlTextTokenSyntax.cs
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
-{
- internal class HtmlTextTokenSyntax : SyntaxToken
- {
- internal HtmlTextTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
- : base(SyntaxKind.HtmlTextLiteralToken, text, null, null, diagnostics, null)
- {
- }
-
- internal HtmlTextTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(SyntaxKind.HtmlTextLiteralToken, text, leadingTrivia, trailingTrivia)
- {
- }
-
- protected HtmlTextTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(kind, name, leadingTrivia, trailingTrivia)
- {
- }
-
- protected HtmlTextTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
- : base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
- {
- }
-
- internal override SyntaxNode CreateRed(SyntaxNode parent, int position)
- {
- return new Syntax.HtmlTextTokenSyntax(this, parent, position);
- }
-
- public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
- {
- return new HtmlTextTokenSyntax(Kind, Text, trivia, TrailingTrivia);
- }
-
- public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
- {
- return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, trivia);
- }
-
- internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
- {
- return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
- }
-
- internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
- {
- return new HtmlTextTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/NewLineTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/NewLineTokenSyntax.cs
deleted file mode 100644
index 97d47e9fb9..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/NewLineTokenSyntax.cs
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
-{
- internal class NewLineTokenSyntax : SyntaxToken
- {
- internal NewLineTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
- : base(SyntaxKind.NewLine, text, null, null, diagnostics, null)
- {
- }
-
- internal NewLineTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(SyntaxKind.NewLine, text, leadingTrivia, trailingTrivia)
- {
- }
-
- protected NewLineTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(kind, name, leadingTrivia, trailingTrivia)
- {
- }
-
- protected NewLineTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
- : base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
- {
- }
-
- internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.NewLineTokenSyntax(this, parent, position);
-
- public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
- {
- return new NewLineTokenSyntax(Kind, Text, trivia, TrailingTrivia);
- }
-
- public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
- {
- return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, trivia);
- }
-
- internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
- {
- return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
- }
-
- internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
- {
- return new NewLineTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/PunctuationSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/PunctuationSyntax.cs
deleted file mode 100644
index 0c2bb06b61..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/PunctuationSyntax.cs
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
-{
- internal class PunctuationSyntax : SyntaxToken
- {
- internal PunctuationSyntax(SyntaxKind kind, string name, RazorDiagnostic[] diagnostics)
- : this(kind, name, null, null, diagnostics, null)
- {
- }
-
- internal PunctuationSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : this(kind, name, leadingTrivia, trailingTrivia, null, null)
- {
- }
-
- internal PunctuationSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
- : base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
- {
- }
-
- internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.PunctuationSyntax(this, parent, position);
-
- public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
- {
- return new PunctuationSyntax(Kind, Text, trivia, TrailingTrivia);
- }
-
- public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
- {
- return new PunctuationSyntax(Kind, Text, LeadingTrivia, trivia);
- }
-
- internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
- {
- return new PunctuationSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
- }
-
- internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
- {
- return new PunctuationSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxFactory.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxFactory.cs
index a6f5ce2e1f..78b7f447d2 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxFactory.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxFactory.cs
@@ -1,6 +1,9 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using System.Collections.Generic;
+using System.Linq;
+
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
internal static class SyntaxFactory
@@ -10,29 +13,14 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return new HtmlTextSyntax(textTokens.Node);
}
- internal static HtmlTextTokenSyntax HtmlTextToken(string text, params RazorDiagnostic[] diagnostics)
+ internal static SyntaxToken Token(SyntaxKind kind, string content, IEnumerable diagnostics)
{
- return new HtmlTextTokenSyntax(text, diagnostics);
+ return Token(kind, content, diagnostics.ToArray());
}
- internal static WhitespaceTokenSyntax WhitespaceToken(string text, params RazorDiagnostic[] diagnostics)
+ internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
{
- return new WhitespaceTokenSyntax(text, diagnostics);
- }
-
- internal static NewLineTokenSyntax NewLineToken(string text, params RazorDiagnostic[] diagnostics)
- {
- return new NewLineTokenSyntax(text, diagnostics);
- }
-
- internal static PunctuationSyntax Punctuation(SyntaxKind syntaxKind, string text, params RazorDiagnostic[] diagnostics)
- {
- return new PunctuationSyntax(syntaxKind, text, diagnostics);
- }
-
- internal static UnknownTokenSyntax UnknownToken(string text, params RazorDiagnostic[] diagnostics)
- {
- return new UnknownTokenSyntax(text, diagnostics);
+ return new SyntaxToken(kind, content, diagnostics);
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxToken.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxToken.cs
index d9167704b5..b7471591c8 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxToken.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxToken.cs
@@ -4,32 +4,39 @@
using System;
using System.Collections.Generic;
using System.IO;
+using Microsoft.AspNetCore.Razor.Language.Legacy;
namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
- internal abstract class SyntaxToken : GreenNode
+ internal class SyntaxToken : GreenNode
{
- internal SyntaxToken(SyntaxKind tokenKind, string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(tokenKind, text.Length)
+ internal SyntaxToken(SyntaxKind kind, string content, RazorDiagnostic[] diagnostics)
+ : base(kind, content.Length, diagnostics, annotations: null)
{
- Text = text;
+ Content = content;
+ }
+
+ internal SyntaxToken(SyntaxKind kind, string content, GreenNode leadingTrivia, GreenNode trailingTrivia)
+ : base(kind, content.Length)
+ {
+ Content = content;
LeadingTrivia = leadingTrivia;
AdjustFlagsAndWidth(leadingTrivia);
TrailingTrivia = trailingTrivia;
AdjustFlagsAndWidth(trailingTrivia);
}
- internal SyntaxToken(SyntaxKind tokenKind, string text, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
- : base(tokenKind, text.Length, diagnostics, annotations)
+ internal SyntaxToken(SyntaxKind kind, string content, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
+ : base(kind, content.Length, diagnostics, annotations)
{
- Text = text;
+ Content = content;
LeadingTrivia = leadingTrivia;
AdjustFlagsAndWidth(leadingTrivia);
TrailingTrivia = trailingTrivia;
AdjustFlagsAndWidth(trailingTrivia);
}
- public string Text { get; }
+ public string Content { get; }
public GreenNode LeadingTrivia { get; }
@@ -37,7 +44,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
internal override bool IsToken => true;
- public override int Width => Text.Length;
+ public override int Width => Content.Length;
+
+ internal override SyntaxNode CreateRed(SyntaxNode parent, int position)
+ {
+ return new Syntax.SyntaxToken(this, parent, position);
+ }
protected override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
@@ -50,7 +62,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
}
}
- writer.Write(Text);
+ writer.Write(Content);
if (trailing)
{
@@ -87,14 +99,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return TokenWithLeadingTrivia(trivia);
}
- public abstract SyntaxToken TokenWithLeadingTrivia(GreenNode trivia);
+ public virtual SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
+ {
+ return new SyntaxToken(Kind, Content, trivia, TrailingTrivia, GetDiagnostics(), GetAnnotations());
+ }
public sealed override GreenNode WithTrailingTrivia(GreenNode trivia)
{
return TokenWithTrailingTrivia(trivia);
}
- public abstract SyntaxToken TokenWithTrailingTrivia(GreenNode trivia);
+ public virtual SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
+ {
+ return new SyntaxToken(Kind, Content, LeadingTrivia, trivia, GetDiagnostics(), GetAnnotations());
+ }
+
+ internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
+ {
+ return new SyntaxToken(Kind, Content, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
+ }
+
+ internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
+ {
+ return new SyntaxToken(Kind, Content, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
+ }
protected override sealed int GetSlotCount()
{
@@ -103,7 +131,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
internal override sealed GreenNode GetSlot(int index)
{
- throw new InvalidOperationException();
+ throw new InvalidOperationException("Tokens don't have slots.");
}
internal override GreenNode Accept(SyntaxVisitor visitor)
@@ -111,9 +139,56 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
return visitor.VisitSyntaxToken(this);
}
+ public override bool IsEquivalentTo(GreenNode other)
+ {
+ if (!base.IsEquivalentTo(other))
+ {
+ return false;
+ }
+
+ var otherToken = (SyntaxToken)other;
+
+ if (Content != otherToken.Content)
+ {
+ return false;
+ }
+
+ var thisLeading = GetLeadingTrivia();
+ var otherLeading = otherToken.GetLeadingTrivia();
+ if (thisLeading != otherLeading)
+ {
+ if (thisLeading == null || otherLeading == null)
+ {
+ return false;
+ }
+
+ if (!thisLeading.IsEquivalentTo(otherLeading))
+ {
+ return false;
+ }
+ }
+
+ var thisTrailing = GetTrailingTrivia();
+ var otherTrailing = otherToken.GetTrailingTrivia();
+ if (thisTrailing != otherTrailing)
+ {
+ if (thisTrailing == null || otherTrailing == null)
+ {
+ return false;
+ }
+
+ if (!thisTrailing.IsEquivalentTo(otherTrailing))
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
public override string ToString()
{
- return Text;
+ return Content;
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxTrivia.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxTrivia.cs
index 6618b20c69..53867dc274 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxTrivia.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxTrivia.cs
@@ -76,5 +76,20 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
{
return new SyntaxTrivia(Kind, Text, GetDiagnostics(), annotations);
}
+
+ public override bool IsEquivalentTo(GreenNode other)
+ {
+ if (!base.IsEquivalentTo(other))
+ {
+ return false;
+ }
+
+ if (Text != ((SyntaxTrivia)other).Text)
+ {
+ return false;
+ }
+
+ return true;
+ }
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/UnknownTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/UnknownTokenSyntax.cs
deleted file mode 100644
index 4c6cd4608b..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/UnknownTokenSyntax.cs
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
-{
- internal class UnknownTokenSyntax : SyntaxToken
- {
- internal UnknownTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
- : base(SyntaxKind.Unknown, text, null, null, diagnostics, null)
- {
- }
-
- internal UnknownTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(SyntaxKind.Unknown, text, leadingTrivia, trailingTrivia)
- {
- }
-
- protected UnknownTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(kind, name, leadingTrivia, trailingTrivia)
- {
- }
-
- protected UnknownTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
- : base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
- {
- }
-
- internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.UnknownTokenSyntax(this, parent, position);
-
- public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
- {
- return new UnknownTokenSyntax(Kind, Text, trivia, TrailingTrivia);
- }
-
- public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
- {
- return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, trivia);
- }
-
- internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
- {
- return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
- }
-
- internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
- {
- return new UnknownTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/WhitespaceTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/WhitespaceTokenSyntax.cs
deleted file mode 100644
index b576b60e3a..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/WhitespaceTokenSyntax.cs
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
-{
- internal class WhitespaceTokenSyntax : SyntaxToken
- {
- internal WhitespaceTokenSyntax(string text, params RazorDiagnostic[] diagnostics)
- : base(SyntaxKind.Whitespace, text, null, null, diagnostics, null)
- {
- }
-
- internal WhitespaceTokenSyntax(string text, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(SyntaxKind.Whitespace, text, leadingTrivia, trailingTrivia)
- {
- }
-
- protected WhitespaceTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia)
- : base(kind, name, leadingTrivia, trailingTrivia)
- {
- }
-
- protected WhitespaceTokenSyntax(SyntaxKind kind, string name, GreenNode leadingTrivia, GreenNode trailingTrivia, RazorDiagnostic[] diagnostics, SyntaxAnnotation[] annotations)
- : base(kind, name, leadingTrivia, trailingTrivia, diagnostics, annotations)
- {
- }
-
- internal override SyntaxNode CreateRed(SyntaxNode parent, int position) => new Syntax.WhitespaceTokenSyntax(this, parent, position);
-
- public override SyntaxToken TokenWithLeadingTrivia(GreenNode trivia)
- {
- return new WhitespaceTokenSyntax(Kind, Text, trivia, TrailingTrivia);
- }
-
- public override SyntaxToken TokenWithTrailingTrivia(GreenNode trivia)
- {
- return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, trivia);
- }
-
- internal override GreenNode SetDiagnostics(RazorDiagnostic[] diagnostics)
- {
- return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, diagnostics, GetAnnotations());
- }
-
- internal override GreenNode SetAnnotations(SyntaxAnnotation[] annotations)
- {
- return new WhitespaceTokenSyntax(Kind, Text, LeadingTrivia, TrailingTrivia, GetDiagnostics(), annotations);
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/NewLineTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/NewLineTokenSyntax.cs
deleted file mode 100644
index d164f069e9..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/NewLineTokenSyntax.cs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax
-{
- internal class NewLineTokenSyntax : SyntaxToken
- {
- internal NewLineTokenSyntax(GreenNode green, SyntaxNode parent, int position)
- : base(green, parent, position)
- {
- }
-
- internal new InternalSyntax.NewLineTokenSyntax Green => (InternalSyntax.NewLineTokenSyntax)base.Green;
-
- public string Value => Text;
-
- internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.NewLineTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as NewLineTokenSyntax;
- }
-
- internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.NewLineTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as NewLineTokenSyntax;
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/PunctuationSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/PunctuationSyntax.cs
deleted file mode 100644
index 0f25740ecd..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/PunctuationSyntax.cs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax
-{
- internal class PunctuationSyntax : SyntaxToken
- {
- internal PunctuationSyntax(GreenNode green, SyntaxNode parent, int position)
- : base(green, parent, position)
- {
- }
-
- internal new InternalSyntax.PunctuationSyntax Green => (InternalSyntax.PunctuationSyntax)base.Green;
-
- public string Punctuation => Text;
-
- internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.PunctuationSyntax(Kind, Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as PunctuationSyntax;
- }
-
- internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.PunctuationSyntax(Kind, Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as PunctuationSyntax;
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxFactory.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxFactory.cs
new file mode 100644
index 0000000000..46bfd219dc
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxFactory.cs
@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.AspNetCore.Razor.Language.Syntax
+{
+ internal static class SyntaxFactory
+ {
+ internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
+ {
+ return new SyntaxToken(InternalSyntax.SyntaxFactory.Token(kind, content), parent: null, position: 0);
+ }
+ }
+}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxKind.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxKind.cs
index 46dad28dee..1cad1ed9f2 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxKind.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxKind.cs
@@ -5,28 +5,93 @@ namespace Microsoft.AspNetCore.Razor.Language
{
internal enum SyntaxKind : byte
{
+ #region Nodes
+ HtmlText,
+ HtmlDocument,
+ HtmlDeclaration,
+ #endregion
+
+ #region Tokens
+ // Common
Unknown,
List,
Whitespace,
NewLine,
+ Colon,
+ QuestionMark,
+ RightBracket,
+ LeftBracket,
+ Equals,
+ Transition,
// HTML
- HtmlText,
- HtmlDocument,
- HtmlDeclaration,
- HtmlTextLiteralToken,
+ HtmlTextLiteral,
OpenAngle,
Bang,
ForwardSlash,
- QuestionMark,
DoubleHyphen,
- LeftBracket,
CloseAngle,
- RightBracket,
- Equals,
DoubleQuote,
SingleQuote,
- Transition,
- Colon,
+
+ // CSharp literals
+ Identifier,
+ Keyword,
+ IntegerLiteral,
+ CSharpComment,
+ RealLiteral,
+ CharacterLiteral,
+ StringLiteral,
+
+ // CSharp operators
+ Arrow,
+ Minus,
+ Decrement,
+ MinusAssign,
+ NotEqual,
+ Not,
+ Modulo,
+ ModuloAssign,
+ AndAssign,
+ And,
+ DoubleAnd,
+ LeftParenthesis,
+ RightParenthesis,
+ Star,
+ MultiplyAssign,
+ Comma,
+ Dot,
+ Slash,
+ DivideAssign,
+ DoubleColon,
+ Semicolon,
+ NullCoalesce,
+ XorAssign,
+ Xor,
+ LeftBrace,
+ OrAssign,
+ DoubleOr,
+ Or,
+ RightBrace,
+ Tilde,
+ Plus,
+ PlusAssign,
+ Increment,
+ LessThan,
+ LessThanEqual,
+ LeftShift,
+ LeftShiftAssign,
+ Assign,
+ GreaterThan,
+ GreaterThanEqual,
+ RightShift,
+ RightShiftAssign,
+ Hash,
+
+ // Razor specific
+ RazorComment,
+ RazorCommentStar,
+ RazorCommentTransition,
+ #endregion
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxNode.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxNode.cs
index d55504d46e..f0634d10d3 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxNode.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxNode.cs
@@ -239,6 +239,21 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
return Green.GetAnnotations();
}
+ public bool IsEquivalentTo(SyntaxNode other)
+ {
+ if (this == other)
+ {
+ return true;
+ }
+
+ if (other == null)
+ {
+ return false;
+ }
+
+ return Green.IsEquivalentTo(other.Green);
+ }
+
public override string ToString()
{
return Green.ToString();
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxToken.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxToken.cs
index d3ffa0096c..c022f3634b 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxToken.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxToken.cs
@@ -3,29 +3,70 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.Linq;
+using Microsoft.AspNetCore.Razor.Language.Legacy;
namespace Microsoft.AspNetCore.Razor.Language.Syntax
{
- internal abstract class SyntaxToken : SyntaxNode
+ internal class SyntaxToken : SyntaxNode
{
internal SyntaxToken(GreenNode green, SyntaxNode parent, int position)
+ : this(green, parent, null, position)
+ {
+ }
+
+ // Temporary plumbing
+ internal SyntaxToken(GreenNode green, SyntaxNode parent, Span parentSpan, int position)
: base(green, parent, position)
{
+ Debug.Assert(parent == null || !parent.Green.IsList, "list cannot be a parent");
+ Debug.Assert(green == null || green.IsToken, "green must be a token");
+
+ ParentSpan = parentSpan;
+ }
+
+ // Temporary plumbing
+ internal Span ParentSpan { get; }
+
+ // Temporary plumbing
+ internal SourceLocation Start
+ {
+ get
+ {
+ if (ParentSpan == null)
+ {
+ return SourceLocation.Undefined;
+ }
+
+ var tracker = new SourceLocationTracker(ParentSpan.Start);
+ for (var i = 0; i < ParentSpan.Tokens.Count; i++)
+ {
+ var token = ParentSpan.Tokens[i];
+ if (object.ReferenceEquals(this, token))
+ {
+ break;
+ }
+
+ tracker.UpdateLocation(token.Content);
+ }
+
+ return tracker.CurrentLocation;
+ }
}
internal new InternalSyntax.SyntaxToken Green => (InternalSyntax.SyntaxToken)base.Green;
- public string Text => Green.Text;
+ public string Content => Green.Content;
internal override sealed SyntaxNode GetCachedSlot(int index)
{
- throw new InvalidOperationException();
+ throw new InvalidOperationException("Tokens can't have slots.");
}
internal override sealed SyntaxNode GetNodeSlot(int slot)
{
- throw new InvalidOperationException();
+ throw new InvalidOperationException("Tokens can't have slots.");
}
internal override SyntaxNode Accept(SyntaxVisitor visitor)
@@ -33,24 +74,30 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
return visitor.VisitSyntaxToken(this);
}
- internal abstract SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia);
+ public SyntaxToken WithLeadingTrivia(SyntaxNode trivia)
+ {
+ return Green != null
+ ? new SyntaxToken(Green.WithLeadingTrivia(trivia.Green), parent: null, position: 0)
+ : default(SyntaxToken);
+ }
- internal abstract SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia);
-
- public SyntaxToken WithLeadingTrivia(SyntaxNode trivia) => WithLeadingTriviaCore(trivia);
-
- public SyntaxToken WithTrailingTrivia(SyntaxNode trivia) => WithTrailingTriviaCore(trivia);
+ public SyntaxToken WithTrailingTrivia(SyntaxNode trivia)
+ {
+ return Green != null
+ ? new SyntaxToken(Green.WithTrailingTrivia(trivia.Green), parent: null, position: 0)
+ : default(SyntaxToken);
+ }
public SyntaxToken WithLeadingTrivia(IEnumerable trivia)
{
var greenList = trivia?.Select(t => t.Green);
- return WithLeadingTriviaCore(Green.CreateList(greenList)?.CreateRed());
+ return WithLeadingTrivia(Green.CreateList(greenList)?.CreateRed());
}
public SyntaxToken WithTrailingTrivia(IEnumerable trivia)
{
var greenList = trivia?.Select(t => t.Green);
- return WithTrailingTriviaCore(Green.CreateList(greenList)?.CreateRed());
+ return WithTrailingTrivia(Green.CreateList(greenList)?.CreateRed());
}
public override SyntaxTriviaList GetLeadingTrivia()
@@ -85,7 +132,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax
public override string ToString()
{
- return Text;
+ return Content;
}
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/UnknownTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/UnknownTokenSyntax.cs
deleted file mode 100644
index 238b36d1af..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/UnknownTokenSyntax.cs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax
-{
- internal class UnknownTokenSyntax : SyntaxToken
- {
- internal UnknownTokenSyntax(GreenNode green, SyntaxNode parent, int position)
- : base(green, parent, position)
- {
- }
-
- internal new InternalSyntax.UnknownTokenSyntax Green => (InternalSyntax.UnknownTokenSyntax)base.Green;
-
- public string Value => Text;
-
- internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.UnknownTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as UnknownTokenSyntax;
- }
-
- internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.UnknownTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as UnknownTokenSyntax;
- }
- }
-}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Syntax/WhitespaceTokenSyntax.cs b/src/Microsoft.AspNetCore.Razor.Language/Syntax/WhitespaceTokenSyntax.cs
deleted file mode 100644
index 5a8222f802..0000000000
--- a/src/Microsoft.AspNetCore.Razor.Language/Syntax/WhitespaceTokenSyntax.cs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-namespace Microsoft.AspNetCore.Razor.Language.Syntax
-{
- internal class WhitespaceTokenSyntax : SyntaxToken
- {
- internal WhitespaceTokenSyntax(GreenNode green, SyntaxNode parent, int position)
- : base(green, parent, position)
- {
- }
-
- internal new InternalSyntax.WhitespaceTokenSyntax Green => (InternalSyntax.WhitespaceTokenSyntax)base.Green;
-
- public string Value => Text;
-
- internal override SyntaxToken WithLeadingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.WhitespaceTokenSyntax(Text, trivia?.Green, GetTrailingTrivia().Node?.Green).CreateRed(Parent, Position) as WhitespaceTokenSyntax;
- }
-
- internal override SyntaxToken WithTrailingTriviaCore(SyntaxNode trivia)
- {
- return new InternalSyntax.WhitespaceTokenSyntax(Text, GetLeadingTrivia().Node?.Green, trivia?.Green).CreateRed(Parent, Position) as WhitespaceTokenSyntax;
- }
- }
-}
diff --git a/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorCompletionFactsService.cs b/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorCompletionFactsService.cs
index 57ac8ddb8d..2f9146eab2 100644
--- a/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorCompletionFactsService.cs
+++ b/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorCompletionFactsService.cs
@@ -7,6 +7,7 @@ using System.ComponentModel.Composition;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language;
using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.VisualStudio.Editor.Razor
{
@@ -83,16 +84,11 @@ namespace Microsoft.VisualStudio.Editor.Razor
}
// Internal for testing
- internal static bool IsDirectiveCompletableToken(IToken token)
+ internal static bool IsDirectiveCompletableToken(SyntaxToken token)
{
- if (!(token is CSharpToken csharpToken))
- {
- return false;
- }
-
- return csharpToken.Type == CSharpTokenType.Identifier ||
+ return token.Kind == SyntaxKind.Identifier ||
// Marker symbol
- csharpToken.Type == CSharpTokenType.Unknown;
+ token.Kind == SyntaxKind.Unknown;
}
}
}
diff --git a/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs b/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
index f69274fa15..f8776d6993 100644
--- a/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
+++ b/src/Microsoft.VisualStudio.Editor.Razor/DefaultRazorIndentationFactsService.cs
@@ -143,8 +143,7 @@ namespace Microsoft.VisualStudio.Editor.Razor
{
return currentChild is Span currentSpan &&
currentSpan.Tokens.Count == 1 &&
- currentSpan.Tokens[0] is CSharpToken symbol &&
- symbol.Type == CSharpTokenType.LeftBrace;
+ currentSpan.Tokens[0].Kind == SyntaxKind.LeftBrace;
}
}
}
diff --git a/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs b/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
index 607270c978..2b4788dad9 100644
--- a/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
+++ b/src/Microsoft.VisualStudio.Editor.Razor/RazorDirectiveCompletionProvider.cs
@@ -206,17 +206,5 @@ namespace Microsoft.VisualStudio.Editor.Razor
return false;
}
-
- private static bool IsDirectiveCompletableSymbol(AspNetCore.Razor.Language.Legacy.IToken symbol)
- {
- if (!(symbol is CSharpToken csharpSymbol))
- {
- return false;
- }
-
- return csharpSymbol.Type == CSharpTokenType.Identifier ||
- // Marker symbol
- csharpSymbol.Type == CSharpTokenType.Unknown;
- }
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
index d54ef401df..643c8d54fc 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/BlockTest.cs
@@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Linq;
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@@ -13,7 +14,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
// Arrange
var spanBuilder = new SpanBuilder(SourceLocation.Zero);
- spanBuilder.Accept(new HtmlToken("hello", HtmlTokenType.Text));
+ spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hello"));
var span = spanBuilder.Build();
var blockBuilder = new BlockBuilder()
{
@@ -29,7 +30,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
var parentBlock = blockBuilder.Build();
var originalBlockLength = parentBlock.Length;
spanBuilder = new SpanBuilder(SourceLocation.Zero);
- spanBuilder.Accept(new HtmlToken("hi", HtmlTokenType.Text));
+ spanBuilder.Accept(SyntaxFactory.Token(SyntaxKind.HtmlTextLiteral, "hi"));
span.ReplaceWith(spanBuilder);
// Wire up parents now so we can re-trigger ChildChanged to cause cache refresh.
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
index e8eeb8c586..84802523b2 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpLanguageCharacteristicsTest.cs
@@ -11,7 +11,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
public void GetSample_RightShiftAssign_ReturnsCorrectToken()
{
// Arrange & Act
- var token = CSharpLanguageCharacteristics.Instance.GetSample(CSharpTokenType.RightShiftAssign);
+ var token = CSharpLanguageCharacteristics.Instance.GetSample(SyntaxKind.RightShiftAssign);
// Assert
Assert.Equal(">>=", token);
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
index 64dafafa16..4abaea783f 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerCommentTest.cs
@@ -1,22 +1,23 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
{
- private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
+ private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer(
"@* Foo * Bar * Baz *",
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken(" Foo * Bar * Baz *", CSharpTokenType.RazorComment));
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz *"));
}
[Fact]
@@ -24,11 +25,11 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo * Bar * Baz *@",
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken(" Foo * Bar * Baz ", CSharpTokenType.RazorComment),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo * Bar * Baz "),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
}
[Fact]
@@ -36,59 +37,59 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@* Foo Bar Baz *@",
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken(" Foo Bar Baz ", CSharpTokenType.RazorComment),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition));
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorComment, " Foo Bar Baz "),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{
- TestTokenizer("// Foo Bar Baz", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment));
+ TestTokenizer("// Foo Bar Baz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"));
}
[Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline()
{
- TestTokenizer("// Foo Bar Baz\na", new CSharpToken("// Foo Bar Baz", CSharpTokenType.Comment), IgnoreRemaining);
+ TestTokenizer("// Foo Bar Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"), IgnoreRemaining);
}
[Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{
- TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpToken("// Foo/*Bar*/ Baz", CSharpTokenType.Comment), IgnoreRemaining);
+ TestTokenizer("// Foo/*Bar*/ Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo/*Bar*/ Baz"), IgnoreRemaining);
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{
- TestTokenizer("/* Foo\nBar\nBaz */", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment));
+ TestTokenizer("/* Foo\nBar\nBaz */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"));
}
[Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{
- TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpToken("/* Foo\nBar\nBaz */", CSharpTokenType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo\nBar\nBaz */a", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"), IgnoreRemaining);
}
[Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{
- TestTokenizer("/* Foo\nBar\nBaz", new CSharpToken("/* Foo\nBar\nBaz", CSharpTokenType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo\nBar\nBaz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz"), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{
- TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpToken("/* Foo/*\nBar\nBaz*/", CSharpTokenType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo/*\nBar\nBaz*/ */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo/*\nBar\nBaz*/"), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{
- TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpToken("/* Foo\nBar\nBaz* */", CSharpTokenType.Comment), IgnoreRemaining);
+ TestTokenizer("/* Foo\nBar\nBaz* */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz* */"), IgnoreRemaining);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
index b24c976dc9..e5a3a466ac 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerIdentifierTest.cs
@@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@@ -10,73 +11,73 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact]
public void Simple_Identifier_Is_Recognized()
{
- TestTokenizer("foo", new CSharpToken("foo", CSharpTokenType.Identifier));
+ TestTokenizer("foo", SyntaxFactory.Token(SyntaxKind.Identifier, "foo"));
}
[Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized()
{
- TestTokenizer("_foo", new CSharpToken("_foo", CSharpTokenType.Identifier));
+ TestTokenizer("_foo", SyntaxFactory.Token(SyntaxKind.Identifier, "_foo"));
}
[Fact]
public void Identifier_Can_Contain_Digits()
{
- TestTokenizer("foo4", new CSharpToken("foo4", CSharpTokenType.Identifier));
+ TestTokenizer("foo4", SyntaxFactory.Token(SyntaxKind.Identifier, "foo4"));
}
[Fact]
public void Identifier_Can_Start_With_Titlecase_Letter()
{
- TestTokenizer("ῼfoo", new CSharpToken("ῼfoo", CSharpTokenType.Identifier));
+ TestTokenizer("ῼfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ῼfoo"));
}
[Fact]
public void Identifier_Can_Start_With_Letter_Modifier()
{
- TestTokenizer("ᵊfoo", new CSharpToken("ᵊfoo", CSharpTokenType.Identifier));
+ TestTokenizer("ᵊfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ᵊfoo"));
}
[Fact]
public void Identifier_Can_Start_With_Other_Letter()
{
- TestTokenizer("ƻfoo", new CSharpToken("ƻfoo", CSharpTokenType.Identifier));
+ TestTokenizer("ƻfoo", SyntaxFactory.Token(SyntaxKind.Identifier, "ƻfoo"));
}
[Fact]
public void Identifier_Can_Start_With_Number_Letter()
{
- TestTokenizer("Ⅽool", new CSharpToken("Ⅽool", CSharpTokenType.Identifier));
+ TestTokenizer("Ⅽool", SyntaxFactory.Token(SyntaxKind.Identifier, "Ⅽool"));
}
[Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark()
{
- TestTokenizer("foo\u0300", new CSharpToken("foo\u0300", CSharpTokenType.Identifier));
+ TestTokenizer("foo\u0300", SyntaxFactory.Token(SyntaxKind.Identifier, "foo\u0300"));
}
[Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark()
{
- TestTokenizer("fooः", new CSharpToken("fooः", CSharpTokenType.Identifier));
+ TestTokenizer("fooः", SyntaxFactory.Token(SyntaxKind.Identifier, "fooः"));
}
[Fact]
public void Identifier_Can_Contain_Non_English_Digit()
{
- TestTokenizer("foo١", new CSharpToken("foo١", CSharpTokenType.Identifier));
+ TestTokenizer("foo١", SyntaxFactory.Token(SyntaxKind.Identifier, "foo١"));
}
[Fact]
public void Identifier_Can_Contain_Connector_Punctuation()
{
- TestTokenizer("foo‿bar", new CSharpToken("foo‿bar", CSharpTokenType.Identifier));
+ TestTokenizer("foo‿bar", SyntaxFactory.Token(SyntaxKind.Identifier, "foo‿bar"));
}
[Fact]
public void Identifier_Can_Contain_Format_Character()
{
- TestTokenizer("foobar", new CSharpToken("foobar", CSharpTokenType.Identifier));
+ TestTokenizer("foobar", SyntaxFactory.Token(SyntaxKind.Identifier, "foobar"));
}
[Fact]
@@ -164,7 +165,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType)
{
- TestTokenizer(keyword, new CSharpToken(keyword, CSharpTokenType.Keyword) { Keyword = keywordType });
+ TestTokenizer(keyword, SyntaxFactory.Token(SyntaxKind.Keyword, keyword));
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
index a10aea8ba9..c7c250cce3 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerLiteralTest.cs
@@ -2,286 +2,287 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase
{
- private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
+ private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact]
public void Simple_Integer_Literal_Is_Recognized()
{
- TestSingleToken("01189998819991197253", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("01189998819991197253", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized()
{
- TestSingleToken("42U", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42u", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42U", SyntaxKind.IntegerLiteral);
+ TestSingleToken("42u", SyntaxKind.IntegerLiteral);
- TestSingleToken("42L", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42l", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42L", SyntaxKind.IntegerLiteral);
+ TestSingleToken("42l", SyntaxKind.IntegerLiteral);
- TestSingleToken("42UL", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42Ul", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42UL", SyntaxKind.IntegerLiteral);
+ TestSingleToken("42Ul", SyntaxKind.IntegerLiteral);
- TestSingleToken("42uL", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42ul", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42uL", SyntaxKind.IntegerLiteral);
+ TestSingleToken("42ul", SyntaxKind.IntegerLiteral);
- TestSingleToken("42LU", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42Lu", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42LU", SyntaxKind.IntegerLiteral);
+ TestSingleToken("42Lu", SyntaxKind.IntegerLiteral);
- TestSingleToken("42lU", CSharpTokenType.IntegerLiteral);
- TestSingleToken("42lu", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("42lU", SyntaxKind.IntegerLiteral);
+ TestSingleToken("42lu", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{
- TestTokenizer("42a", new CSharpToken("42", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
+ TestTokenizer("42a", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "42"), IgnoreRemaining);
}
[Fact]
public void Simple_Hex_Literal_Is_Recognized()
{
- TestSingleToken("0x0123456789ABCDEF", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0x0123456789ABCDEF", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal()
{
- TestSingleToken("0xDEADBEEFU", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFu", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFU", SyntaxKind.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFu", SyntaxKind.IntegerLiteral);
- TestSingleToken("0xDEADBEEFL", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFl", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFL", SyntaxKind.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFl", SyntaxKind.IntegerLiteral);
- TestSingleToken("0xDEADBEEFUL", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFUl", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFUL", SyntaxKind.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFUl", SyntaxKind.IntegerLiteral);
- TestSingleToken("0xDEADBEEFuL", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFul", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFuL", SyntaxKind.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFul", SyntaxKind.IntegerLiteral);
- TestSingleToken("0xDEADBEEFLU", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFLu", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFLU", SyntaxKind.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFLu", SyntaxKind.IntegerLiteral);
- TestSingleToken("0xDEADBEEFlU", CSharpTokenType.IntegerLiteral);
- TestSingleToken("0xDEADBEEFlu", CSharpTokenType.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFlU", SyntaxKind.IntegerLiteral);
+ TestSingleToken("0xDEADBEEFlu", SyntaxKind.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{
- TestTokenizer("0xDEADBEEFz", new CSharpToken("0xDEADBEEF", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
+ TestTokenizer("0xDEADBEEFz", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "0xDEADBEEF"), IgnoreRemaining);
}
[Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{
- TestTokenizer("3.a", new CSharpToken("3", CSharpTokenType.IntegerLiteral), IgnoreRemaining);
+ TestTokenizer("3.a", SyntaxFactory.Token(SyntaxKind.IntegerLiteral, "3"), IgnoreRemaining);
}
[Fact]
public void Simple_Real_Literal_Is_Recognized()
{
- TestTokenizer("3.14159", new CSharpToken("3.14159", CSharpTokenType.RealLiteral));
+ TestTokenizer("3.14159", SyntaxFactory.Token(SyntaxKind.RealLiteral, "3.14159"));
}
[Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{
- TestTokenizer(".14159", new CSharpToken(".14159", CSharpTokenType.RealLiteral));
+ TestTokenizer(".14159", SyntaxFactory.Token(SyntaxKind.RealLiteral, ".14159"));
}
[Fact]
public void Integer_With_Real_Type_Suffix_Is_Recognized()
{
- TestSingleToken("42F", CSharpTokenType.RealLiteral);
- TestSingleToken("42f", CSharpTokenType.RealLiteral);
- TestSingleToken("42D", CSharpTokenType.RealLiteral);
- TestSingleToken("42d", CSharpTokenType.RealLiteral);
- TestSingleToken("42M", CSharpTokenType.RealLiteral);
- TestSingleToken("42m", CSharpTokenType.RealLiteral);
+ TestSingleToken("42F", SyntaxKind.RealLiteral);
+ TestSingleToken("42f", SyntaxKind.RealLiteral);
+ TestSingleToken("42D", SyntaxKind.RealLiteral);
+ TestSingleToken("42d", SyntaxKind.RealLiteral);
+ TestSingleToken("42M", SyntaxKind.RealLiteral);
+ TestSingleToken("42m", SyntaxKind.RealLiteral);
}
[Fact]
public void Integer_With_Exponent_Is_Recognized()
{
- TestSingleToken("1e10", CSharpTokenType.RealLiteral);
- TestSingleToken("1E10", CSharpTokenType.RealLiteral);
- TestSingleToken("1e+10", CSharpTokenType.RealLiteral);
- TestSingleToken("1E+10", CSharpTokenType.RealLiteral);
- TestSingleToken("1e-10", CSharpTokenType.RealLiteral);
- TestSingleToken("1E-10", CSharpTokenType.RealLiteral);
+ TestSingleToken("1e10", SyntaxKind.RealLiteral);
+ TestSingleToken("1E10", SyntaxKind.RealLiteral);
+ TestSingleToken("1e+10", SyntaxKind.RealLiteral);
+ TestSingleToken("1E+10", SyntaxKind.RealLiteral);
+ TestSingleToken("1e-10", SyntaxKind.RealLiteral);
+ TestSingleToken("1E-10", SyntaxKind.RealLiteral);
}
[Fact]
public void Real_Number_With_Type_Suffix_Is_Recognized()
{
- TestSingleToken("3.14F", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14f", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14D", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14d", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14M", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14m", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14F", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14f", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14D", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14d", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14M", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14m", SyntaxKind.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_Is_Recognized()
{
- TestSingleToken("3.14E10", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14e10", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14E+10", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14e+10", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14E-10", CSharpTokenType.RealLiteral);
- TestSingleToken("3.14e-10", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14E10", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14e10", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14E+10", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14e+10", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14E-10", SyntaxKind.RealLiteral);
+ TestSingleToken("3.14e-10", SyntaxKind.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized()
{
- TestSingleToken("3.14E+10F", CSharpTokenType.RealLiteral);
+ TestSingleToken("3.14E+10F", SyntaxKind.RealLiteral);
}
[Fact]
public void Single_Character_Literal_Is_Recognized()
{
- TestSingleToken("'f'", CSharpTokenType.CharacterLiteral);
+ TestSingleToken("'f'", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Multi_Character_Literal_Is_Recognized()
{
- TestSingleToken("'foo'", CSharpTokenType.CharacterLiteral);
+ TestSingleToken("'foo'", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
- TestSingleToken("'foo bar", CSharpTokenType.CharacterLiteral);
+ TestSingleToken("'foo bar", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Character_Literal_Not_Terminated_By_Escaped_Quote()
{
- TestSingleToken("'foo\\'bar'", CSharpTokenType.CharacterLiteral);
+ TestSingleToken("'foo\\'bar'", SyntaxKind.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
- TestTokenizer("'foo\n", new CSharpToken("'foo", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\n", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("'foo\\\n", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\\n", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer("'foo\\\nflarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\\nflarg", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("'foo\\" + Environment.NewLine, new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\" + Environment.NewLine, SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpToken("'foo\\", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer($"'foo\\{Environment.NewLine}flarg", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\"), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Allows_Escaped_Escape()
{
- TestTokenizer("'foo\\\\'blah", new CSharpToken("'foo\\\\'", CSharpTokenType.CharacterLiteral), IgnoreRemaining);
+ TestTokenizer("'foo\\\\'blah", SyntaxFactory.Token(SyntaxKind.CharacterLiteral, "'foo\\\\'"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Is_Recognized()
{
- TestSingleToken("\"foo\"", CSharpTokenType.StringLiteral);
+ TestSingleToken("\"foo\"", SyntaxKind.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
- TestSingleToken("\"foo bar", CSharpTokenType.StringLiteral);
+ TestSingleToken("\"foo bar", SyntaxKind.StringLiteral);
}
[Fact]
public void String_Literal_Not_Terminated_By_Escaped_Quote()
{
- TestSingleToken("\"foo\\\"bar\"", CSharpTokenType.StringLiteral);
+ TestSingleToken("\"foo\\\"bar\"", SyntaxKind.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
- TestTokenizer("\"foo\n", new CSharpToken("\"foo", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\n", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("\"foo\\\n", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\\n", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer("\"foo\\\nflarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\\nflarg", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
- TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\" + Environment.NewLine, SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
- TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpToken("\"foo\\", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer($"\"foo\\{Environment.NewLine}flarg", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\"), IgnoreRemaining);
}
[Fact]
public void String_Literal_Allows_Escaped_Escape()
{
- TestTokenizer("\"foo\\\\\"blah", new CSharpToken("\"foo\\\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("\"foo\\\\\"blah", SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"foo\\\\\""), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Can_Contain_Newlines()
{
- TestSingleToken("@\"foo\nbar\nbaz\"", CSharpTokenType.StringLiteral);
+ TestSingleToken("@\"foo\nbar\nbaz\"", SyntaxKind.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote()
{
- TestSingleToken("@\"foo\"\"bar\"", CSharpTokenType.StringLiteral);
+ TestSingleToken("@\"foo\"\"bar\"", SyntaxKind.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{
- TestTokenizer("@\"foo\\\"bar\"", new CSharpToken("@\"foo\\\"", CSharpTokenType.StringLiteral), IgnoreRemaining);
+ TestTokenizer("@\"foo\\\"bar\"", SyntaxFactory.Token(SyntaxKind.StringLiteral, "@\"foo\\\""), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_EOF()
{
- TestSingleToken("@\"foo", CSharpTokenType.StringLiteral);
+ TestSingleToken("@\"foo", SyntaxKind.StringLiteral);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
index 0b600ea9bc..fdf2718387 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerOperatorsTest.cs
@@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@@ -10,287 +11,287 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
[Fact]
public void LeftBrace_Is_Recognized()
{
- TestSingleToken("{", CSharpTokenType.LeftBrace);
+ TestSingleToken("{", SyntaxKind.LeftBrace);
}
[Fact]
public void Plus_Is_Recognized()
{
- TestSingleToken("+", CSharpTokenType.Plus);
+ TestSingleToken("+", SyntaxKind.Plus);
}
[Fact]
public void Assign_Is_Recognized()
{
- TestSingleToken("=", CSharpTokenType.Assign);
+ TestSingleToken("=", SyntaxKind.Assign);
}
[Fact]
public void Arrow_Is_Recognized()
{
- TestSingleToken("->", CSharpTokenType.Arrow);
+ TestSingleToken("->", SyntaxKind.Arrow);
}
[Fact]
public void AndAssign_Is_Recognized()
{
- TestSingleToken("&=", CSharpTokenType.AndAssign);
+ TestSingleToken("&=", SyntaxKind.AndAssign);
}
[Fact]
public void RightBrace_Is_Recognized()
{
- TestSingleToken("}", CSharpTokenType.RightBrace);
+ TestSingleToken("}", SyntaxKind.RightBrace);
}
[Fact]
public void Minus_Is_Recognized()
{
- TestSingleToken("-", CSharpTokenType.Minus);
+ TestSingleToken("-", SyntaxKind.Minus);
}
[Fact]
public void LessThan_Is_Recognized()
{
- TestSingleToken("<", CSharpTokenType.LessThan);
+ TestSingleToken("<", SyntaxKind.LessThan);
}
[Fact]
public void Equals_Is_Recognized()
{
- TestSingleToken("==", CSharpTokenType.Equals);
+ TestSingleToken("==", SyntaxKind.Equals);
}
[Fact]
public void OrAssign_Is_Recognized()
{
- TestSingleToken("|=", CSharpTokenType.OrAssign);
+ TestSingleToken("|=", SyntaxKind.OrAssign);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
- TestSingleToken("[", CSharpTokenType.LeftBracket);
+ TestSingleToken("[", SyntaxKind.LeftBracket);
}
[Fact]
public void Star_Is_Recognized()
{
- TestSingleToken("*", CSharpTokenType.Star);
+ TestSingleToken("*", SyntaxKind.Star);
}
[Fact]
public void GreaterThan_Is_Recognized()
{
- TestSingleToken(">", CSharpTokenType.GreaterThan);
+ TestSingleToken(">", SyntaxKind.GreaterThan);
}
[Fact]
public void NotEqual_Is_Recognized()
{
- TestSingleToken("!=", CSharpTokenType.NotEqual);
+ TestSingleToken("!=", SyntaxKind.NotEqual);
}
[Fact]
public void XorAssign_Is_Recognized()
{
- TestSingleToken("^=", CSharpTokenType.XorAssign);
+ TestSingleToken("^=", SyntaxKind.XorAssign);
}
[Fact]
public void RightBracket_Is_Recognized()
{
- TestSingleToken("]", CSharpTokenType.RightBracket);
+ TestSingleToken("]", SyntaxKind.RightBracket);
}
[Fact]
public void Slash_Is_Recognized()
{
- TestSingleToken("/", CSharpTokenType.Slash);
+ TestSingleToken("/", SyntaxKind.Slash);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
- TestSingleToken("?", CSharpTokenType.QuestionMark);
+ TestSingleToken("?", SyntaxKind.QuestionMark);
}
[Fact]
public void LessThanEqual_Is_Recognized()
{
- TestSingleToken("<=", CSharpTokenType.LessThanEqual);
+ TestSingleToken("<=", SyntaxKind.LessThanEqual);
}
[Fact]
public void LeftShift_Is_Not_Specially_Recognized()
{
TestTokenizer("<<",
- new CSharpToken("<", CSharpTokenType.LessThan),
- new CSharpToken("<", CSharpTokenType.LessThan));
+ SyntaxFactory.Token(SyntaxKind.LessThan, "<"),
+ SyntaxFactory.Token(SyntaxKind.LessThan, "<"));
}
[Fact]
public void LeftParen_Is_Recognized()
{
- TestSingleToken("(", CSharpTokenType.LeftParenthesis);
+ TestSingleToken("(", SyntaxKind.LeftParenthesis);
}
[Fact]
public void Modulo_Is_Recognized()
{
- TestSingleToken("%", CSharpTokenType.Modulo);
+ TestSingleToken("%", SyntaxKind.Modulo);
}
[Fact]
public void NullCoalesce_Is_Recognized()
{
- TestSingleToken("??", CSharpTokenType.NullCoalesce);
+ TestSingleToken("??", SyntaxKind.NullCoalesce);
}
[Fact]
public void GreaterThanEqual_Is_Recognized()
{
- TestSingleToken(">=", CSharpTokenType.GreaterThanEqual);
+ TestSingleToken(">=", SyntaxKind.GreaterThanEqual);
}
[Fact]
public void EqualGreaterThan_Is_Recognized()
{
- TestSingleToken("=>", CSharpTokenType.GreaterThanEqual);
+ TestSingleToken("=>", SyntaxKind.GreaterThanEqual);
}
[Fact]
public void RightParen_Is_Recognized()
{
- TestSingleToken(")", CSharpTokenType.RightParenthesis);
+ TestSingleToken(")", SyntaxKind.RightParenthesis);
}
[Fact]
public void And_Is_Recognized()
{
- TestSingleToken("&", CSharpTokenType.And);
+ TestSingleToken("&", SyntaxKind.And);
}
[Fact]
public void DoubleColon_Is_Recognized()
{
- TestSingleToken("::", CSharpTokenType.DoubleColon);
+ TestSingleToken("::", SyntaxKind.DoubleColon);
}
[Fact]
public void PlusAssign_Is_Recognized()
{
- TestSingleToken("+=", CSharpTokenType.PlusAssign);
+ TestSingleToken("+=", SyntaxKind.PlusAssign);
}
[Fact]
public void Semicolon_Is_Recognized()
{
- TestSingleToken(";", CSharpTokenType.Semicolon);
+ TestSingleToken(";", SyntaxKind.Semicolon);
}
[Fact]
public void Tilde_Is_Recognized()
{
- TestSingleToken("~", CSharpTokenType.Tilde);
+ TestSingleToken("~", SyntaxKind.Tilde);
}
[Fact]
public void DoubleOr_Is_Recognized()
{
- TestSingleToken("||", CSharpTokenType.DoubleOr);
+ TestSingleToken("||", SyntaxKind.DoubleOr);
}
[Fact]
public void ModuloAssign_Is_Recognized()
{
- TestSingleToken("%=", CSharpTokenType.ModuloAssign);
+ TestSingleToken("%=", SyntaxKind.ModuloAssign);
}
[Fact]
public void Colon_Is_Recognized()
{
- TestSingleToken(":", CSharpTokenType.Colon);
+ TestSingleToken(":", SyntaxKind.Colon);
}
[Fact]
public void Not_Is_Recognized()
{
- TestSingleToken("!", CSharpTokenType.Not);
+ TestSingleToken("!", SyntaxKind.Not);
}
[Fact]
public void DoubleAnd_Is_Recognized()
{
- TestSingleToken("&&", CSharpTokenType.DoubleAnd);
+ TestSingleToken("&&", SyntaxKind.DoubleAnd);
}
[Fact]
public void DivideAssign_Is_Recognized()
{
- TestSingleToken("/=", CSharpTokenType.DivideAssign);
+ TestSingleToken("/=", SyntaxKind.DivideAssign);
}
[Fact]
public void Comma_Is_Recognized()
{
- TestSingleToken(",", CSharpTokenType.Comma);
+ TestSingleToken(",", SyntaxKind.Comma);
}
[Fact]
public void Xor_Is_Recognized()
{
- TestSingleToken("^", CSharpTokenType.Xor);
+ TestSingleToken("^", SyntaxKind.Xor);
}
[Fact]
public void Decrement_Is_Recognized()
{
- TestSingleToken("--", CSharpTokenType.Decrement);
+ TestSingleToken("--", SyntaxKind.Decrement);
}
[Fact]
public void MultiplyAssign_Is_Recognized()
{
- TestSingleToken("*=", CSharpTokenType.MultiplyAssign);
+ TestSingleToken("*=", SyntaxKind.MultiplyAssign);
}
[Fact]
public void Dot_Is_Recognized()
{
- TestSingleToken(".", CSharpTokenType.Dot);
+ TestSingleToken(".", SyntaxKind.Dot);
}
[Fact]
public void Or_Is_Recognized()
{
- TestSingleToken("|", CSharpTokenType.Or);
+ TestSingleToken("|", SyntaxKind.Or);
}
[Fact]
public void Increment_Is_Recognized()
{
- TestSingleToken("++", CSharpTokenType.Increment);
+ TestSingleToken("++", SyntaxKind.Increment);
}
[Fact]
public void MinusAssign_Is_Recognized()
{
- TestSingleToken("-=", CSharpTokenType.MinusAssign);
+ TestSingleToken("-=", SyntaxKind.MinusAssign);
}
[Fact]
public void RightShift_Is_Not_Specially_Recognized()
{
TestTokenizer(">>",
- new CSharpToken(">", CSharpTokenType.GreaterThan),
- new CSharpToken(">", CSharpTokenType.GreaterThan));
+ SyntaxFactory.Token(SyntaxKind.GreaterThan, ">"),
+ SyntaxFactory.Token(SyntaxKind.GreaterThan, ">"));
}
[Fact]
public void Hash_Is_Recognized()
{
- TestSingleToken("#", CSharpTokenType.Hash);
+ TestSingleToken("#", SyntaxKind.Hash);
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
index cf98382ef4..5ee0715a7e 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTest.cs
@@ -1,13 +1,14 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public class CSharpTokenizerTest : CSharpTokenizerTestBase
{
- private new CSharpToken IgnoreRemaining => (CSharpToken)base.IgnoreRemaining;
+ private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
@@ -20,8 +21,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\ra",
- new CSharpToken("\r", CSharpTokenType.NewLine),
- new CSharpToken("\r", CSharpTokenType.NewLine),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r"),
IgnoreRemaining);
}
@@ -30,8 +31,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\n\na",
- new CSharpToken("\n", CSharpTokenType.NewLine),
- new CSharpToken("\n", CSharpTokenType.NewLine),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
IgnoreRemaining);
}
@@ -41,8 +42,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// NEL: Unicode "Next Line" U+0085
TestTokenizer(
"\u0085\u0085a",
- new CSharpToken("\u0085", CSharpTokenType.NewLine),
- new CSharpToken("\u0085", CSharpTokenType.NewLine),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\u0085"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\u0085"),
IgnoreRemaining);
}
@@ -52,8 +53,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Line Separator" U+2028
TestTokenizer(
"\u2028\u2028a",
- new CSharpToken("\u2028", CSharpTokenType.NewLine),
- new CSharpToken("\u2028", CSharpTokenType.NewLine),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\u2028"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\u2028"),
IgnoreRemaining);
}
@@ -63,8 +64,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
// Unicode "Paragraph Separator" U+2029
TestTokenizer(
"\u2029\u2029a",
- new CSharpToken("\u2029", CSharpTokenType.NewLine),
- new CSharpToken("\u2029", CSharpTokenType.NewLine),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\u2029"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\u2029"),
IgnoreRemaining);
}
@@ -73,8 +74,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n\r\na",
- new CSharpToken("\r\n", CSharpTokenType.NewLine),
- new CSharpToken("\r\n", CSharpTokenType.NewLine),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
IgnoreRemaining);
}
@@ -83,15 +84,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
" \f\t\u000B \n ",
- new CSharpToken(" \f\t\u000B ", CSharpTokenType.WhiteSpace),
- new CSharpToken("\n", CSharpTokenType.NewLine),
- new CSharpToken(" ", CSharpTokenType.WhiteSpace));
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " \f\t\u000B "),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\n"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "));
}
[Fact]
public void Transition_Is_Recognized()
{
- TestSingleToken("@", CSharpTokenType.Transition);
+ TestSingleToken("@", SyntaxKind.Transition);
}
[Fact]
@@ -99,8 +100,8 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@(",
- new CSharpToken("@", CSharpTokenType.Transition),
- new CSharpToken("(", CSharpTokenType.LeftParenthesis));
+ SyntaxFactory.Token(SyntaxKind.Transition, "@"),
+ SyntaxFactory.Token(SyntaxKind.LeftParenthesis, "("));
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
index 6869138c1e..60652e402f 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/CSharpTokenizerTestBase.cs
@@ -1,11 +1,13 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
+
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{
- private static CSharpToken _ignoreRemaining = new CSharpToken(string.Empty, CSharpTokenType.Unknown);
+ private static SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Unknown, string.Empty);
internal override object IgnoreRemaining
{
@@ -17,14 +19,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
return new CSharpTokenizer(source);
}
- internal void TestSingleToken(string text, CSharpTokenType expectedTokenType)
+ internal void TestSingleToken(string text, SyntaxKind expectedTokenKind)
{
- TestTokenizer(text, new CSharpToken(text, expectedTokenType));
- }
-
- internal void TestTokenizer(string input, params CSharpToken[] expectedTokens)
- {
- base.TestTokenizer(input, expectedTokens);
+ TestTokenizer(text, SyntaxFactory.Token(expectedTokenKind, text));
}
}
}
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
index 7a1e0c070a..55f2626925 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveCSharpTokenizerTest.cs
@@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@@ -12,12 +13,12 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n @something \r\n @this is ignored",
- new CSharpToken("\r\n", CSharpTokenType.NewLine),
- new CSharpToken(" ", CSharpTokenType.WhiteSpace),
- new CSharpToken("@", CSharpTokenType.Transition),
- new CSharpToken("something", CSharpTokenType.Identifier),
- new CSharpToken(" ", CSharpTokenType.WhiteSpace),
- new CSharpToken("\r\n", CSharpTokenType.NewLine));
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.Transition, "@"),
+ SyntaxFactory.Token(SyntaxKind.Identifier, "something"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
}
[Fact]
@@ -25,18 +26,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"@*included*@\r\n @something \"value\"\r\n @this is ignored",
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken("included", CSharpTokenType.RazorComment),
- new CSharpToken("*", CSharpTokenType.RazorCommentStar),
- new CSharpToken("@", CSharpTokenType.RazorCommentTransition),
- new CSharpToken("\r\n", CSharpTokenType.NewLine),
- new CSharpToken(" ", CSharpTokenType.WhiteSpace),
- new CSharpToken("@", CSharpTokenType.Transition),
- new CSharpToken("something", CSharpTokenType.Identifier),
- new CSharpToken(" ", CSharpTokenType.WhiteSpace),
- new CSharpToken("\"value\"", CSharpTokenType.StringLiteral),
- new CSharpToken("\r\n", CSharpTokenType.NewLine));
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorComment, "included"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.Transition, "@"),
+ SyntaxFactory.Token(SyntaxKind.Identifier, "something"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.StringLiteral, "\"value\""),
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
}
internal override object CreateTokenizer(ITextDocument source)
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
index cffd0a0d0d..0df36e7a81 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/DirectiveHtmlTokenizerTest.cs
@@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Legacy
@@ -12,9 +13,9 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n Ignored
",
- new HtmlToken("\r\n", HtmlTokenType.NewLine),
- new HtmlToken(" ", HtmlTokenType.WhiteSpace),
- new HtmlToken("<", HtmlTokenType.OpenAngle));
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
}
[Fact]
@@ -22,15 +23,15 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
TestTokenizer(
"\r\n @*included*@ Ignored
",
- new HtmlToken("\r\n", HtmlTokenType.NewLine),
- new HtmlToken(" ", HtmlTokenType.WhiteSpace),
- new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
- new HtmlToken("*", HtmlTokenType.RazorCommentStar),
- new HtmlToken("included", HtmlTokenType.RazorComment),
- new HtmlToken("*", HtmlTokenType.RazorCommentStar),
- new HtmlToken("@", HtmlTokenType.RazorCommentTransition),
- new HtmlToken(" ", HtmlTokenType.WhiteSpace),
- new HtmlToken("<", HtmlTokenType.OpenAngle));
+ SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorComment, "included"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+ SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+ SyntaxFactory.Token(SyntaxKind.Whitespace, " "),
+ SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
}
internal override object CreateTokenizer(ITextDocument source)
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
index 839eb92353..7a08ee0e0f 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/Legacy/HtmlMarkupParserTests.cs
@@ -2,23 +2,24 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Language.Test.Legacy
{
public class HtmlMarkupParserTests
{
- private static readonly HtmlToken doubleHyphenToken = new HtmlToken("--", HtmlTokenType.DoubleHyphen);
+ private static readonly SyntaxToken doubleHyphenToken = SyntaxFactory.Token(SyntaxKind.DoubleHyphen, "--");
public static IEnumerable