huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Get rid of locations on Tokens.

This commit is contained in:
Ryan Nowak 2016-12-20 16:04:29 -08:00
parent bd4300d8cc
commit f191cc62a2
33 changed files with 413 additions and 441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpCodeParser.cs
View File

@ -165,6 +165,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
throw new InvalidOperationException(LegacyResources.Parser_Context_Not_Set); throw new InvalidOperationException(LegacyResources.Parser_Context_Not_Set);
} }
Span.Start = CurrentLocation;
// Unless changed, the block is a statement block // Unless changed, the block is a statement block
using (Context.Builder.StartBlock(BlockType.Statement)) using (Context.Builder.StartBlock(BlockType.Statement))
{ {
@ -179,7 +181,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
var split = Language.SplitSymbol(CurrentSymbol, 1, CSharpSymbolType.Transition); var split = Language.SplitSymbol(CurrentSymbol, 1, CSharpSymbolType.Transition);
current = split.Item1; current = split.Item1;
Context.Source.Position = split.Item2.Start.AbsoluteIndex;
// Back up to the end of the transition
Context.Source.Position -= split.Item2.Content.Length;
NextToken(); NextToken();
} }
else if (At(CSharpSymbolType.Transition)) else if (At(CSharpSymbolType.Transition))
@ -201,6 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// No "@" => Jump straight to AfterTransition // No "@" => Jump straight to AfterTransition
AfterTransition(); AfterTransition();
} }
Output(SpanKind.Code); Output(SpanKind.Code);
} }
} }
@ -258,7 +263,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
StringComparison.Ordinal)) StringComparison.Ordinal))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatParseError_HelperDirectiveNotAvailable( LegacyResources.FormatParseError_HelperDirectiveNotAvailable(
SyntaxConstants.CSharp.HelperKeyword), SyntaxConstants.CSharp.HelperKeyword),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
@ -297,21 +302,21 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (At(CSharpSymbolType.WhiteSpace) || At(CSharpSymbolType.NewLine)) if (At(CSharpSymbolType.WhiteSpace) || At(CSharpSymbolType.NewLine))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.ParseError_Unexpected_WhiteSpace_At_Start_Of_CodeBlock_CS, LegacyResources.ParseError_Unexpected_WhiteSpace_At_Start_Of_CodeBlock_CS,
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
} }
else if (EndOfFile) else if (EndOfFile)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.ParseError_Unexpected_EndOfFile_At_Start_Of_CodeBlock, LegacyResources.ParseError_Unexpected_EndOfFile_At_Start_Of_CodeBlock,
length: 1 /* end of file */); length: 1 /* end of file */);
} }
else else
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatParseError_Unexpected_Character_At_Start_Of_CodeBlock_CS( LegacyResources.FormatParseError_Unexpected_Character_At_Start_Of_CodeBlock_CS(
CurrentSymbol.Content), CurrentSymbol.Content),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
@ -328,7 +333,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void VerbatimBlock() private void VerbatimBlock()
{ {
Assert(CSharpSymbolType.LeftBrace); Assert(CSharpSymbolType.LeftBrace);
var block = new Block(LegacyResources.BlockName_Code, CurrentLocation); var block = new Block(LegacyResources.BlockName_Code, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
// Set up the "{" span and output // Set up the "{" span and output
@ -567,7 +572,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void ExplicitExpression() private void ExplicitExpression()
{ {
var block = new Block(LegacyResources.BlockName_ExplicitExpression, CurrentLocation); var block = new Block(LegacyResources.BlockName_ExplicitExpression, CurrentStart);
Assert(CSharpSymbolType.LeftParenthesis); Assert(CSharpSymbolType.LeftParenthesis);
AcceptAndMoveNext(); AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.None; Span.EditHandler.AcceptedCharacters = AcceptedCharacters.None;
@ -595,7 +600,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// If necessary, put an empty-content marker symbol here // If necessary, put an empty-content marker symbol here
if (Span.Symbols.Count == 0) if (Span.Symbols.Count == 0)
{ {
Accept(new CSharpSymbol(CurrentLocation, string.Empty, CSharpSymbolType.Unknown)); Accept(new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown));
} }
// Output the content span and then capture the ")" // Output the content span and then capture the ")"
@ -617,7 +622,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (Context.Builder.ActiveBlocks.Any(block => block.Type == BlockType.Template)) if (Context.Builder.ActiveBlocks.Any(block => block.Type == BlockType.Template))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.ParseError_InlineMarkup_Blocks_Cannot_Be_Nested, LegacyResources.ParseError_InlineMarkup_Blocks_Cannot_Be_Nested,
length: 1 /* @ */); length: 1 /* @ */);
} }
@ -643,12 +648,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void NestedBlock() private void NestedBlock()
{ {
Output(SpanKind.Code); Output(SpanKind.Code);
var wasNested = IsNested; var wasNested = IsNested;
IsNested = true; IsNested = true;
using (PushSpanConfig()) using (PushSpanConfig())
{ {
ParseBlock(); ParseBlock();
} }
Span.Start = CurrentLocation;
Initialize(Span); Initialize(Span);
IsNested = wasNested; IsNested = wasNested;
NextToken(); NextToken();
@ -683,12 +691,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// as C#; it should be handled as a period because it's wrapped in markup. // as C#; it should be handled as a period because it's wrapped in markup.
var wasNested = IsNested; var wasNested = IsNested;
IsNested = false; IsNested = false;
using (PushSpanConfig()) using (PushSpanConfig())
{ {
parseAction(HtmlParser); parseAction(HtmlParser);
} }
Span.Start = CurrentLocation;
Initialize(Span); Initialize(Span);
IsNested = wasNested; IsNested = wasNested;
NextToken(); NextToken();
} }
@ -712,7 +725,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected virtual void ReservedDirective(bool topLevel) protected virtual void ReservedDirective(bool topLevel)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatParseError_ReservedWord(CurrentSymbol.Content), LegacyResources.FormatParseError_ReservedWord(CurrentSymbol.Content),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -780,7 +793,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void UsingKeyword(bool topLevel) private void UsingKeyword(bool topLevel)
{ {
Assert(CSharpKeyword.Using); Assert(CSharpKeyword.Using);
var block = new Block(CurrentSymbol); var block = new Block(CurrentSymbol, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
@ -817,6 +830,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Set block type to directive // Set block type to directive
Context.Builder.CurrentBlock.Type = BlockType.Directive; Context.Builder.CurrentBlock.Type = BlockType.Directive;
var start = CurrentStart;
if (At(CSharpSymbolType.Identifier)) if (At(CSharpSymbolType.Identifier))
{ {
// non-static using // non-static using
@ -849,8 +863,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.AnyExceptNewline; Span.EditHandler.AcceptedCharacters = AcceptedCharacters.AnyExceptNewline;
Span.ChunkGenerator = new AddImportChunkGenerator( Span.ChunkGenerator = new AddImportChunkGenerator(new LocationTagged<string>(
Span.GetContent(symbols => symbols.Skip(1))); string.Concat(Span.Symbols.Skip(1).Select(s => s.Content)),
start));
// Optional ";" // Optional ";"
if (EnsureCurrent()) if (EnsureCurrent())
@ -990,7 +1005,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
return; return;
} }
var block = new Block(CurrentSymbol); var block = new Block(CurrentSymbol, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -1016,7 +1031,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (!At(CSharpSymbolType.LeftBrace)) if (!At(CSharpSymbolType.LeftBrace))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatParseError_SingleLine_ControlFlowStatements_Not_Allowed( LegacyResources.FormatParseError_SingleLine_ControlFlowStatements_Not_Allowed(
Language.GetSample(CSharpSymbolType.LeftBrace), Language.GetSample(CSharpSymbolType.LeftBrace),
CurrentSymbol.Content), CurrentSymbol.Content),
@ -1031,7 +1046,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void UnconditionalBlock() private void UnconditionalBlock()
{ {
Assert(CSharpSymbolType.Keyword); Assert(CSharpSymbolType.Keyword);
var block = new Block(CurrentSymbol); var block = new Block(CurrentSymbol, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
ExpectCodeBlock(block); ExpectCodeBlock(block);
@ -1041,7 +1056,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
Assert(CSharpKeyword.Catch); Assert(CSharpKeyword.Catch);
var block = new Block(CurrentSymbol); var block = new Block(CurrentSymbol, CurrentStart);
// Accept "catch" // Accept "catch"
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -1075,7 +1090,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void ConditionalBlock(bool topLevel) private void ConditionalBlock(bool topLevel)
{ {
Assert(CSharpSymbolType.Keyword); Assert(CSharpSymbolType.Keyword);
var block = new Block(CurrentSymbol); var block = new Block(CurrentSymbol, CurrentStart);
ConditionalBlock(block); ConditionalBlock(block);
if (topLevel) if (topLevel)
{ {
@ -1125,8 +1140,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Accept whitespace but always keep the last whitespace node so we can put it back if necessary // Accept whitespace but always keep the last whitespace node so we can put it back if necessary
var lastWhitespace = AcceptWhiteSpaceInLines(); var lastWhitespace = AcceptWhiteSpaceInLines();
Debug.Assert(lastWhitespace == null ||
(lastWhitespace.Start.AbsoluteIndex + lastWhitespace.Content.Length == CurrentLocation.AbsoluteIndex));
if (EndOfFile) if (EndOfFile)
{ {
@ -1138,7 +1151,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
var type = CurrentSymbol.Type; var type = CurrentSymbol.Type;
var loc = CurrentLocation; var loc = CurrentStart;
// Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text // Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text
var isSingleLineMarkup = type == CSharpSymbolType.Transition && var isSingleLineMarkup = type == CSharpSymbolType.Transition &&
@ -1208,7 +1221,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
break; break;
case CSharpSymbolType.LeftBrace: case CSharpSymbolType.LeftBrace:
// Verbatim Block // Verbatim Block
block = block ?? new Block(LegacyResources.BlockName_Code, CurrentLocation); block = block ?? new Block(LegacyResources.BlockName_Code, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
CodeBlock(block); CodeBlock(block);
break; break;
@ -1260,7 +1273,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (At(CSharpSymbolType.LeftBrace)) if (At(CSharpSymbolType.LeftBrace))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.ParseError_Unexpected_Nested_CodeBlock, LegacyResources.ParseError_Unexpected_Nested_CodeBlock,
length: 1 /* { */); length: 1 /* { */);
} }
@ -1280,7 +1293,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
while (!EndOfFile) while (!EndOfFile)
{ {
var bookmark = CurrentLocation.AbsoluteIndex; var bookmark = CurrentStart.AbsoluteIndex;
IEnumerable<CSharpSymbol> read = ReadWhile(sym => sym.Type != CSharpSymbolType.Semicolon && IEnumerable<CSharpSymbol> read = ReadWhile(sym => sym.Type != CSharpSymbolType.Semicolon &&
sym.Type != CSharpSymbolType.RazorCommentTransition && sym.Type != CSharpSymbolType.RazorCommentTransition &&
sym.Type != CSharpSymbolType.Transition && sym.Type != CSharpSymbolType.Transition &&
@ -1479,7 +1492,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (EndOfFile) if (EndOfFile)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, tokenDescriptor.Kind.ToString().ToLowerInvariant()), LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, tokenDescriptor.Kind.ToString().ToLowerInvariant()),
length: 1); length: 1);
return; return;
@ -1492,7 +1505,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (!NamespaceOrTypeName()) if (!NamespaceOrTypeName())
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatDirectiveExpectsTypeName(descriptor.Name), LegacyResources.FormatDirectiveExpectsTypeName(descriptor.Name),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
@ -1509,7 +1522,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else else
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatDirectiveExpectsIdentifier(descriptor.Name), LegacyResources.FormatDirectiveExpectsIdentifier(descriptor.Name),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
return; return;
@ -1528,7 +1541,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else else
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, tokenDescriptor.Value), LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, tokenDescriptor.Value),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
return; return;
@ -1555,7 +1568,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else if (!EndOfFile) else if (!EndOfFile)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, Environment.NewLine), LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, Environment.NewLine),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
} }
@ -1573,12 +1586,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// as C#; it should be handled as a period because it's wrapped in markup. // as C#; it should be handled as a period because it's wrapped in markup.
var wasNested = IsNested; var wasNested = IsNested;
IsNested = false; IsNested = false;
using (PushSpanConfig()) using (PushSpanConfig())
{ {
HtmlParser.ParseRazorBlock(Tuple.Create("{", "}"), caseSensitive: true); HtmlParser.ParseRazorBlock(Tuple.Create("{", "}"), caseSensitive: true);
} }
Span.Start = CurrentLocation;
Initialize(Span); Initialize(Span);
IsNested = wasNested; IsNested = wasNested;
NextToken(); NextToken();
}); });
break; break;
@ -1602,14 +1620,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (EndOfFile) if (EndOfFile)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, "{"), LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, "{"),
length: 1 /* { */); length: 1 /* { */);
} }
else if (!At(CSharpSymbolType.LeftBrace)) else if (!At(CSharpSymbolType.LeftBrace))
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, "{"), LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, "{"),
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
} }
@ -1617,7 +1635,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
var editHandler = new AutoCompleteEditHandler(Language.TokenizeString, autoCompleteAtEndOfSpan: true); var editHandler = new AutoCompleteEditHandler(Language.TokenizeString, autoCompleteAtEndOfSpan: true);
Span.EditHandler = editHandler; Span.EditHandler = editHandler;
var startingBraceLocation = CurrentLocation; var startingBraceLocation = CurrentStart;
Accept(CurrentSymbol); Accept(CurrentSymbol);
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKind.MetaCode, AcceptedCharacters.None); Output(SpanKind.MetaCode, AcceptedCharacters.None);
@ -1678,7 +1696,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Set the block type // Set the block type
Context.Builder.CurrentBlock.Type = BlockType.Directive; Context.Builder.CurrentBlock.Type = BlockType.Directive;
var keywordLength = Span.GetContent().Value.Length; var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
// Accept whitespace // Accept whitespace
var remainingWhitespace = AcceptSingleWhiteSpaceCharacter(); var remainingWhitespace = AcceptSingleWhiteSpaceCharacter();
@ -1714,7 +1732,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
// Pull out the type name // Pull out the type name
string baseType = Span.GetContent(); string baseType = string.Concat(Span.Symbols.Select(s => s.Content));
// Set up chunk generation // Set up chunk generation
Span.ChunkGenerator = createChunkGenerator(baseType.Trim()); Span.ChunkGenerator = createChunkGenerator(baseType.Trim());
@ -1727,7 +1745,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void TagHelperDirective(string keyword, Func<string, ISpanChunkGenerator> chunkGeneratorFactory) private void TagHelperDirective(string keyword, Func<string, ISpanChunkGenerator> chunkGeneratorFactory)
{ {
AssertDirective(keyword); AssertDirective(keyword);
var keywordStartLocation = CurrentLocation; var keywordStartLocation = CurrentStart;
// Accept the directive name // Accept the directive name
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -1735,7 +1753,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Set the block type // Set the block type
Context.Builder.CurrentBlock.Type = BlockType.Directive; Context.Builder.CurrentBlock.Type = BlockType.Directive;
var keywordLength = Span.GetContent().Value.Length; var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
var foundWhitespace = At(CSharpSymbolType.WhiteSpace); var foundWhitespace = At(CSharpSymbolType.WhiteSpace);
AcceptWhile(CSharpSymbolType.WhiteSpace); AcceptWhile(CSharpSymbolType.WhiteSpace);
@ -1757,14 +1775,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else else
{ {
// Need to grab the current location before we accept until the end of the line. // Need to grab the current location before we accept until the end of the line.
var startLocation = CurrentLocation; var startLocation = CurrentStart;
// Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code // Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code
// etc. // etc.
AcceptUntil(CSharpSymbolType.NewLine); AcceptUntil(CSharpSymbolType.NewLine);
// Pull out the value and remove whitespaces and optional quotes // Pull out the value and remove whitespaces and optional quotes
var rawValue = Span.GetContent().Value.Trim(); var rawValue = string.Concat(Span.Symbols.Select(s => s.Content)).Trim();
var startsWithQuote = rawValue.StartsWith("\"", StringComparison.Ordinal); var startsWithQuote = rawValue.StartsWith("\"", StringComparison.Ordinal);
var endsWithQuote = rawValue.EndsWith("\"", StringComparison.Ordinal); var endsWithQuote = rawValue.EndsWith("\"", StringComparison.Ordinal);
@ -1807,8 +1825,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Start = start; Start = start;
} }
public Block(CSharpSymbol symbol) public Block(CSharpSymbol symbol, SourceLocation start)
: this(GetName(symbol), symbol.Start) : this(GetName(symbol), start)
{ {
} }

8
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpLanguageCharacteristics.cs
View File

@ -74,9 +74,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return new CSharpTokenizer(source); return new CSharpTokenizer(source);
} }
protected override CSharpSymbol CreateSymbol(SourceLocation location, string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors) protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors)
{ {
return new CSharpSymbol(location, content, type, errors); return new CSharpSymbol(content, type, errors);
} }
public override string GetSample(CSharpSymbolType type) public override string GetSample(CSharpSymbolType type)
@ -111,9 +111,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return sample; return sample;
} }
public override CSharpSymbol CreateMarkerSymbol(SourceLocation location) public override CSharpSymbol CreateMarkerSymbol()
{ {
return new CSharpSymbol(location, string.Empty, CSharpSymbolType.Unknown); return new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown);
} }
public override CSharpSymbolType GetKnownSymbolType(KnownSymbolType type) public override CSharpSymbolType GetKnownSymbolType(KnownSymbolType type)

33
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpSymbol.cs
View File

@ -8,17 +8,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
internal class CSharpSymbol : SymbolBase<CSharpSymbolType> internal class CSharpSymbol : SymbolBase<CSharpSymbolType>
{ {
public CSharpSymbol(int absoluteIndex, int lineIndex, int characterIndex, string content, CSharpSymbolType type) public CSharpSymbol(
: this(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, RazorError.EmptyArray) string content,
{ CSharpSymbolType type)
if (content == null) : base(content, type, RazorError.EmptyArray)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpSymbol(SourceLocation start, string content, CSharpSymbolType type)
: this(start, content, type, RazorError.EmptyArray)
{ {
if (content == null) if (content == null)
{ {
@ -27,26 +20,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
public CSharpSymbol( public CSharpSymbol(
int offset,
int line,
int column,
string content, string content,
CSharpSymbolType type, CSharpSymbolType type,
IReadOnlyList<RazorError> errors) IReadOnlyList<RazorError> errors)
: base(new SourceLocation(offset, line, column), content, type, errors) : base(content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpSymbol(
SourceLocation start,
string content,
CSharpSymbolType type,
IReadOnlyList<RazorError> errors)
: base(start, content, type, errors)
{ {
if (content == null) if (content == null)
{ {

13
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpTokenizer.cs
View File

@ -168,9 +168,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
} }
protected override CSharpSymbol CreateSymbol(SourceLocation start, string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors) protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors)
{ {
return new CSharpSymbol(start, content, type, errors); return new CSharpSymbol(content, type, errors);
} }
private StateResult Data() private StateResult Data()
@ -548,13 +548,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
type = CSharpSymbolType.Keyword; type = CSharpSymbolType.Keyword;
} }
symbol = new CSharpSymbol(CurrentStart, symbolContent, type) symbol = new CSharpSymbol(symbolContent, type)
{ {
Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null, Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null,
}; };
Buffer.Clear();
CurrentErrors.Clear();
} }
StartSymbol();
return Stay(symbol); return Stay(symbol);
} }

2
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ConditionalAttributeCollapser.cs
View File

@ -39,7 +39,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
// Create a new span containing this content // Create a new span containing this content
var span = new SpanBuilder(); var span = new SpanBuilder(block.Children[0].Start);
span.EditHandler = SpanEditHandler.CreateDefault(HtmlLanguageCharacteristics.Instance.TokenizeString); span.EditHandler = SpanEditHandler.CreateDefault(HtmlLanguageCharacteristics.Instance.TokenizeString);
Debug.Assert(block.Children.Count > 0); Debug.Assert(block.Children.Count > 0);

8
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlLanguageCharacteristics.cs
View File

@ -95,9 +95,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
} }
public override HtmlSymbol CreateMarkerSymbol(SourceLocation location) public override HtmlSymbol CreateMarkerSymbol()
{ {
return new HtmlSymbol(location, string.Empty, HtmlSymbolType.Unknown); return new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
} }
public override HtmlSymbolType GetKnownSymbolType(KnownSymbolType type) public override HtmlSymbolType GetKnownSymbolType(KnownSymbolType type)
@ -125,9 +125,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
} }
protected override HtmlSymbol CreateSymbol(SourceLocation location, string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors) protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors)
{ {
return new HtmlSymbol(location, content, type, errors); return new HtmlSymbol(content, type, errors);
} }
} }
} }

78
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlMarkupParser.cs
View File

@ -219,10 +219,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
AddMarkerSymbolIfNecessary(); AddMarkerSymbolIfNecessary();
Output(SpanKind.Markup); Output(SpanKind.Markup);
using (PushSpanConfig()) using (PushSpanConfig())
{ {
CodeParser.ParseBlock(); CodeParser.ParseBlock();
} }
Span.Start = CurrentLocation;
Initialize(Span); Initialize(Span);
NextToken(); NextToken();
} }
@ -271,6 +274,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
using (Context.Builder.StartBlock(BlockType.Markup)) using (Context.Builder.StartBlock(BlockType.Markup))
{ {
Span.Start = CurrentLocation;
if (!NextToken()) if (!NextToken())
{ {
return; return;
@ -305,7 +310,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else else
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentSymbol.Start, CurrentStart,
LegacyResources.ParseError_MarkupBlock_Must_Start_With_Tag, LegacyResources.ParseError_MarkupBlock_Must_Start_With_Tag,
CurrentSymbol.Content.Length); CurrentSymbol.Content.Length);
} }
@ -393,10 +398,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else else
{ {
_bufferedOpenAngle = null; _bufferedOpenAngle = null;
_lastTagStart = CurrentLocation; _lastTagStart = CurrentStart;
Assert(HtmlSymbolType.OpenAngle); Assert(HtmlSymbolType.OpenAngle);
_bufferedOpenAngle = CurrentSymbol; _bufferedOpenAngle = CurrentSymbol;
var tagStart = CurrentLocation; var tagStart = CurrentStart;
if (!NextToken()) if (!NextToken())
{ {
Accept(_bufferedOpenAngle); Accept(_bufferedOpenAngle);
@ -467,7 +472,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (tags.Count == 0) if (tags.Count == 0)
{ {
Context.ErrorSink.OnError( Context.ErrorSink.OnError(
CurrentLocation, CurrentStart,
LegacyResources.ParseError_OuterTagMissingName, LegacyResources.ParseError_OuterTagMissingName,
length: 1 /* end of file */); length: 1 /* end of file */);
} }
@ -620,7 +625,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Accept(_bufferedOpenAngle); Accept(_bufferedOpenAngle);
Accept(solidus); Accept(solidus);
var textLocation = CurrentLocation; var textLocation = CurrentStart;
Assert(HtmlSymbolType.Text); Assert(HtmlSymbolType.Text);
AcceptAndMoveNext(); AcceptAndMoveNext();
@ -789,8 +794,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
IEnumerable<HtmlSymbol> whitespaceAfterAttributeName) IEnumerable<HtmlSymbol> whitespaceAfterAttributeName)
{ {
// First, determine if this is a 'data-' attribute (since those can't use conditional attributes) // First, determine if this is a 'data-' attribute (since those can't use conditional attributes)
var name = nameSymbols.GetContent(Span.Start); var name = string.Concat(nameSymbols.Select(s => s.Content));
var attributeCanBeConditional = !name.Value.StartsWith("data-", StringComparison.OrdinalIgnoreCase); var attributeCanBeConditional = !name.StartsWith("data-", StringComparison.OrdinalIgnoreCase);
// Accept the whitespace and name // Accept the whitespace and name
Accept(whitespace); Accept(whitespace);
@ -818,7 +823,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
// We now have the prefix: (i.e. ' foo="') // We now have the prefix: (i.e. ' foo="')
var prefix = Span.GetContent(); var prefix = new LocationTagged<string>(string.Concat(Span.Symbols.Select(s => s.Content)), Span.Start);
if (attributeCanBeConditional) if (attributeCanBeConditional)
{ {
@ -837,10 +842,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
// Capture the suffix // Capture the suffix
var suffix = new LocationTagged<string>(string.Empty, CurrentLocation); var suffix = new LocationTagged<string>(string.Empty, CurrentStart);
if (quote != HtmlSymbolType.Unknown && At(quote)) if (quote != HtmlSymbolType.Unknown && At(quote))
{ {
suffix = CurrentSymbol.GetContent(); suffix = new LocationTagged<string>(CurrentSymbol.Content, CurrentStart);
AcceptAndMoveNext(); AcceptAndMoveNext();
} }
@ -881,7 +886,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void AttributeValue(HtmlSymbolType quote) private void AttributeValue(HtmlSymbolType quote)
{ {
var prefixStart = CurrentLocation; var prefixStart = CurrentStart;
var prefix = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine); var prefix = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine);
if (At(HtmlSymbolType.Transition)) if (At(HtmlSymbolType.Transition))
@ -895,8 +900,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Render a single "@" in place of "@@". // Render a single "@" in place of "@@".
Span.ChunkGenerator = new LiteralAttributeChunkGenerator( Span.ChunkGenerator = new LiteralAttributeChunkGenerator(
prefix.GetContent(prefixStart), new LocationTagged<string>(string.Concat(prefix.Select(s => s.Content)), prefixStart),
new LocationTagged<string>(CurrentSymbol.GetContent(), CurrentLocation)); new LocationTagged<string>(CurrentSymbol.Content, CurrentStart));
AcceptAndMoveNext(); AcceptAndMoveNext();
Output(SpanKind.Markup, AcceptedCharacters.None); Output(SpanKind.Markup, AcceptedCharacters.None);
@ -908,7 +913,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else else
{ {
Accept(prefix); Accept(prefix);
var valueStart = CurrentLocation; var valueStart = CurrentStart;
PutCurrentBack(); PutCurrentBack();
// Output the prefix but as a null-span. DynamicAttributeBlockChunkGenerator will render it // Output the prefix but as a null-span. DynamicAttributeBlockChunkGenerator will render it
@ -918,7 +923,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
using (Context.Builder.StartBlock(BlockType.Markup)) using (Context.Builder.StartBlock(BlockType.Markup))
{ {
Context.Builder.CurrentBlock.ChunkGenerator = Context.Builder.CurrentBlock.ChunkGenerator =
new DynamicAttributeBlockChunkGenerator(prefix.GetContent(prefixStart), valueStart); new DynamicAttributeBlockChunkGenerator(
new LocationTagged<string>(string.Concat(prefix.Select(s => s.Content)), prefixStart),
valueStart);
OtherParserBlock(); OtherParserBlock();
} }
@ -931,6 +938,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Literal value // Literal value
// 'quote' should be "Unknown" if not quoted and symbols coming from the tokenizer should never have // 'quote' should be "Unknown" if not quoted and symbols coming from the tokenizer should never have
// "Unknown" type. // "Unknown" type.
var valueStart = CurrentStart;
var value = ReadWhile(sym => var value = ReadWhile(sym =>
// These three conditions find separators which break the attribute value into portions // These three conditions find separators which break the attribute value into portions
sym.Type != HtmlSymbolType.WhiteSpace && sym.Type != HtmlSymbolType.WhiteSpace &&
@ -941,8 +949,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
!IsEndOfAttributeValue(quote, sym)); !IsEndOfAttributeValue(quote, sym));
Accept(value); Accept(value);
Span.ChunkGenerator = new LiteralAttributeChunkGenerator( Span.ChunkGenerator = new LiteralAttributeChunkGenerator(
prefix.GetContent(prefixStart), new LocationTagged<string>(string.Concat(prefix.Select(s => s.Content)), prefixStart),
value.GetContent(prefixStart)); new LocationTagged<string>(string.Concat(value.Select(s => s.Content)), valueStart));
} }
Output(SpanKind.Markup); Output(SpanKind.Markup);
} }
@ -1037,11 +1045,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (potentialTagNameSymbol == null || potentialTagNameSymbol.Type != HtmlSymbolType.Text) if (potentialTagNameSymbol == null || potentialTagNameSymbol.Type != HtmlSymbolType.Text)
{ {
tagName = new HtmlSymbol(potentialTagNameSymbol.Start, string.Empty, HtmlSymbolType.Unknown); tagName = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
} }
else if (bangSymbol != null) else if (bangSymbol != null)
{ {
tagName = new HtmlSymbol(bangSymbol.Start, "!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text); tagName = new HtmlSymbol("!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text);
} }
else else
{ {
@ -1059,12 +1067,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null; Span.ChunkGenerator = SpanChunkGenerator.Null;
Accept(_bufferedOpenAngle); Accept(_bufferedOpenAngle);
var textLocation = CurrentLocation; var textLocation = CurrentStart;
Assert(HtmlSymbolType.Text); Assert(HtmlSymbolType.Text);
AcceptAndMoveNext(); AcceptAndMoveNext();
var bookmark = CurrentLocation.AbsoluteIndex; var bookmark = CurrentStart.AbsoluteIndex;
IEnumerable<HtmlSymbol> tokens = ReadWhile(IsSpacingToken(includeNewLines: true)); IEnumerable<HtmlSymbol> tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
var empty = At(HtmlSymbolType.ForwardSlash); var empty = At(HtmlSymbolType.ForwardSlash);
if (empty) if (empty)
@ -1072,7 +1080,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Accept(tokens); Accept(tokens);
Assert(HtmlSymbolType.ForwardSlash); Assert(HtmlSymbolType.ForwardSlash);
AcceptAndMoveNext(); AcceptAndMoveNext();
bookmark = CurrentLocation.AbsoluteIndex; bookmark = CurrentStart.AbsoluteIndex;
tokens = ReadWhile(IsSpacingToken(includeNewLines: true)); tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
} }
@ -1151,7 +1159,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Technically, void elements like "meta" are not allowed to have end tags. Just in case they do, // Technically, void elements like "meta" are not allowed to have end tags. Just in case they do,
// we need to look ahead at the next set of tokens. If we see "<", "/", tag name, accept it and the ">" following it // we need to look ahead at the next set of tokens. If we see "<", "/", tag name, accept it and the ">" following it
// Place a bookmark // Place a bookmark
var bookmark = CurrentLocation.AbsoluteIndex; var bookmark = CurrentStart.AbsoluteIndex;
// Skip whitespace // Skip whitespace
IEnumerable<HtmlSymbol> whiteSpace = ReadWhile(IsSpacingToken(includeNewLines: true)); IEnumerable<HtmlSymbol> whiteSpace = ReadWhile(IsSpacingToken(includeNewLines: true));
@ -1230,7 +1238,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
while (!seenEndScript && !EndOfFile) while (!seenEndScript && !EndOfFile)
{ {
SkipToAndParseCode(HtmlSymbolType.OpenAngle); SkipToAndParseCode(HtmlSymbolType.OpenAngle);
var tagStart = CurrentLocation; var tagStart = CurrentStart;
if (NextIs(HtmlSymbolType.ForwardSlash)) if (NextIs(HtmlSymbolType.ForwardSlash))
{ {
@ -1438,6 +1446,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
using (Context.Builder.StartBlock(BlockType.Markup)) using (Context.Builder.StartBlock(BlockType.Markup))
{ {
Span.Start = CurrentLocation;
NextToken(); NextToken();
while (!EndOfFile) while (!EndOfFile)
{ {
@ -1586,6 +1596,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
using (PushSpanConfig(DefaultMarkupSpan)) using (PushSpanConfig(DefaultMarkupSpan))
{ {
Span.Start = CurrentLocation;
using (Context.Builder.StartBlock(BlockType.Markup)) using (Context.Builder.StartBlock(BlockType.Markup))
{ {
NextToken(); NextToken();
@ -1652,7 +1664,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
EnsureCurrent(); EnsureCurrent();
if (string.Equals(CurrentSymbol.Content, nestingSequenceComponents[0], Comparison)) if (string.Equals(CurrentSymbol.Content, nestingSequenceComponents[0], Comparison))
{ {
var bookmark = CurrentSymbol.Start.AbsoluteIndex; var bookmark = Context.Source.Position - CurrentSymbol.Content.Length;
try try
{ {
foreach (string component in nestingSequenceComponents) foreach (string component in nestingSequenceComponents)
@ -1706,7 +1718,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (string.Equals(possibleStart, sequence, Comparison)) if (string.Equals(possibleStart, sequence, Comparison))
{ {
// Capture the current symbol and "put it back" (really we just want to clear CurrentSymbol) // Capture the current symbol and "put it back" (really we just want to clear CurrentSymbol)
var bookmark = Context.Source.Position; var bookmark = CurrentStart;
var sym = CurrentSymbol; var sym = CurrentSymbol;
PutCurrentBack(); PutCurrentBack();
@ -1717,6 +1729,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
pair = Language.SplitSymbol(pair.Item2, sequence.Length, HtmlSymbolType.Text); pair = Language.SplitSymbol(pair.Item2, sequence.Length, HtmlSymbolType.Text);
var sequenceToken = pair.Item1; var sequenceToken = pair.Item1;
var postSequence = pair.Item2; var postSequence = pair.Item2;
var postSequenceBookmark = bookmark.AbsoluteIndex + preSequence.Content.Length + pair.Item1.Content.Length;
// Accept the first chunk (up to the nesting sequence we just saw) // Accept the first chunk (up to the nesting sequence we just saw)
if (!string.IsNullOrEmpty(preSequence.Content)) if (!string.IsNullOrEmpty(preSequence.Content))
@ -1729,22 +1742,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// This is 'popping' the final entry on the stack of nesting sequences // This is 'popping' the final entry on the stack of nesting sequences
// A caller higher in the parsing stack will accept the sequence token, so advance // A caller higher in the parsing stack will accept the sequence token, so advance
// to it // to it
Context.Source.Position = sequenceToken.Start.AbsoluteIndex; Context.Source.Position = bookmark.AbsoluteIndex + preSequence.Content.Length;
} }
else else
{ {
// This isn't the end of the last nesting sequence, accept the token and keep going // This isn't the end of the last nesting sequence, accept the token and keep going
Accept(sequenceToken); Accept(sequenceToken);
// Position at the start of the postSequence symbol // Position at the start of the postSequence symbol, which might be null.
if (postSequence != null) Context.Source.Position = postSequenceBookmark;
{
Context.Source.Position = postSequence.Start.AbsoluteIndex;
}
else
{
Context.Source.Position = bookmark;
}
} }
// Return the value we were asked to return if matched, since we found a nesting sequence // Return the value we were asked to return if matched, since we found a nesting sequence

31
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlSymbol.cs
View File

@ -8,17 +8,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
internal class HtmlSymbol : SymbolBase<HtmlSymbolType> internal class HtmlSymbol : SymbolBase<HtmlSymbolType>
{ {
public HtmlSymbol(int absoluteIndex, int lineIndex, int characterIndex, string content, HtmlSymbolType type) public HtmlSymbol(string content, HtmlSymbolType type)
: this(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, RazorError.EmptyArray) : base(content, type, RazorError.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public HtmlSymbol(SourceLocation start, string content, HtmlSymbolType type)
: base(start, content, type, RazorError.EmptyArray)
{ {
if (content == null) if (content == null)
{ {
@ -27,26 +18,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
public HtmlSymbol( public HtmlSymbol(
int absoluteIndex,
int lineIndex,
int characterIndex,
string content, string content,
HtmlSymbolType type, HtmlSymbolType type,
IReadOnlyList<RazorError> errors) IReadOnlyList<RazorError> errors)
: base(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, errors) : base(content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public HtmlSymbol(
SourceLocation start,
string content,
HtmlSymbolType type,
IReadOnlyList<RazorError> errors)
: base(start, content, type, errors)
{ {
if (content == null) if (content == null)
{ {

4
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlTokenizer.cs
View File

@ -36,9 +36,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
get { return HtmlSymbolType.RazorCommentStar; } get { return HtmlSymbolType.RazorCommentStar; }
} }
protected override HtmlSymbol CreateSymbol(SourceLocation start, string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors) protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors)
{ {
return new HtmlSymbol(start, content, type, errors); return new HtmlSymbol(content, type, errors);
} }
protected override StateResult Dispatch() protected override StateResult Dispatch()

6
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ISymbol.cs
View File

@ -5,10 +5,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
internal interface ISymbol internal interface ISymbol
{ {
SourceLocation Start { get; } Span Parent { get; set; }
string Content { get; } string Content { get; }
void OffsetStart(SourceLocation documentStart); SourceLocation Start { get; }
void ChangeStart(SourceLocation newStart);
} }
} }

11
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LanguageCharacteristics.cs
View File

@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public abstract string GetSample(TSymbolType type); public abstract string GetSample(TSymbolType type);
public abstract TTokenizer CreateTokenizer(ITextDocument source); public abstract TTokenizer CreateTokenizer(ITextDocument source);
public abstract TSymbolType FlipBracket(TSymbolType bracket); public abstract TSymbolType FlipBracket(TSymbolType bracket);
public abstract TSymbol CreateMarkerSymbol(SourceLocation location); public abstract TSymbol CreateMarkerSymbol();
public virtual IEnumerable<TSymbol> TokenizeString(string content) public virtual IEnumerable<TSymbol> TokenizeString(string content)
{ {
@ -30,7 +30,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
TSymbol sym; TSymbol sym;
while ((sym = tok.NextSymbol()) != null) while ((sym = tok.NextSymbol()) != null)
{ {
sym.OffsetStart(start);
yield return sym; yield return sym;
} }
} }
@ -88,12 +87,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public virtual Tuple<TSymbol, TSymbol> SplitSymbol(TSymbol symbol, int splitAt, TSymbolType leftType) public virtual Tuple<TSymbol, TSymbol> SplitSymbol(TSymbol symbol, int splitAt, TSymbolType leftType)
{ {
var left = CreateSymbol(symbol.Start, symbol.Content.Substring(0, splitAt), leftType, RazorError.EmptyArray); var left = CreateSymbol(symbol.Content.Substring(0, splitAt), leftType, RazorError.EmptyArray);
TSymbol right = null; TSymbol right = null;
if (splitAt < symbol.Content.Length) if (splitAt < symbol.Content.Length)
{ {
right = CreateSymbol(SourceLocationTracker.CalculateNewLocation(symbol.Start, left.Content), symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors); right = CreateSymbol(symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors);
} }
return Tuple.Create(left, right); return Tuple.Create(left, right);
} }
@ -104,6 +105,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return type == KnownSymbolType.Unknown || !Equals(GetKnownSymbolType(type), GetKnownSymbolType(KnownSymbolType.Unknown)); return type == KnownSymbolType.Unknown || !Equals(GetKnownSymbolType(type), GetKnownSymbolType(KnownSymbolType.Unknown));
} }
protected abstract TSymbol CreateSymbol(SourceLocation location, string content, TSymbolType type, IReadOnlyList<RazorError> errors); protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList<RazorError> errors);
} }
} }

51
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanBuilder.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
@ -8,26 +9,43 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
internal class SpanBuilder internal class SpanBuilder
{ {
private SourceLocation _start;
private List<ISymbol> _symbols; private List<ISymbol> _symbols;
private SourceLocationTracker _tracker = new SourceLocationTracker(); private SourceLocationTracker _tracker;
public SpanBuilder(Span original) public SpanBuilder(Span original)
{ {
Kind = original.Kind; Kind = original.Kind;
_symbols = new List<ISymbol>(original.Symbols);
EditHandler = original.EditHandler; EditHandler = original.EditHandler;
Start = original.Start; _start = original.Start;
ChunkGenerator = original.ChunkGenerator; ChunkGenerator = original.ChunkGenerator;
_symbols = new List<ISymbol>(original.Symbols);
_tracker = new SourceLocationTracker(original.Start);
} }
public SpanBuilder() public SpanBuilder(SourceLocation location)
{ {
_tracker = new SourceLocationTracker();
Reset(); Reset();
Start = location;
} }
public ISpanChunkGenerator ChunkGenerator { get; set; } public ISpanChunkGenerator ChunkGenerator { get; set; }
public SourceLocation Start { get; set; } public SourceLocation Start
{
get { return _start; }
set
{
_start = value;
_tracker.CurrentLocation = value;
}
}
public SourceLocation End => _tracker.CurrentLocation;
public SpanKind Kind { get; set; } public SpanKind Kind { get; set; }
@ -51,15 +69,24 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Need to potentially allocate a new list because Span.ReplaceWith takes ownership // Need to potentially allocate a new list because Span.ReplaceWith takes ownership
// of the original list. // of the original list.
_symbols = null; _symbols = null;
_symbols = new List<ISymbol>();
EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<ISymbol>()); EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<ISymbol>());
ChunkGenerator = SpanChunkGenerator.Null; ChunkGenerator = SpanChunkGenerator.Null;
Start = SourceLocation.Zero; Start = SourceLocation.Undefined;
} }
public Span Build() public Span Build()
{ {
return new Span(this); var span = new Span(this);
for (var i = 0; i < span.Symbols.Count; i++)
{
var symbol = span.Symbols[i];
symbol.Parent = span;
}
return span;
} }
public void ClearSymbols() public void ClearSymbols()
@ -74,15 +101,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return; return;
} }
if (Symbols.Count == 0) if (Start.Equals(SourceLocation.Undefined))
{ {
Start = symbol.Start; throw new InvalidOperationException("SpanBuilder must have a valid location");
symbol.ChangeStart(SourceLocation.Zero);
_tracker.CurrentLocation = SourceLocation.Zero;
}
else
{
symbol.ChangeStart(_tracker.CurrentLocation);
} }
_symbols.Add(symbol); _symbols.Add(symbol);

1
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanEditHandler.cs
View File

@ -72,7 +72,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
newSpan.ClearSymbols(); newSpan.ClearSymbols();
foreach (ISymbol sym in Tokenizer(newContent)) foreach (ISymbol sym in Tokenizer(newContent))
{ {
sym.OffsetStart(target.Start);
newSpan.Accept(sym); newSpan.Accept(sym);
} }
if (target.Next != null) if (target.Next != null)

50
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolBase.cs
View File

@ -11,7 +11,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
internal abstract class SymbolBase<TType> : ISymbol where TType : struct internal abstract class SymbolBase<TType> : ISymbol where TType : struct
{ {
protected SymbolBase( protected SymbolBase(
SourceLocation start,
string content, string content,
TType type, TType type,
IReadOnlyList<RazorError> errors) IReadOnlyList<RazorError> errors)
@ -21,13 +20,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
throw new ArgumentNullException(nameof(content)); throw new ArgumentNullException(nameof(content));
} }
Start = start;
Content = content; Content = content;
Type = type; Type = type;
Errors = errors; Errors = errors;
} }
public SourceLocation Start { get; private set; } public Span Parent { get; set; }
public IReadOnlyList<RazorError> Errors { get; } public IReadOnlyList<RazorError> Errors { get; }
@ -35,11 +33,35 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public TType Type { get; } public TType Type { get; }
public SourceLocation Start
{
get
{
if (Parent == null)
{
return SourceLocation.Undefined;
}
var tracker = new SourceLocationTracker(Parent.Start);
for (var i = 0; i < Parent.Symbols.Count; i++)
{
var symbol = Parent.Symbols[i];
if (object.ReferenceEquals(this, symbol))
{
break;
}
tracker.UpdateLocation(symbol.Content);
}
return tracker.CurrentLocation;
}
}
public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var other = obj as SymbolBase<TType>; var other = obj as SymbolBase<TType>;
return other != null && return other != null &&
Start.Equals(other.Start) &&
string.Equals(Content, other.Content, StringComparison.Ordinal) && string.Equals(Content, other.Content, StringComparison.Ordinal) &&
Type.Equals(other.Type); Type.Equals(other.Type);
} }
@ -47,26 +69,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public override int GetHashCode() public override int GetHashCode()
{ {
// Hash code should include only immutable properties. // Hash code should include only immutable properties.
var hashCodeCombiner = HashCodeCombiner.Start(); var hash = HashCodeCombiner.Start();
hashCodeCombiner.Add(Content, StringComparer.Ordinal); hash.Add(Content, StringComparer.Ordinal);
hashCodeCombiner.Add(Type); hash.Add(Type);
return hashCodeCombiner; return hash;
} }
public override string ToString() public override string ToString()
{ {
return string.Format(CultureInfo.InvariantCulture, "{0} {1} - [{2}]", Start, Type, Content); return string.Format(CultureInfo.InvariantCulture, "{0} [{1}]", Type, Content);
}
public void OffsetStart(SourceLocation documentStart)
{
Start = documentStart + Start;
}
public void ChangeStart(SourceLocation newStart)
{
Start = newStart;
} }
} }
} }

39
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolExtensions.cs
View File

@ -1,39 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal static class SymbolExtensions
{
public static LocationTagged<string> GetContent(this SpanBuilder span)
{
return GetContent(span, e => e);
}
public static LocationTagged<string> GetContent(this SpanBuilder span, Func<IEnumerable<ISymbol>, IEnumerable<ISymbol>> filter)
{
return GetContent(filter(span.Symbols), span.Start);
}
public static LocationTagged<string> GetContent(this IEnumerable<ISymbol> symbols, SourceLocation spanStart)
{
if (symbols.Any())
{
return new LocationTagged<string>(string.Concat(symbols.Select(s => s.Content)), spanStart + symbols.First().Start);
}
else
{
return new LocationTagged<string>(string.Empty, spanStart);
}
}
public static LocationTagged<string> GetContent(this ISymbol symbol)
{
return new LocationTagged<string>(symbol.Content, symbol.Start);
}
}
}

9
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TagHelperBlockRewriter.cs
View File

@ -149,7 +149,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
ErrorSink errorSink) ErrorSink errorSink)
{ {
var afterEquals = false; var afterEquals = false;
var builder = new SpanBuilder var builder = new SpanBuilder(span.Start)
{ {
ChunkGenerator = span.ChunkGenerator, ChunkGenerator = span.ChunkGenerator,
EditHandler = span.EditHandler, EditHandler = span.EditHandler,
@ -190,7 +190,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
capturedAttributeValueStart = true; capturedAttributeValueStart = true;
attributeValueStartLocation = span.Start + symbol.Start; attributeValueStartLocation = symbol.Start;
} }
builder.Accept(symbol); builder.Accept(symbol);
@ -260,7 +260,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
attributeValueStartLocation = attributeValueStartLocation =
span.Start +
symbolStartLocation + symbolStartLocation +
new SourceLocation(absoluteIndex: 1, lineIndex: 0, characterIndex: 1); new SourceLocation(absoluteIndex: 1, lineIndex: 0, characterIndex: 1);
@ -280,7 +279,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
// After all symbols have been added we need to set the builders start position so we do not indirectly // After all symbols have been added we need to set the builders start position so we do not indirectly
// modify each symbol's Start location. // modify the span's start location.
builder.Start = attributeValueStartLocation; builder.Start = attributeValueStartLocation;
if (name == null) if (name == null)
@ -613,7 +612,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
.OfType<HtmlSymbol>() .OfType<HtmlSymbol>()
.First(sym => sym.Type != HtmlSymbolType.WhiteSpace && sym.Type != HtmlSymbolType.NewLine); .First(sym => sym.Type != HtmlSymbolType.WhiteSpace && sym.Type != HtmlSymbolType.NewLine);
return nodeStart + firstNonWhitespaceSymbol.Start; return firstNonWhitespaceSymbol.Start;
} }
private static Span CreateMarkupAttribute(SpanBuilder builder, bool isBoundNonStringAttribute) private static Span CreateMarkupAttribute(SpanBuilder builder, bool isBoundNonStringAttribute)

33
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/Tokenizer.cs
View File

@ -60,14 +60,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
} }
protected SourceLocation CurrentLocation public SourceLocation CurrentLocation => Source.Location;
{
get { return Source.Location; }
}
protected SourceLocation CurrentStart { get; private set; } public SourceLocation CurrentStart { get; private set; }
protected abstract TSymbol CreateSymbol(SourceLocation start, string content, TSymbolType type, IReadOnlyList<RazorError> errors); protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList<RazorError> errors);
protected abstract StateResult Dispatch(); protected abstract StateResult Dispatch();
@ -92,6 +89,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Post-Condition: Buffer should be empty at the end of Next() // Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0); Debug.Assert(Buffer.Length == 0);
// Post-Condition: Token should be non-zero length unless we're at EOF.
Debug.Assert(EndOfFile || !CurrentStart.Equals(CurrentLocation));
return symbol; return symbol;
} }
@ -193,19 +193,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected void StartSymbol() protected void StartSymbol()
{ {
Buffer.Clear(); Debug.Assert(Buffer.Length == 0);
Debug.Assert(CurrentErrors.Count == 0);
CurrentStart = CurrentLocation; CurrentStart = CurrentLocation;
CurrentErrors.Clear();
} }
protected TSymbol EndSymbol(TSymbolType type) protected TSymbol EndSymbol(TSymbolType type)
{ {
return EndSymbol(CurrentStart, type); TSymbol symbol = null;
}
protected TSymbol EndSymbol(SourceLocation start, TSymbolType type)
{
TSymbol sym = null;
if (HaveContent) if (HaveContent)
{ {
// Perf: Don't allocate a new errors array unless necessary. // Perf: Don't allocate a new errors array unless necessary.
@ -215,10 +211,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
errors[i] = CurrentErrors[i]; errors[i] = CurrentErrors[i];
} }
sym = CreateSymbol(start, GetSymbolContent(type), type, errors); symbol = CreateSymbol(GetSymbolContent(type), type, errors);
Buffer.Clear();
CurrentErrors.Clear();
} }
StartSymbol();
return sym; return symbol;
} }
protected virtual string GetSymbolContent(TSymbolType type) protected virtual string GetSymbolContent(TSymbolType type)

37
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerBackedParser.cs
View File

@ -18,10 +18,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TSymbol, TSymbolType> language, ParserContext context) protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TSymbol, TSymbolType> language, ParserContext context)
: base(context) : base(context)
{ {
Span = new SpanBuilder();
Language = language; Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source); var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView<TTokenizer, TSymbol, TSymbolType>(languageTokenizer); _tokenizer = new TokenizerView<TTokenizer, TSymbol, TSymbolType>(languageTokenizer);
Span = new SpanBuilder(CurrentLocation);
} }
protected SpanBuilder Span { get; private set; } protected SpanBuilder Span { get; private set; }
@ -35,10 +36,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected TSymbol PreviousSymbol { get; private set; } protected TSymbol PreviousSymbol { get; private set; }
protected SourceLocation CurrentLocation protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
{
get { return (EndOfFile || CurrentSymbol == null) ? Context.Source.Location : CurrentSymbol.Start; } protected SourceLocation CurrentStart => _tokenizer.Tokenizer.CurrentStart;
}
protected bool EndOfFile protected bool EndOfFile
{ {
@ -163,7 +163,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
var left = CurrentSymbol.Type; var left = CurrentSymbol.Type;
var right = Language.FlipBracket(left); var right = Language.FlipBracket(left);
var start = CurrentLocation; var start = CurrentStart;
AcceptAndMoveNext(); AcceptAndMoveNext();
if (EndOfFile && ((mode & BalancingModes.NoErrorOnFailure) != BalancingModes.NoErrorOnFailure)) if (EndOfFile && ((mode & BalancingModes.NoErrorOnFailure) != BalancingModes.NoErrorOnFailure))
{ {
@ -180,7 +180,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start) protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start)
{ {
var startPosition = CurrentLocation.AbsoluteIndex; var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1; var nesting = 1;
if (!EndOfFile) if (!EndOfFile)
{ {
@ -196,7 +196,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
HandleEmbeddedTransition(); HandleEmbeddedTransition();
// Reset backtracking since we've already outputted some spans. // Reset backtracking since we've already outputted some spans.
startPosition = CurrentLocation.AbsoluteIndex; startPosition = CurrentStart.AbsoluteIndex;
} }
if (At(left)) if (At(left))
{ {
@ -328,15 +328,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
protected internal void AddMarkerSymbolIfNecessary() protected internal void AddMarkerSymbolIfNecessary()
{
AddMarkerSymbolIfNecessary(CurrentLocation);
}
protected internal void AddMarkerSymbolIfNecessary(SourceLocation location)
{ {
if (Span.Symbols.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharacters.Any) if (Span.Symbols.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharacters.Any)
{ {
Accept(Language.CreateMarkerSymbol(location)); Accept(Language.CreateMarkerSymbol());
} }
} }
@ -362,9 +357,18 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
if (Span.Symbols.Count > 0) if (Span.Symbols.Count > 0)
{ {
var nextStart = Span.End;
var builtSpan = Span.Build(); var builtSpan = Span.Build();
Context.Builder.Add(builtSpan); Context.Builder.Add(builtSpan);
Initialize(Span); Initialize(Span);
// Ensure spans are contiguous.
//
// Note: Using Span.End here to avoid CurrentLocation. CurrentLocation will
// vary depending on what tokens have been read. We often read a token and *then*
// make a decision about whether to include it in the current span.
Span.Start = nextStart;
} }
} }
@ -464,7 +468,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
errorLength = Math.Max(CurrentSymbol.Content.Length, 1); errorLength = Math.Max(CurrentSymbol.Content.Length, 1);
} }
Context.ErrorSink.OnError(CurrentLocation, errorBase(error), errorLength); Context.ErrorSink.OnError(CurrentStart, errorBase(error), errorLength);
} }
return found; return found;
} }
@ -475,6 +479,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
return NextToken(); return NextToken();
} }
return true; return true;
} }
@ -612,7 +617,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
using (Context.Builder.StartBlock(BlockType.Comment)) using (Context.Builder.StartBlock(BlockType.Comment))
{ {
Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator(); Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator();
var start = CurrentLocation; var start = CurrentStart;
Expected(KnownSymbolType.CommentStart); Expected(KnownSymbolType.CommentStart);
Output(SpanKind.Transition, AcceptedCharacters.None); Output(SpanKind.Transition, AcceptedCharacters.None);

9
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerView.cs
View File

@ -34,15 +34,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void PutBack(TSymbol symbol) public void PutBack(TSymbol symbol)
{ {
Debug.Assert(Source.Position == symbol.Start.AbsoluteIndex + symbol.Content.Length);
if (Source.Position != symbol.Start.AbsoluteIndex + symbol.Content.Length)
{
// We've already passed this symbol
throw new InvalidOperationException(
LegacyResources.FormatTokenizerView_CannotPutBack(
symbol.Start.AbsoluteIndex + symbol.Content.Length,
Source.Position));
}
Source.Position -= symbol.Content.Length; Source.Position -= symbol.Content.Length;
Current = null; Current = null;
EndOfFile = Source.Position >= Source.Length; EndOfFile = Source.Position >= Source.Length;

4
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/BlockTest.cs
View File

@ -13,7 +13,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
// Arrange // Arrange
var builder = new BlockBuilder() { Type = BlockType.Comment }; var builder = new BlockBuilder() { Type = BlockType.Comment };
var span = new SpanBuilder() { Kind = SpanKind.Code }.Build(); var span = new SpanBuilder(SourceLocation.Undefined) { Kind = SpanKind.Code }.Build();
builder.Children.Add(span); builder.Children.Add(span);
// Act // Act
@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void ConstructorTransfersChildrenFromBlockBuilder() public void ConstructorTransfersChildrenFromBlockBuilder()
{ {
// Arrange // Arrange
var expected = new SpanBuilder() { Kind = SpanKind.Code }.Build(); var expected = new SpanBuilder(SourceLocation.Undefined) { Kind = SpanKind.Code }.Build();
var builder = new BlockBuilder() var builder = new BlockBuilder()
{ {
Type = BlockType.Functions Type = BlockType.Functions

2
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpAutoCompleteTest.cs
View File

@ -121,7 +121,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.Markup("<p>").Accepts(AcceptedCharacters.None)), Factory.Markup("<p>").Accepts(AcceptedCharacters.None)),
new MarkupTagBlock( new MarkupTagBlock(
Factory.Markup("</p>").Accepts(AcceptedCharacters.None))), Factory.Markup("</p>").Accepts(AcceptedCharacters.None))),
Factory.Span(SpanKind.Code, new CSharpSymbol(Factory.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown)) Factory.Span(SpanKind.Code, new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown))
.With(new StatementChunkGenerator()) .With(new StatementChunkGenerator())
), ),
new RazorError( new RazorError(

26
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpRazorCommentsTest.cs
View File

@ -19,10 +19,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(
Factory.LocationTracker.CurrentLocation, SpanKind.Comment,
string.Empty, new HtmlSymbol(
HtmlSymbolType.Unknown)) string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any))), .Accepts(AcceptedCharacters.Any))),
new RazorError( new RazorError(
LegacyResources.ParseError_RazorComment_Not_Terminated, LegacyResources.ParseError_RazorComment_Not_Terminated,
@ -42,7 +43,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -70,7 +70,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar) Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new CSharpSymbol( Factory.Span(SpanKind.Comment, new CSharpSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
CSharpSymbolType.Unknown)) CSharpSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -102,7 +101,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar) Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new CSharpSymbol( Factory.Span(SpanKind.Comment, new CSharpSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
CSharpSymbolType.Unknown)) CSharpSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any)))), .Accepts(AcceptedCharacters.Any)))),
@ -142,7 +140,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -183,9 +180,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar) Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new CSharpSymbol(Factory.LocationTracker.CurrentLocation, Factory.Span(SpanKind.Comment, new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown))
string.Empty,
CSharpSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any)))), .Accepts(AcceptedCharacters.Any)))),
new RazorError( new RazorError(
LegacyResources.ParseError_RazorComment_Not_Terminated, LegacyResources.ParseError_RazorComment_Not_Terminated,
@ -215,7 +210,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -248,7 +242,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -263,7 +256,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -294,7 +286,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -310,7 +301,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty, string.Empty,
HtmlSymbolType.Unknown)) HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -341,7 +331,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
" hello ", " hello ",
HtmlSymbolType.RazorComment)) HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -356,7 +345,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
" world ", " world ",
HtmlSymbolType.RazorComment)) HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -390,7 +378,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
" content ", " content ",
HtmlSymbolType.RazorComment)) HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),
@ -405,7 +392,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None), .Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol( Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
Environment.NewLine + "content" + Environment.NewLine, Environment.NewLine + "content" + Environment.NewLine,
HtmlSymbolType.RazorComment)) HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any), .Accepts(AcceptedCharacters.Any),

51
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerCommentTest.cs
View File

@ -12,80 +12,83 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment() public void Next_Ignores_Star_At_EOF_In_RazorComment()
{ {
TestTokenizer("@* Foo * Bar * Baz *", TestTokenizer(
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition), "@* Foo * Bar * Baz *",
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar), new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz *", CSharpSymbolType.RazorComment)); new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(" Foo * Bar * Baz *", CSharpSymbolType.RazorComment));
} }
[Fact] [Fact]
public void Next_Ignores_Star_Without_Trailing_At() public void Next_Ignores_Star_Without_Trailing_At()
{ {
TestTokenizer("@* Foo * Bar * Baz *@", TestTokenizer(
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition), "@* Foo * Bar * Baz *@",
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar), new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz ", CSharpSymbolType.RazorComment), new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(19, 0, 19, "*", CSharpSymbolType.RazorCommentStar), new CSharpSymbol(" Foo * Bar * Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol(20, 0, 20, "@", CSharpSymbolType.RazorCommentTransition)); new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition));
} }
[Fact] [Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment() public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{ {
TestTokenizer("@* Foo Bar Baz *@", TestTokenizer(
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition), "@* Foo Bar Baz *@",
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar), new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(2, 0, 2, " Foo Bar Baz ", CSharpSymbolType.RazorComment), new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(15, 0, 15, "*", CSharpSymbolType.RazorCommentStar), new CSharpSymbol(" Foo Bar Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol(16, 0, 16, "@", CSharpSymbolType.RazorCommentTransition)); new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition));
} }
[Fact] [Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment() public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{ {
TestTokenizer("// Foo Bar Baz", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment)); TestTokenizer("// Foo Bar Baz", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment));
} }
[Fact] [Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline() public void Single_Line_Comment_Is_Terminated_By_Newline()
{ {
TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect() public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{ {
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol(0, 0, 0, "// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol("// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment() public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{ {
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment)); TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment));
} }
[Fact] [Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence() public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{ {
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF() public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{ {
TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol("/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence() public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{ {
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol(0, 0, 0, "/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol("/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining);
} }
[Fact] [Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence() public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{ {
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining); TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol("/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining);
} }
} }
} }

26
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerIdentifierTest.cs
View File

@ -10,73 +10,73 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Simple_Identifier_Is_Recognized() public void Simple_Identifier_Is_Recognized()
{ {
TestTokenizer("foo", new CSharpSymbol(0, 0, 0, "foo", CSharpSymbolType.Identifier)); TestTokenizer("foo", new CSharpSymbol("foo", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized() public void Identifier_Starting_With_Underscore_Is_Recognized()
{ {
TestTokenizer("_foo", new CSharpSymbol(0, 0, 0, "_foo", CSharpSymbolType.Identifier)); TestTokenizer("_foo", new CSharpSymbol("_foo", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Digits() public void Identifier_Can_Contain_Digits()
{ {
TestTokenizer("foo4", new CSharpSymbol(0, 0, 0, "foo4", CSharpSymbolType.Identifier)); TestTokenizer("foo4", new CSharpSymbol("foo4", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Titlecase_Letter() public void Identifier_Can_Start_With_Titlecase_Letter()
{ {
TestTokenizer("ῼfoo", new CSharpSymbol(0, 0, 0, "ῼfoo", CSharpSymbolType.Identifier)); TestTokenizer("ῼfoo", new CSharpSymbol("ῼfoo", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Letter_Modifier() public void Identifier_Can_Start_With_Letter_Modifier()
{ {
TestTokenizer("ᵊfoo", new CSharpSymbol(0, 0, 0, "ᵊfoo", CSharpSymbolType.Identifier)); TestTokenizer("ᵊfoo", new CSharpSymbol("ᵊfoo", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Other_Letter() public void Identifier_Can_Start_With_Other_Letter()
{ {
TestTokenizer("ƻfoo", new CSharpSymbol(0, 0, 0, "ƻfoo", CSharpSymbolType.Identifier)); TestTokenizer("ƻfoo", new CSharpSymbol("ƻfoo", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Start_With_Number_Letter() public void Identifier_Can_Start_With_Number_Letter()
{ {
TestTokenizer("Ⅽool", new CSharpSymbol(0, 0, 0, "Ⅽool", CSharpSymbolType.Identifier)); TestTokenizer("Ⅽool", new CSharpSymbol("Ⅽool", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark() public void Identifier_Can_Contain_Non_Spacing_Mark()
{ {
TestTokenizer("foo\u0300", new CSharpSymbol(0, 0, 0, "foo\u0300", CSharpSymbolType.Identifier)); TestTokenizer("foo\u0300", new CSharpSymbol("foo\u0300", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark() public void Identifier_Can_Contain_Spacing_Combining_Mark()
{ {
TestTokenizer("fooः", new CSharpSymbol(0, 0, 0, "fooः", CSharpSymbolType.Identifier)); TestTokenizer("fooः", new CSharpSymbol("fooः", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Non_English_Digit() public void Identifier_Can_Contain_Non_English_Digit()
{ {
TestTokenizer("foo١", new CSharpSymbol(0, 0, 0, "foo١", CSharpSymbolType.Identifier)); TestTokenizer("foo١", new CSharpSymbol("foo١", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Connector_Punctuation() public void Identifier_Can_Contain_Connector_Punctuation()
{ {
TestTokenizer("foo‿bar", new CSharpSymbol(0, 0, 0, "foo‿bar", CSharpSymbolType.Identifier)); TestTokenizer("foo‿bar", new CSharpSymbol("foo‿bar", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
public void Identifier_Can_Contain_Format_Character() public void Identifier_Can_Contain_Format_Character()
{ {
TestTokenizer("foo؃bar", new CSharpSymbol(0, 0, 0, "foo؃bar", CSharpSymbolType.Identifier)); TestTokenizer("foo؃bar", new CSharpSymbol("foo؃bar", CSharpSymbolType.Identifier));
} }
[Fact] [Fact]
@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType) private void TestKeyword(string keyword, CSharpKeyword keywordType)
{ {
TestTokenizer(keyword, new CSharpSymbol(0, 0, 0, keyword, CSharpSymbolType.Keyword) { Keyword = keywordType }); TestTokenizer(keyword, new CSharpSymbol(keyword, CSharpSymbolType.Keyword) { Keyword = keywordType });
} }
} }
} }

36
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerLiteralTest.cs
View File

@ -41,7 +41,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix() public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{ {
TestTokenizer("42a", new CSharpSymbol(0, 0, 0, "42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); TestTokenizer("42a", new CSharpSymbol("42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
@ -75,25 +75,25 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix() public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{ {
TestTokenizer("0xDEADBEEFz", new CSharpSymbol(0, 0, 0, "0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); TestTokenizer("0xDEADBEEFz", new CSharpSymbol("0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal() public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{ {
TestTokenizer("3.a", new CSharpSymbol(0, 0, 0, "3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); TestTokenizer("3.a", new CSharpSymbol("3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Simple_Real_Literal_Is_Recognized() public void Simple_Real_Literal_Is_Recognized()
{ {
TestTokenizer("3.14159", new CSharpSymbol(0, 0, 0, "3.14159", CSharpSymbolType.RealLiteral)); TestTokenizer("3.14159", new CSharpSymbol("3.14159", CSharpSymbolType.RealLiteral));
} }
[Fact] [Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized() public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{ {
TestTokenizer(".14159", new CSharpSymbol(0, 0, 0, ".14159", CSharpSymbolType.RealLiteral)); TestTokenizer(".14159", new CSharpSymbol(".14159", CSharpSymbolType.RealLiteral));
} }
[Fact] [Fact]
@ -173,37 +173,37 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated() public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{ {
TestTokenizer("'foo\n", new CSharpSymbol(0, 0, 0, "'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\n", new CSharpSymbol("'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("'foo\\\n", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\n", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer("'foo\\\nflarg", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\nflarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void Character_Literal_Allows_Escaped_Escape() public void Character_Literal_Allows_Escaped_Escape()
{ {
TestTokenizer("'foo\\\\'blah", new CSharpSymbol(0, 0, 0, "'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); TestTokenizer("'foo\\\\'blah", new CSharpSymbol("'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
@ -227,37 +227,37 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated() public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{ {
TestTokenizer("\"foo\n", new CSharpSymbol(0, 0, 0, "\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\n", new CSharpSymbol("\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("\"foo\\\n", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\n", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer("\"foo\\\nflarg", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\nflarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{ {
TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{ {
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
public void String_Literal_Allows_Escaped_Escape() public void String_Literal_Allows_Escaped_Escape()
{ {
TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol(0, 0, 0, "\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol("\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]
@ -275,7 +275,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote() public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{ {
TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol(0, 0, 0, "@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol("@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
} }
[Fact] [Fact]

8
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerOperatorsTest.cs
View File

@ -125,8 +125,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void LeftShift_Is_Not_Specially_Recognized() public void LeftShift_Is_Not_Specially_Recognized()
{ {
TestTokenizer("<<", TestTokenizer("<<",
new CSharpSymbol(new SourceLocation(0, 0, 0), "<", CSharpSymbolType.LessThan), new CSharpSymbol("<", CSharpSymbolType.LessThan),
new CSharpSymbol(new SourceLocation(1, 0, 1), "<", CSharpSymbolType.LessThan)); new CSharpSymbol("<", CSharpSymbolType.LessThan));
} }
[Fact] [Fact]
@ -283,8 +283,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void RightShift_Is_Not_Specially_Recognized() public void RightShift_Is_Not_Specially_Recognized()
{ {
TestTokenizer(">>", TestTokenizer(">>",
new CSharpSymbol(0, 0, 0, ">", CSharpSymbolType.GreaterThan), new CSharpSymbol(">", CSharpSymbolType.GreaterThan),
new CSharpSymbol(1, 0, 1, ">", CSharpSymbolType.GreaterThan)); new CSharpSymbol(">", CSharpSymbolType.GreaterThan));
} }
[Fact] [Fact]

70
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTest.cs
View File

@ -18,67 +18,74 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Next_Returns_Newline_Token_For_Single_CR() public void Next_Returns_Newline_Token_For_Single_CR()
{ {
TestTokenizer("\r\ra", TestTokenizer(
new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine), "\r\ra",
new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine), new CSharpSymbol("\r", CSharpSymbolType.NewLine),
IgnoreRemaining); new CSharpSymbol("\r", CSharpSymbolType.NewLine),
IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Newline_Token_For_Single_LF() public void Next_Returns_Newline_Token_For_Single_LF()
{ {
TestTokenizer("\n\na", TestTokenizer(
new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine), "\n\na",
new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine), new CSharpSymbol("\n", CSharpSymbolType.NewLine),
IgnoreRemaining); new CSharpSymbol("\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Newline_Token_For_Single_NEL() public void Next_Returns_Newline_Token_For_Single_NEL()
{ {
// NEL: Unicode "Next Line" U+0085 // NEL: Unicode "Next Line" U+0085
TestTokenizer("\u0085\u0085a", TestTokenizer(
new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine), "\u0085\u0085a",
new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine), new CSharpSymbol("\u0085", CSharpSymbolType.NewLine),
IgnoreRemaining); new CSharpSymbol("\u0085", CSharpSymbolType.NewLine),
IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Newline_Token_For_Single_Line_Separator() public void Next_Returns_Newline_Token_For_Single_Line_Separator()
{ {
// Unicode "Line Separator" U+2028 // Unicode "Line Separator" U+2028
TestTokenizer("\u2028\u2028a", TestTokenizer(
new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine), "\u2028\u2028a",
new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine), new CSharpSymbol("\u2028", CSharpSymbolType.NewLine),
IgnoreRemaining); new CSharpSymbol("\u2028", CSharpSymbolType.NewLine),
IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator() public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
{ {
// Unicode "Paragraph Separator" U+2029 // Unicode "Paragraph Separator" U+2029
TestTokenizer("\u2029\u2029a", TestTokenizer(
new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine), "\u2029\u2029a",
new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine), new CSharpSymbol("\u2029", CSharpSymbolType.NewLine),
IgnoreRemaining); new CSharpSymbol("\u2029", CSharpSymbolType.NewLine),
IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Single_Newline_Token_For_CRLF() public void Next_Returns_Single_Newline_Token_For_CRLF()
{ {
TestTokenizer("\r\n\r\na", TestTokenizer(
new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine), "\r\n\r\na",
new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine), new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
IgnoreRemaining); new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
} }
[Fact] [Fact]
public void Next_Returns_Token_For_Whitespace_Characters() public void Next_Returns_Token_For_Whitespace_Characters()
{ {
TestTokenizer(" \f\t\u000B \n ", TestTokenizer(
new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace), " \f\t\u000B \n ",
new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine), new CSharpSymbol(" \f\t\u000B ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace)); new CSharpSymbol("\n", CSharpSymbolType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace));
} }
[Fact] [Fact]
@ -90,9 +97,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Transition_Is_Recognized_As_SingleCharacter() public void Transition_Is_Recognized_As_SingleCharacter()
{ {
TestTokenizer("@(", TestTokenizer(
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition), "@(",
new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis)); new CSharpSymbol("@", CSharpSymbolType.Transition),
new CSharpSymbol("(", CSharpSymbolType.LeftParenthesis));
} }
} }
} }

4
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTestBase.cs
View File

@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
public abstract class CSharpTokenizerTestBase : TokenizerTestBase public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{ {
private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(0, 0, 0, string.Empty, CSharpSymbolType.Unknown); private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown);
internal override object IgnoreRemaining internal override object IgnoreRemaining
{ {
@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
internal void TestSingleToken(string text, CSharpSymbolType expectedSymbolType) internal void TestSingleToken(string text, CSharpSymbolType expectedSymbolType)
{ {
TestTokenizer(text, new CSharpSymbol(0, 0, 0, text, expectedSymbolType)); TestTokenizer(text, new CSharpSymbol(text, expectedSymbolType));
} }
internal void TestTokenizer(string input, params CSharpSymbol[] expectedSymbols) internal void TestTokenizer(string input, params CSharpSymbol[] expectedSymbols)

45
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTest.cs
View File

@ -17,23 +17,23 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void Text_Is_Recognized() public void Text_Is_Recognized()
{ {
TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384", TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384",
new HtmlSymbol(0, 0, 0, "foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text)); new HtmlSymbol("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text));
} }
[Fact] [Fact]
public void Whitespace_Is_Recognized() public void Whitespace_Is_Recognized()
{ {
TestTokenizer(" \t\f ", TestTokenizer(" \t\f ",
new HtmlSymbol(0, 0, 0, " \t\f ", HtmlSymbolType.WhiteSpace)); new HtmlSymbol(" \t\f ", HtmlSymbolType.WhiteSpace));
} }
[Fact] [Fact]
public void Newline_Is_Recognized() public void Newline_Is_Recognized()
{ {
TestTokenizer("\n\r\r\n", TestTokenizer("\n\r\r\n",
new HtmlSymbol(0, 0, 0, "\n", HtmlSymbolType.NewLine), new HtmlSymbol("\n", HtmlSymbolType.NewLine),
new HtmlSymbol(1, 1, 0, "\r", HtmlSymbolType.NewLine), new HtmlSymbol("\r", HtmlSymbolType.NewLine),
new HtmlSymbol(2, 2, 0, "\r\n", HtmlSymbolType.NewLine)); new HtmlSymbol("\r\n", HtmlSymbolType.NewLine));
} }
[Fact] [Fact]
@ -129,32 +129,35 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact] [Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment() public void Next_Ignores_Star_At_EOF_In_RazorComment()
{ {
TestTokenizer("@* Foo * Bar * Baz *", TestTokenizer(
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition), "@* Foo * Bar * Baz *",
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar), new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz *", HtmlSymbolType.RazorComment)); new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(" Foo * Bar * Baz *", HtmlSymbolType.RazorComment));
} }
[Fact] [Fact]
public void Next_Ignores_Star_Without_Trailing_At() public void Next_Ignores_Star_Without_Trailing_At()
{ {
TestTokenizer("@* Foo * Bar * Baz *@", TestTokenizer(
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition), "@* Foo * Bar * Baz *@",
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar), new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz ", HtmlSymbolType.RazorComment), new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(19, 0, 19, "*", HtmlSymbolType.RazorCommentStar), new HtmlSymbol(" Foo * Bar * Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol(20, 0, 20, "@", HtmlSymbolType.RazorCommentTransition)); new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition));
} }
[Fact] [Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment() public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{ {
TestTokenizer("@* Foo Bar Baz *@", TestTokenizer(
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition), "@* Foo Bar Baz *@",
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar), new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(2, 0, 2, " Foo Bar Baz ", HtmlSymbolType.RazorComment), new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(15, 0, 15, "*", HtmlSymbolType.RazorCommentStar), new HtmlSymbol(" Foo Bar Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol(16, 0, 16, "@", HtmlSymbolType.RazorCommentTransition)); new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition));
} }
} }
} }

4
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTestBase.cs
View File

@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
public abstract class HtmlTokenizerTestBase : TokenizerTestBase public abstract class HtmlTokenizerTestBase : TokenizerTestBase
{ {
private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(0, 0, 0, string.Empty, HtmlSymbolType.Unknown); private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
internal override object IgnoreRemaining internal override object IgnoreRemaining
{ {
@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
internal void TestSingleToken(string text, HtmlSymbolType expectedSymbolType) internal void TestSingleToken(string text, HtmlSymbolType expectedSymbolType)
{ {
TestTokenizer(text, new HtmlSymbol(0, 0, 0, text, expectedSymbolType)); TestTokenizer(text, new HtmlSymbol(text, expectedSymbolType));
} }
internal void TestTokenizer(string input, params HtmlSymbol[] expectedSymbols) internal void TestTokenizer(string input, params HtmlSymbol[] expectedSymbols)

9
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RawTextSymbol.cs
View File

@ -1,21 +1,24 @@
// Copyright (c) .NET Foundation. All rights reserved. // Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Globalization; using System.Globalization;
namespace Microsoft.AspNetCore.Razor.Evolution.Legacy namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{ {
internal class RawTextSymbol : ISymbol internal class RawTextSymbol : ISymbol
{ {
public SourceLocation Start { get; private set; }
public string Content { get; }
public RawTextSymbol(SourceLocation start, string content) public RawTextSymbol(SourceLocation start, string content)
{ {
Start = start; Start = start;
Content = content; Content = content;
} }
public SourceLocation Start { get; private set; }
public string Content { get; }
public Span Parent { get; set; }
public int Offset { get; set; }
public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var other = obj as RawTextSymbol; var other = obj as RawTextSymbol;

47
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TestSpanBuilder.cs
View File

@ -14,7 +14,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return new UnclassifiedCodeSpanConstructor( return new UnclassifiedCodeSpanConstructor(
self.Span( self.Span(
SpanKind.Code, SpanKind.Code,
new CSharpSymbol(self.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown))); new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown)));
} }
public static SpanConstructor EmptyHtml(this SpanFactory self) public static SpanConstructor EmptyHtml(this SpanFactory self)
@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return self return self
.Span( .Span(
SpanKind.Markup, SpanKind.Markup,
new HtmlSymbol(self.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown))
.With(new MarkupChunkGenerator()); .With(new MarkupChunkGenerator());
} }
@ -178,34 +178,41 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public SpanConstructor Span(SpanKind kind, string content, CSharpSymbolType type) public SpanConstructor Span(SpanKind kind, string content, CSharpSymbolType type)
{ {
return CreateSymbolSpan(kind, content, st => new CSharpSymbol(st, content, type)); return CreateSymbolSpan(kind, content, () => new CSharpSymbol(content, type));
} }
public SpanConstructor Span(SpanKind kind, string content, HtmlSymbolType type) public SpanConstructor Span(SpanKind kind, string content, HtmlSymbolType type)
{ {
return CreateSymbolSpan(kind, content, st => new HtmlSymbol(st, content, type)); return CreateSymbolSpan(kind, content, () => new HtmlSymbol(content, type));
} }
public SpanConstructor Span(SpanKind kind, string content, bool markup) public SpanConstructor Span(SpanKind kind, string content, bool markup)
{ {
return new SpanConstructor(kind, Tokenize(new[] { content }, markup)); return new SpanConstructor(kind, LocationTracker.CurrentLocation, Tokenize(new[] { content }, markup));
} }
public SpanConstructor Span(SpanKind kind, string[] content, bool markup) public SpanConstructor Span(SpanKind kind, string[] content, bool markup)
{ {
return new SpanConstructor(kind, Tokenize(content, markup)); return new SpanConstructor(kind, LocationTracker.CurrentLocation, Tokenize(content, markup));
} }
public SpanConstructor Span(SpanKind kind, params ISymbol[] symbols) public SpanConstructor Span(SpanKind kind, params ISymbol[] symbols)
{ {
return new SpanConstructor(kind, symbols); var start = LocationTracker.CurrentLocation;
foreach (var symbol in symbols)
{
LocationTracker.UpdateLocation(symbol.Content);
}
return new SpanConstructor(kind, start, symbols);
} }
private SpanConstructor CreateSymbolSpan(SpanKind kind, string content, Func<SourceLocation, ISymbol> ctor) private SpanConstructor CreateSymbolSpan(SpanKind kind, string content, Func<ISymbol> ctor)
{ {
var start = LocationTracker.CurrentLocation; var start = LocationTracker.CurrentLocation;
LocationTracker.UpdateLocation(content); LocationTracker.UpdateLocation(content);
return new SpanConstructor(kind, new[] { ctor(start) });
return new SpanConstructor(kind, start, new[] { ctor() });
} }
public void Reset() public void Reset()
@ -220,15 +227,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private IEnumerable<ISymbol> Tokenize(string content, bool markup) private IEnumerable<ISymbol> Tokenize(string content, bool markup)
{ {
var tok = MakeTokenizer(markup, new SeekableTextReader(content)); var tokenizer = MakeTokenizer(markup, new SeekableTextReader(content));
ISymbol sym; ISymbol symbol;
ISymbol last = null; ISymbol last = null;
while ((sym = tok.NextSymbol()) != null)
while ((symbol = tokenizer.NextSymbol()) != null)
{ {
OffsetStart(sym, LocationTracker.CurrentLocation); last = symbol;
last = sym; yield return symbol;
yield return sym;
} }
LocationTracker.UpdateLocation(content); LocationTracker.UpdateLocation(content);
} }
@ -243,11 +251,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return CodeTokenizerFactory(seekableTextReader); return CodeTokenizerFactory(seekableTextReader);
} }
} }
private void OffsetStart(ISymbol sym, SourceLocation sourceLocation)
{
sym.OffsetStart(sourceLocation);
}
} }
internal static class SpanConstructorExtensions internal static class SpanConstructorExtensions
@ -359,9 +362,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
yield return new RawTextSymbol(SourceLocation.Zero, str); yield return new RawTextSymbol(SourceLocation.Zero, str);
} }
public SpanConstructor(SpanKind kind, IEnumerable<ISymbol> symbols) public SpanConstructor(SpanKind kind, SourceLocation location, IEnumerable<ISymbol> symbols)
{ {
Builder = new SpanBuilder(); Builder = new SpanBuilder(location);
Builder.Kind = kind; Builder.Kind = kind;
Builder.EditHandler = SpanEditHandler.CreateDefault((content) => SpanConstructor.TestTokenizer(content)); Builder.EditHandler = SpanEditHandler.CreateDefault((content) => SpanConstructor.TestTokenizer(content));
foreach (ISymbol sym in symbols) foreach (ISymbol sym in symbols)

1
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs
View File

@ -104,7 +104,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
} }
protected override CSharpSymbol CreateSymbol( protected override CSharpSymbol CreateSymbol(
SourceLocation start,
string content, string content,
CSharpSymbolType type, CSharpSymbolType type,
IReadOnlyList<RazorError> errors) IReadOnlyList<RazorError> errors)