huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Get rid of locations on Tokens.

This commit is contained in:
Ryan Nowak 2016-12-20 16:04:29 -08:00
parent bd4300d8cc
commit f191cc62a2
33 changed files with 413 additions and 441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpCodeParser.cs
View File

@ -165,6 +165,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
throw new InvalidOperationException(LegacyResources.Parser_Context_Not_Set);
}
Span.Start = CurrentLocation;
// Unless changed, the block is a statement block
using (Context.Builder.StartBlock(BlockType.Statement))
{
@ -179,7 +181,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
var split = Language.SplitSymbol(CurrentSymbol, 1, CSharpSymbolType.Transition);
current = split.Item1;
Context.Source.Position = split.Item2.Start.AbsoluteIndex;
// Back up to the end of the transition
Context.Source.Position -= split.Item2.Content.Length;
NextToken();
}
else if (At(CSharpSymbolType.Transition))
@ -201,6 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// No "@" => Jump straight to AfterTransition
AfterTransition();
}
Output(SpanKind.Code);
}
}
@ -258,7 +263,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
StringComparison.Ordinal))
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatParseError_HelperDirectiveNotAvailable(
SyntaxConstants.CSharp.HelperKeyword),
CurrentSymbol.Content.Length);
@ -297,21 +302,21 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (At(CSharpSymbolType.WhiteSpace) || At(CSharpSymbolType.NewLine))
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.ParseError_Unexpected_WhiteSpace_At_Start_Of_CodeBlock_CS,
CurrentSymbol.Content.Length);
}
else if (EndOfFile)
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.ParseError_Unexpected_EndOfFile_At_Start_Of_CodeBlock,
length: 1 /* end of file */);
}
else
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatParseError_Unexpected_Character_At_Start_Of_CodeBlock_CS(
CurrentSymbol.Content),
CurrentSymbol.Content.Length);
@ -328,7 +333,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void VerbatimBlock()
{
Assert(CSharpSymbolType.LeftBrace);
var block = new Block(LegacyResources.BlockName_Code, CurrentLocation);
var block = new Block(LegacyResources.BlockName_Code, CurrentStart);
AcceptAndMoveNext();
// Set up the "{" span and output
@ -567,7 +572,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void ExplicitExpression()
{
var block = new Block(LegacyResources.BlockName_ExplicitExpression, CurrentLocation);
var block = new Block(LegacyResources.BlockName_ExplicitExpression, CurrentStart);
Assert(CSharpSymbolType.LeftParenthesis);
AcceptAndMoveNext();
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.None;
@ -595,7 +600,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// If necessary, put an empty-content marker symbol here
if (Span.Symbols.Count == 0)
{
Accept(new CSharpSymbol(CurrentLocation, string.Empty, CSharpSymbolType.Unknown));
Accept(new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown));
}
// Output the content span and then capture the ")"
@ -617,7 +622,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (Context.Builder.ActiveBlocks.Any(block => block.Type == BlockType.Template))
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.ParseError_InlineMarkup_Blocks_Cannot_Be_Nested,
length: 1 /* @ */);
}
@ -643,12 +648,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void NestedBlock()
{
Output(SpanKind.Code);
var wasNested = IsNested;
IsNested = true;
using (PushSpanConfig())
{
ParseBlock();
}
Span.Start = CurrentLocation;
Initialize(Span);
IsNested = wasNested;
NextToken();
@ -683,12 +691,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// as C#; it should be handled as a period because it's wrapped in markup.
var wasNested = IsNested;
IsNested = false;
using (PushSpanConfig())
{
parseAction(HtmlParser);
}
Span.Start = CurrentLocation;
Initialize(Span);
IsNested = wasNested;
NextToken();
}
@ -712,7 +725,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected virtual void ReservedDirective(bool topLevel)
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatParseError_ReservedWord(CurrentSymbol.Content),
CurrentSymbol.Content.Length);
AcceptAndMoveNext();
@ -780,7 +793,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void UsingKeyword(bool topLevel)
{
Assert(CSharpKeyword.Using);
var block = new Block(CurrentSymbol);
var block = new Block(CurrentSymbol, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
@ -817,6 +830,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Set block type to directive
Context.Builder.CurrentBlock.Type = BlockType.Directive;
var start = CurrentStart;
if (At(CSharpSymbolType.Identifier))
{
// non-static using
@ -849,8 +863,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
Span.EditHandler.AcceptedCharacters = AcceptedCharacters.AnyExceptNewline;
Span.ChunkGenerator = new AddImportChunkGenerator(
Span.GetContent(symbols => symbols.Skip(1)));
Span.ChunkGenerator = new AddImportChunkGenerator(new LocationTagged<string>(
string.Concat(Span.Symbols.Skip(1).Select(s => s.Content)),
start));
// Optional ";"
if (EnsureCurrent())
@ -990,7 +1005,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
return;
}
var block = new Block(CurrentSymbol);
var block = new Block(CurrentSymbol, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
@ -1016,7 +1031,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (!At(CSharpSymbolType.LeftBrace))
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatParseError_SingleLine_ControlFlowStatements_Not_Allowed(
Language.GetSample(CSharpSymbolType.LeftBrace),
CurrentSymbol.Content),
@ -1031,7 +1046,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void UnconditionalBlock()
{
Assert(CSharpSymbolType.Keyword);
var block = new Block(CurrentSymbol);
var block = new Block(CurrentSymbol, CurrentStart);
AcceptAndMoveNext();
AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
ExpectCodeBlock(block);
@ -1041,7 +1056,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
Assert(CSharpKeyword.Catch);
var block = new Block(CurrentSymbol);
var block = new Block(CurrentSymbol, CurrentStart);
// Accept "catch"
AcceptAndMoveNext();
@ -1075,7 +1090,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void ConditionalBlock(bool topLevel)
{
Assert(CSharpSymbolType.Keyword);
var block = new Block(CurrentSymbol);
var block = new Block(CurrentSymbol, CurrentStart);
ConditionalBlock(block);
if (topLevel)
{
@ -1125,8 +1140,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Accept whitespace but always keep the last whitespace node so we can put it back if necessary
var lastWhitespace = AcceptWhiteSpaceInLines();
Debug.Assert(lastWhitespace == null ||
(lastWhitespace.Start.AbsoluteIndex + lastWhitespace.Content.Length == CurrentLocation.AbsoluteIndex));
if (EndOfFile)
{
@ -1138,7 +1151,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
var type = CurrentSymbol.Type;
var loc = CurrentLocation;
var loc = CurrentStart;
// Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text
var isSingleLineMarkup = type == CSharpSymbolType.Transition &&
@ -1208,7 +1221,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
break;
case CSharpSymbolType.LeftBrace:
// Verbatim Block
block = block ?? new Block(LegacyResources.BlockName_Code, CurrentLocation);
block = block ?? new Block(LegacyResources.BlockName_Code, CurrentStart);
AcceptAndMoveNext();
CodeBlock(block);
break;
@ -1260,7 +1273,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (At(CSharpSymbolType.LeftBrace))
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.ParseError_Unexpected_Nested_CodeBlock,
length: 1 /* { */);
}
@ -1280,7 +1293,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
while (!EndOfFile)
{
var bookmark = CurrentLocation.AbsoluteIndex;
var bookmark = CurrentStart.AbsoluteIndex;
IEnumerable<CSharpSymbol> read = ReadWhile(sym => sym.Type != CSharpSymbolType.Semicolon &&
sym.Type != CSharpSymbolType.RazorCommentTransition &&
sym.Type != CSharpSymbolType.Transition &&
@ -1479,7 +1492,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (EndOfFile)
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, tokenDescriptor.Kind.ToString().ToLowerInvariant()),
length: 1);
return;
@ -1492,7 +1505,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (!NamespaceOrTypeName())
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatDirectiveExpectsTypeName(descriptor.Name),
CurrentSymbol.Content.Length);
@ -1509,7 +1522,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatDirectiveExpectsIdentifier(descriptor.Name),
CurrentSymbol.Content.Length);
return;
@ -1528,7 +1541,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, tokenDescriptor.Value),
CurrentSymbol.Content.Length);
return;
@ -1555,7 +1568,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else if (!EndOfFile)
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, Environment.NewLine),
CurrentSymbol.Content.Length);
}
@ -1573,12 +1586,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// as C#; it should be handled as a period because it's wrapped in markup.
var wasNested = IsNested;
IsNested = false;
using (PushSpanConfig())
{
HtmlParser.ParseRazorBlock(Tuple.Create("{", "}"), caseSensitive: true);
}
Span.Start = CurrentLocation;
Initialize(Span);
IsNested = wasNested;
NextToken();
});
break;
@ -1602,14 +1620,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (EndOfFile)
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, "{"),
length: 1 /* { */);
}
else if (!At(CSharpSymbolType.LeftBrace))
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, "{"),
CurrentSymbol.Content.Length);
}
@ -1617,7 +1635,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
var editHandler = new AutoCompleteEditHandler(Language.TokenizeString, autoCompleteAtEndOfSpan: true);
Span.EditHandler = editHandler;
var startingBraceLocation = CurrentLocation;
var startingBraceLocation = CurrentStart;
Accept(CurrentSymbol);
Span.ChunkGenerator = SpanChunkGenerator.Null;
Output(SpanKind.MetaCode, AcceptedCharacters.None);
@ -1678,7 +1696,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Set the block type
Context.Builder.CurrentBlock.Type = BlockType.Directive;
var keywordLength = Span.GetContent().Value.Length;
var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
// Accept whitespace
var remainingWhitespace = AcceptSingleWhiteSpaceCharacter();
@ -1714,7 +1732,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
// Pull out the type name
string baseType = Span.GetContent();
string baseType = string.Concat(Span.Symbols.Select(s => s.Content));
// Set up chunk generation
Span.ChunkGenerator = createChunkGenerator(baseType.Trim());
@ -1727,7 +1745,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void TagHelperDirective(string keyword, Func<string, ISpanChunkGenerator> chunkGeneratorFactory)
{
AssertDirective(keyword);
var keywordStartLocation = CurrentLocation;
var keywordStartLocation = CurrentStart;
// Accept the directive name
AcceptAndMoveNext();
@ -1735,7 +1753,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Set the block type
Context.Builder.CurrentBlock.Type = BlockType.Directive;
var keywordLength = Span.GetContent().Value.Length;
var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex;
var foundWhitespace = At(CSharpSymbolType.WhiteSpace);
AcceptWhile(CSharpSymbolType.WhiteSpace);
@ -1757,14 +1775,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else
{
// Need to grab the current location before we accept until the end of the line.
var startLocation = CurrentLocation;
var startLocation = CurrentStart;
// Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code
// etc.
AcceptUntil(CSharpSymbolType.NewLine);
// Pull out the value and remove whitespaces and optional quotes
var rawValue = Span.GetContent().Value.Trim();
var rawValue = string.Concat(Span.Symbols.Select(s => s.Content)).Trim();
var startsWithQuote = rawValue.StartsWith("\"", StringComparison.Ordinal);
var endsWithQuote = rawValue.EndsWith("\"", StringComparison.Ordinal);
@ -1807,8 +1825,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Start = start;
}
public Block(CSharpSymbol symbol)
: this(GetName(symbol), symbol.Start)
public Block(CSharpSymbol symbol, SourceLocation start)
: this(GetName(symbol), start)
{
}

8
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpLanguageCharacteristics.cs
View File

@ -74,9 +74,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return new CSharpTokenizer(source);
}
protected override CSharpSymbol CreateSymbol(SourceLocation location, string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors)
protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors)
{
return new CSharpSymbol(location, content, type, errors);
return new CSharpSymbol(content, type, errors);
}
public override string GetSample(CSharpSymbolType type)
@ -111,9 +111,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return sample;
}
public override CSharpSymbol CreateMarkerSymbol(SourceLocation location)
public override CSharpSymbol CreateMarkerSymbol()
{
return new CSharpSymbol(location, string.Empty, CSharpSymbolType.Unknown);
return new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown);
}
public override CSharpSymbolType GetKnownSymbolType(KnownSymbolType type)

33
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpSymbol.cs
View File

@ -8,17 +8,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal class CSharpSymbol : SymbolBase<CSharpSymbolType>
{
public CSharpSymbol(int absoluteIndex, int lineIndex, int characterIndex, string content, CSharpSymbolType type)
: this(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, RazorError.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpSymbol(SourceLocation start, string content, CSharpSymbolType type)
: this(start, content, type, RazorError.EmptyArray)
public CSharpSymbol(
string content,
CSharpSymbolType type)
: base(content, type, RazorError.EmptyArray)
{
if (content == null)
{
@ -27,26 +20,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
public CSharpSymbol(
int offset,
int line,
int column,
string content,
CSharpSymbolType type,
IReadOnlyList<RazorError> errors)
: base(new SourceLocation(offset, line, column), content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public CSharpSymbol(
SourceLocation start,
string content,
CSharpSymbolType type,
IReadOnlyList<RazorError> errors)
: base(start, content, type, errors)
: base(content, type, errors)
{
if (content == null)
{

13
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpTokenizer.cs
View File

@ -168,9 +168,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
}
protected override CSharpSymbol CreateSymbol(SourceLocation start, string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors)
protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList<RazorError> errors)
{
return new CSharpSymbol(start, content, type, errors);
return new CSharpSymbol(content, type, errors);
}
private StateResult Data()
@ -548,13 +548,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
type = CSharpSymbolType.Keyword;
}
symbol = new CSharpSymbol(CurrentStart, symbolContent, type)
symbol = new CSharpSymbol(symbolContent, type)
{
Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null,
};
Buffer.Clear();
CurrentErrors.Clear();
}
StartSymbol();
return Stay(symbol);
}

2
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ConditionalAttributeCollapser.cs
View File

@ -39,7 +39,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
// Create a new span containing this content
var span = new SpanBuilder();
var span = new SpanBuilder(block.Children[0].Start);
span.EditHandler = SpanEditHandler.CreateDefault(HtmlLanguageCharacteristics.Instance.TokenizeString);
Debug.Assert(block.Children.Count > 0);

8
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlLanguageCharacteristics.cs
View File

@ -95,9 +95,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
}
public override HtmlSymbol CreateMarkerSymbol(SourceLocation location)
public override HtmlSymbol CreateMarkerSymbol()
{
return new HtmlSymbol(location, string.Empty, HtmlSymbolType.Unknown);
return new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
}
public override HtmlSymbolType GetKnownSymbolType(KnownSymbolType type)
@ -125,9 +125,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
}
protected override HtmlSymbol CreateSymbol(SourceLocation location, string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors)
protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors)
{
return new HtmlSymbol(location, content, type, errors);
return new HtmlSymbol(content, type, errors);
}
}
}

78
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlMarkupParser.cs
View File

@ -219,10 +219,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
AddMarkerSymbolIfNecessary();
Output(SpanKind.Markup);
using (PushSpanConfig())
{
CodeParser.ParseBlock();
}
Span.Start = CurrentLocation;
Initialize(Span);
NextToken();
}
@ -271,6 +274,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
using (Context.Builder.StartBlock(BlockType.Markup))
{
Span.Start = CurrentLocation;
if (!NextToken())
{
return;
@ -305,7 +310,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else
{
Context.ErrorSink.OnError(
CurrentSymbol.Start,
CurrentStart,
LegacyResources.ParseError_MarkupBlock_Must_Start_With_Tag,
CurrentSymbol.Content.Length);
}
@ -393,10 +398,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else
{
_bufferedOpenAngle = null;
_lastTagStart = CurrentLocation;
_lastTagStart = CurrentStart;
Assert(HtmlSymbolType.OpenAngle);
_bufferedOpenAngle = CurrentSymbol;
var tagStart = CurrentLocation;
var tagStart = CurrentStart;
if (!NextToken())
{
Accept(_bufferedOpenAngle);
@ -467,7 +472,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (tags.Count == 0)
{
Context.ErrorSink.OnError(
CurrentLocation,
CurrentStart,
LegacyResources.ParseError_OuterTagMissingName,
length: 1 /* end of file */);
}
@ -620,7 +625,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Accept(_bufferedOpenAngle);
Accept(solidus);
var textLocation = CurrentLocation;
var textLocation = CurrentStart;
Assert(HtmlSymbolType.Text);
AcceptAndMoveNext();
@ -789,8 +794,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
IEnumerable<HtmlSymbol> whitespaceAfterAttributeName)
{
// First, determine if this is a 'data-' attribute (since those can't use conditional attributes)
var name = nameSymbols.GetContent(Span.Start);
var attributeCanBeConditional = !name.Value.StartsWith("data-", StringComparison.OrdinalIgnoreCase);
var name = string.Concat(nameSymbols.Select(s => s.Content));
var attributeCanBeConditional = !name.StartsWith("data-", StringComparison.OrdinalIgnoreCase);
// Accept the whitespace and name
Accept(whitespace);
@ -818,7 +823,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
// We now have the prefix: (i.e. ' foo="')
var prefix = Span.GetContent();
var prefix = new LocationTagged<string>(string.Concat(Span.Symbols.Select(s => s.Content)), Span.Start);
if (attributeCanBeConditional)
{
@ -837,10 +842,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
// Capture the suffix
var suffix = new LocationTagged<string>(string.Empty, CurrentLocation);
var suffix = new LocationTagged<string>(string.Empty, CurrentStart);
if (quote != HtmlSymbolType.Unknown && At(quote))
{
suffix = CurrentSymbol.GetContent();
suffix = new LocationTagged<string>(CurrentSymbol.Content, CurrentStart);
AcceptAndMoveNext();
}
@ -881,7 +886,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void AttributeValue(HtmlSymbolType quote)
{
var prefixStart = CurrentLocation;
var prefixStart = CurrentStart;
var prefix = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine);
if (At(HtmlSymbolType.Transition))
@ -895,8 +900,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Render a single "@" in place of "@@".
Span.ChunkGenerator = new LiteralAttributeChunkGenerator(
prefix.GetContent(prefixStart),
new LocationTagged<string>(CurrentSymbol.GetContent(), CurrentLocation));
new LocationTagged<string>(string.Concat(prefix.Select(s => s.Content)), prefixStart),
new LocationTagged<string>(CurrentSymbol.Content, CurrentStart));
AcceptAndMoveNext();
Output(SpanKind.Markup, AcceptedCharacters.None);
@ -908,7 +913,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
else
{
Accept(prefix);
var valueStart = CurrentLocation;
var valueStart = CurrentStart;
PutCurrentBack();
// Output the prefix but as a null-span. DynamicAttributeBlockChunkGenerator will render it
@ -918,7 +923,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
using (Context.Builder.StartBlock(BlockType.Markup))
{
Context.Builder.CurrentBlock.ChunkGenerator =
new DynamicAttributeBlockChunkGenerator(prefix.GetContent(prefixStart), valueStart);
new DynamicAttributeBlockChunkGenerator(
new LocationTagged<string>(string.Concat(prefix.Select(s => s.Content)), prefixStart),
valueStart);
OtherParserBlock();
}
@ -931,6 +938,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Literal value
// 'quote' should be "Unknown" if not quoted and symbols coming from the tokenizer should never have
// "Unknown" type.
var valueStart = CurrentStart;
var value = ReadWhile(sym =>
// These three conditions find separators which break the attribute value into portions
sym.Type != HtmlSymbolType.WhiteSpace &&
@ -941,8 +949,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
!IsEndOfAttributeValue(quote, sym));
Accept(value);
Span.ChunkGenerator = new LiteralAttributeChunkGenerator(
prefix.GetContent(prefixStart),
value.GetContent(prefixStart));
new LocationTagged<string>(string.Concat(prefix.Select(s => s.Content)), prefixStart),
new LocationTagged<string>(string.Concat(value.Select(s => s.Content)), valueStart));
}
Output(SpanKind.Markup);
}
@ -1037,11 +1045,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (potentialTagNameSymbol == null || potentialTagNameSymbol.Type != HtmlSymbolType.Text)
{
tagName = new HtmlSymbol(potentialTagNameSymbol.Start, string.Empty, HtmlSymbolType.Unknown);
tagName = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
}
else if (bangSymbol != null)
{
tagName = new HtmlSymbol(bangSymbol.Start, "!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text);
tagName = new HtmlSymbol("!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text);
}
else
{
@ -1059,12 +1067,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Span.ChunkGenerator = SpanChunkGenerator.Null;
Accept(_bufferedOpenAngle);
var textLocation = CurrentLocation;
var textLocation = CurrentStart;
Assert(HtmlSymbolType.Text);
AcceptAndMoveNext();
var bookmark = CurrentLocation.AbsoluteIndex;
var bookmark = CurrentStart.AbsoluteIndex;
IEnumerable<HtmlSymbol> tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
var empty = At(HtmlSymbolType.ForwardSlash);
if (empty)
@ -1072,7 +1080,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Accept(tokens);
Assert(HtmlSymbolType.ForwardSlash);
AcceptAndMoveNext();
bookmark = CurrentLocation.AbsoluteIndex;
bookmark = CurrentStart.AbsoluteIndex;
tokens = ReadWhile(IsSpacingToken(includeNewLines: true));
}
@ -1151,7 +1159,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Technically, void elements like "meta" are not allowed to have end tags. Just in case they do,
// we need to look ahead at the next set of tokens. If we see "<", "/", tag name, accept it and the ">" following it
// Place a bookmark
var bookmark = CurrentLocation.AbsoluteIndex;
var bookmark = CurrentStart.AbsoluteIndex;
// Skip whitespace
IEnumerable<HtmlSymbol> whiteSpace = ReadWhile(IsSpacingToken(includeNewLines: true));
@ -1230,7 +1238,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
while (!seenEndScript && !EndOfFile)
{
SkipToAndParseCode(HtmlSymbolType.OpenAngle);
var tagStart = CurrentLocation;
var tagStart = CurrentStart;
if (NextIs(HtmlSymbolType.ForwardSlash))
{
@ -1438,6 +1446,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
using (Context.Builder.StartBlock(BlockType.Markup))
{
Span.Start = CurrentLocation;
NextToken();
while (!EndOfFile)
{
@ -1586,6 +1596,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
using (PushSpanConfig(DefaultMarkupSpan))
{
Span.Start = CurrentLocation;
using (Context.Builder.StartBlock(BlockType.Markup))
{
NextToken();
@ -1652,7 +1664,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
EnsureCurrent();
if (string.Equals(CurrentSymbol.Content, nestingSequenceComponents[0], Comparison))
{
var bookmark = CurrentSymbol.Start.AbsoluteIndex;
var bookmark = Context.Source.Position - CurrentSymbol.Content.Length;
try
{
foreach (string component in nestingSequenceComponents)
@ -1706,7 +1718,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
if (string.Equals(possibleStart, sequence, Comparison))
{
// Capture the current symbol and "put it back" (really we just want to clear CurrentSymbol)
var bookmark = Context.Source.Position;
var bookmark = CurrentStart;
var sym = CurrentSymbol;
PutCurrentBack();
@ -1717,6 +1729,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
pair = Language.SplitSymbol(pair.Item2, sequence.Length, HtmlSymbolType.Text);
var sequenceToken = pair.Item1;
var postSequence = pair.Item2;
var postSequenceBookmark = bookmark.AbsoluteIndex + preSequence.Content.Length + pair.Item1.Content.Length;
// Accept the first chunk (up to the nesting sequence we just saw)
if (!string.IsNullOrEmpty(preSequence.Content))
@ -1729,22 +1742,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// This is 'popping' the final entry on the stack of nesting sequences
// A caller higher in the parsing stack will accept the sequence token, so advance
// to it
Context.Source.Position = sequenceToken.Start.AbsoluteIndex;
Context.Source.Position = bookmark.AbsoluteIndex + preSequence.Content.Length;
}
else
{
// This isn't the end of the last nesting sequence, accept the token and keep going
Accept(sequenceToken);
// Position at the start of the postSequence symbol
if (postSequence != null)
{
Context.Source.Position = postSequence.Start.AbsoluteIndex;
}
else
{
Context.Source.Position = bookmark;
}
// Position at the start of the postSequence symbol, which might be null.
Context.Source.Position = postSequenceBookmark;
}
// Return the value we were asked to return if matched, since we found a nesting sequence

31
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlSymbol.cs
View File

@ -8,17 +8,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal class HtmlSymbol : SymbolBase<HtmlSymbolType>
{
public HtmlSymbol(int absoluteIndex, int lineIndex, int characterIndex, string content, HtmlSymbolType type)
: this(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, RazorError.EmptyArray)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public HtmlSymbol(SourceLocation start, string content, HtmlSymbolType type)
: base(start, content, type, RazorError.EmptyArray)
public HtmlSymbol(string content, HtmlSymbolType type)
: base(content, type, RazorError.EmptyArray)
{
if (content == null)
{
@ -27,26 +18,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
public HtmlSymbol(
int absoluteIndex,
int lineIndex,
int characterIndex,
string content,
HtmlSymbolType type,
IReadOnlyList<RazorError> errors)
: base(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, errors)
{
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
}
public HtmlSymbol(
SourceLocation start,
string content,
HtmlSymbolType type,
IReadOnlyList<RazorError> errors)
: base(start, content, type, errors)
: base(content, type, errors)
{
if (content == null)
{

4
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlTokenizer.cs
View File

@ -36,9 +36,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
get { return HtmlSymbolType.RazorCommentStar; }
}
protected override HtmlSymbol CreateSymbol(SourceLocation start, string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors)
protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList<RazorError> errors)
{
return new HtmlSymbol(start, content, type, errors);
return new HtmlSymbol(content, type, errors);
}
protected override StateResult Dispatch()

6
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ISymbol.cs
View File

@ -5,10 +5,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal interface ISymbol
{
SourceLocation Start { get; }
Span Parent { get; set; }
string Content { get; }
void OffsetStart(SourceLocation documentStart);
void ChangeStart(SourceLocation newStart);
SourceLocation Start { get; }
}
}

11
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LanguageCharacteristics.cs
View File

@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public abstract string GetSample(TSymbolType type);
public abstract TTokenizer CreateTokenizer(ITextDocument source);
public abstract TSymbolType FlipBracket(TSymbolType bracket);
public abstract TSymbol CreateMarkerSymbol(SourceLocation location);
public abstract TSymbol CreateMarkerSymbol();
public virtual IEnumerable<TSymbol> TokenizeString(string content)
{
@ -30,7 +30,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
TSymbol sym;
while ((sym = tok.NextSymbol()) != null)
{
sym.OffsetStart(start);
yield return sym;
}
}
@ -88,12 +87,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public virtual Tuple<TSymbol, TSymbol> SplitSymbol(TSymbol symbol, int splitAt, TSymbolType leftType)
{
var left = CreateSymbol(symbol.Start, symbol.Content.Substring(0, splitAt), leftType, RazorError.EmptyArray);
var left = CreateSymbol(symbol.Content.Substring(0, splitAt), leftType, RazorError.EmptyArray);
TSymbol right = null;
if (splitAt < symbol.Content.Length)
{
right = CreateSymbol(SourceLocationTracker.CalculateNewLocation(symbol.Start, left.Content), symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors);
right = CreateSymbol(symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors);
}
return Tuple.Create(left, right);
}
@ -104,6 +105,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return type == KnownSymbolType.Unknown || !Equals(GetKnownSymbolType(type), GetKnownSymbolType(KnownSymbolType.Unknown));
}
protected abstract TSymbol CreateSymbol(SourceLocation location, string content, TSymbolType type, IReadOnlyList<RazorError> errors);
protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList<RazorError> errors);
}
}

51
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanBuilder.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
@ -8,26 +9,43 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal class SpanBuilder
{
private SourceLocation _start;
private List<ISymbol> _symbols;
private SourceLocationTracker _tracker = new SourceLocationTracker();
private SourceLocationTracker _tracker;
public SpanBuilder(Span original)
{
Kind = original.Kind;
_symbols = new List<ISymbol>(original.Symbols);
EditHandler = original.EditHandler;
Start = original.Start;
_start = original.Start;
ChunkGenerator = original.ChunkGenerator;
_symbols = new List<ISymbol>(original.Symbols);
_tracker = new SourceLocationTracker(original.Start);
}
public SpanBuilder()
public SpanBuilder(SourceLocation location)
{
_tracker = new SourceLocationTracker();
Reset();
Start = location;
}
public ISpanChunkGenerator ChunkGenerator { get; set; }
public SourceLocation Start { get; set; }
public SourceLocation Start
{
get { return _start; }
set
{
_start = value;
_tracker.CurrentLocation = value;
}
}
public SourceLocation End => _tracker.CurrentLocation;
public SpanKind Kind { get; set; }
@ -51,15 +69,24 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Need to potentially allocate a new list because Span.ReplaceWith takes ownership
// of the original list.
_symbols = null;
_symbols = new List<ISymbol>();
EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty<ISymbol>());
ChunkGenerator = SpanChunkGenerator.Null;
Start = SourceLocation.Zero;
Start = SourceLocation.Undefined;
}
public Span Build()
{
return new Span(this);
var span = new Span(this);
for (var i = 0; i < span.Symbols.Count; i++)
{
var symbol = span.Symbols[i];
symbol.Parent = span;
}
return span;
}
public void ClearSymbols()
@ -74,15 +101,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return;
}
if (Symbols.Count == 0)
if (Start.Equals(SourceLocation.Undefined))
{
Start = symbol.Start;
symbol.ChangeStart(SourceLocation.Zero);
_tracker.CurrentLocation = SourceLocation.Zero;
}
else
{
symbol.ChangeStart(_tracker.CurrentLocation);
throw new InvalidOperationException("SpanBuilder must have a valid location");
}
_symbols.Add(symbol);

1
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanEditHandler.cs
View File

@ -72,7 +72,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
newSpan.ClearSymbols();
foreach (ISymbol sym in Tokenizer(newContent))
{
sym.OffsetStart(target.Start);
newSpan.Accept(sym);
}
if (target.Next != null)

50
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolBase.cs
View File

@ -11,7 +11,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
internal abstract class SymbolBase<TType> : ISymbol where TType : struct
{
protected SymbolBase(
SourceLocation start,
string content,
TType type,
IReadOnlyList<RazorError> errors)
@ -21,13 +20,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
throw new ArgumentNullException(nameof(content));
}
Start = start;
Content = content;
Type = type;
Errors = errors;
}
public SourceLocation Start { get; private set; }
public Span Parent { get; set; }
public IReadOnlyList<RazorError> Errors { get; }
@ -35,11 +33,35 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public TType Type { get; }
public SourceLocation Start
{
get
{
if (Parent == null)
{
return SourceLocation.Undefined;
}
var tracker = new SourceLocationTracker(Parent.Start);
for (var i = 0; i < Parent.Symbols.Count; i++)
{
var symbol = Parent.Symbols[i];
if (object.ReferenceEquals(this, symbol))
{
break;
}
tracker.UpdateLocation(symbol.Content);
}
return tracker.CurrentLocation;
}
}
public override bool Equals(object obj)
{
var other = obj as SymbolBase<TType>;
return other != null &&
Start.Equals(other.Start) &&
string.Equals(Content, other.Content, StringComparison.Ordinal) &&
Type.Equals(other.Type);
}
@ -47,26 +69,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public override int GetHashCode()
{
// Hash code should include only immutable properties.
var hashCodeCombiner = HashCodeCombiner.Start();
hashCodeCombiner.Add(Content, StringComparer.Ordinal);
hashCodeCombiner.Add(Type);
var hash = HashCodeCombiner.Start();
hash.Add(Content, StringComparer.Ordinal);
hash.Add(Type);
return hashCodeCombiner;
return hash;
}
public override string ToString()
{
return string.Format(CultureInfo.InvariantCulture, "{0} {1} - [{2}]", Start, Type, Content);
}
public void OffsetStart(SourceLocation documentStart)
{
Start = documentStart + Start;
}
public void ChangeStart(SourceLocation newStart)
{
Start = newStart;
return string.Format(CultureInfo.InvariantCulture, "{0} [{1}]", Type, Content);
}
}
}

39
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolExtensions.cs
View File

@ -1,39 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal static class SymbolExtensions
{
public static LocationTagged<string> GetContent(this SpanBuilder span)
{
return GetContent(span, e => e);
}
public static LocationTagged<string> GetContent(this SpanBuilder span, Func<IEnumerable<ISymbol>, IEnumerable<ISymbol>> filter)
{
return GetContent(filter(span.Symbols), span.Start);
}
public static LocationTagged<string> GetContent(this IEnumerable<ISymbol> symbols, SourceLocation spanStart)
{
if (symbols.Any())
{
return new LocationTagged<string>(string.Concat(symbols.Select(s => s.Content)), spanStart + symbols.First().Start);
}
else
{
return new LocationTagged<string>(string.Empty, spanStart);
}
}
public static LocationTagged<string> GetContent(this ISymbol symbol)
{
return new LocationTagged<string>(symbol.Content, symbol.Start);
}
}
}

9
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TagHelperBlockRewriter.cs
View File

@ -149,7 +149,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
ErrorSink errorSink)
{
var afterEquals = false;
var builder = new SpanBuilder
var builder = new SpanBuilder(span.Start)
{
ChunkGenerator = span.ChunkGenerator,
EditHandler = span.EditHandler,
@ -190,7 +190,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
capturedAttributeValueStart = true;
attributeValueStartLocation = span.Start + symbol.Start;
attributeValueStartLocation = symbol.Start;
}
builder.Accept(symbol);
@ -260,7 +260,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
attributeValueStartLocation =
span.Start +
symbolStartLocation +
new SourceLocation(absoluteIndex: 1, lineIndex: 0, characterIndex: 1);
@ -280,7 +279,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
// After all symbols have been added we need to set the builders start position so we do not indirectly
// modify each symbol's Start location.
// modify the span's start location.
builder.Start = attributeValueStartLocation;
if (name == null)
@ -613,7 +612,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
.OfType<HtmlSymbol>()
.First(sym => sym.Type != HtmlSymbolType.WhiteSpace && sym.Type != HtmlSymbolType.NewLine);
return nodeStart + firstNonWhitespaceSymbol.Start;
return firstNonWhitespaceSymbol.Start;
}
private static Span CreateMarkupAttribute(SpanBuilder builder, bool isBoundNonStringAttribute)

33
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/Tokenizer.cs
View File

@ -60,14 +60,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
}
protected SourceLocation CurrentLocation
{
get { return Source.Location; }
}
public SourceLocation CurrentLocation => Source.Location;
protected SourceLocation CurrentStart { get; private set; }
public SourceLocation CurrentStart { get; private set; }
protected abstract TSymbol CreateSymbol(SourceLocation start, string content, TSymbolType type, IReadOnlyList<RazorError> errors);
protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList<RazorError> errors);
protected abstract StateResult Dispatch();
@ -92,6 +89,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
// Post-Condition: Buffer should be empty at the end of Next()
Debug.Assert(Buffer.Length == 0);
// Post-Condition: Token should be non-zero length unless we're at EOF.
Debug.Assert(EndOfFile || !CurrentStart.Equals(CurrentLocation));
return symbol;
}
@ -193,19 +193,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected void StartSymbol()
{
Buffer.Clear();
Debug.Assert(Buffer.Length == 0);
Debug.Assert(CurrentErrors.Count == 0);
CurrentStart = CurrentLocation;
CurrentErrors.Clear();
}
protected TSymbol EndSymbol(TSymbolType type)
{
return EndSymbol(CurrentStart, type);
}
protected TSymbol EndSymbol(SourceLocation start, TSymbolType type)
{
TSymbol sym = null;
TSymbol symbol = null;
if (HaveContent)
{
// Perf: Don't allocate a new errors array unless necessary.
@ -215,10 +211,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
errors[i] = CurrentErrors[i];
}
sym = CreateSymbol(start, GetSymbolContent(type), type, errors);
symbol = CreateSymbol(GetSymbolContent(type), type, errors);
Buffer.Clear();
CurrentErrors.Clear();
}
StartSymbol();
return sym;
return symbol;
}
protected virtual string GetSymbolContent(TSymbolType type)

37
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerBackedParser.cs
View File

@ -18,10 +18,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected TokenizerBackedParser(LanguageCharacteristics<TTokenizer, TSymbol, TSymbolType> language, ParserContext context)
: base(context)
{
Span = new SpanBuilder();
Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView<TTokenizer, TSymbol, TSymbolType>(languageTokenizer);
Span = new SpanBuilder(CurrentLocation);
}
protected SpanBuilder Span { get; private set; }
@ -35,10 +36,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected TSymbol PreviousSymbol { get; private set; }
protected SourceLocation CurrentLocation
{
get { return (EndOfFile || CurrentSymbol == null) ? Context.Source.Location : CurrentSymbol.Start; }
}
protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation;
protected SourceLocation CurrentStart => _tokenizer.Tokenizer.CurrentStart;
protected bool EndOfFile
{
@ -163,7 +163,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
var left = CurrentSymbol.Type;
var right = Language.FlipBracket(left);
var start = CurrentLocation;
var start = CurrentStart;
AcceptAndMoveNext();
if (EndOfFile && ((mode & BalancingModes.NoErrorOnFailure) != BalancingModes.NoErrorOnFailure))
{
@ -180,7 +180,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start)
{
var startPosition = CurrentLocation.AbsoluteIndex;
var startPosition = CurrentStart.AbsoluteIndex;
var nesting = 1;
if (!EndOfFile)
{
@ -196,7 +196,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
HandleEmbeddedTransition();
// Reset backtracking since we've already outputted some spans.
startPosition = CurrentLocation.AbsoluteIndex;
startPosition = CurrentStart.AbsoluteIndex;
}
if (At(left))
{
@ -328,15 +328,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
protected internal void AddMarkerSymbolIfNecessary()
{
AddMarkerSymbolIfNecessary(CurrentLocation);
}
protected internal void AddMarkerSymbolIfNecessary(SourceLocation location)
{
if (Span.Symbols.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharacters.Any)
{
Accept(Language.CreateMarkerSymbol(location));
Accept(Language.CreateMarkerSymbol());
}
}
@ -362,9 +357,18 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
if (Span.Symbols.Count > 0)
{
var nextStart = Span.End;
var builtSpan = Span.Build();
Context.Builder.Add(builtSpan);
Initialize(Span);
// Ensure spans are contiguous.
//
// Note: Using Span.End here to avoid CurrentLocation. CurrentLocation will
// vary depending on what tokens have been read. We often read a token and *then*
// make a decision about whether to include it in the current span.
Span.Start = nextStart;
}
}
@ -464,7 +468,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
errorLength = Math.Max(CurrentSymbol.Content.Length, 1);
}
Context.ErrorSink.OnError(CurrentLocation, errorBase(error), errorLength);
Context.ErrorSink.OnError(CurrentStart, errorBase(error), errorLength);
}
return found;
}
@ -475,6 +479,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
return NextToken();
}
return true;
}
@ -612,7 +617,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
using (Context.Builder.StartBlock(BlockType.Comment))
{
Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator();
var start = CurrentLocation;
var start = CurrentStart;
Expected(KnownSymbolType.CommentStart);
Output(SpanKind.Transition, AcceptedCharacters.None);

9
src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerView.cs
View File

@ -34,15 +34,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void PutBack(TSymbol symbol)
{
Debug.Assert(Source.Position == symbol.Start.AbsoluteIndex + symbol.Content.Length);
if (Source.Position != symbol.Start.AbsoluteIndex + symbol.Content.Length)
{
// We've already passed this symbol
throw new InvalidOperationException(
LegacyResources.FormatTokenizerView_CannotPutBack(
symbol.Start.AbsoluteIndex + symbol.Content.Length,
Source.Position));
}
Source.Position -= symbol.Content.Length;
Current = null;
EndOfFile = Source.Position >= Source.Length;

4
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/BlockTest.cs
View File

@ -13,7 +13,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
// Arrange
var builder = new BlockBuilder() { Type = BlockType.Comment };
var span = new SpanBuilder() { Kind = SpanKind.Code }.Build();
var span = new SpanBuilder(SourceLocation.Undefined) { Kind = SpanKind.Code }.Build();
builder.Children.Add(span);
// Act
@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void ConstructorTransfersChildrenFromBlockBuilder()
{
// Arrange
var expected = new SpanBuilder() { Kind = SpanKind.Code }.Build();
var expected = new SpanBuilder(SourceLocation.Undefined) { Kind = SpanKind.Code }.Build();
var builder = new BlockBuilder()
{
Type = BlockType.Functions

2
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpAutoCompleteTest.cs
View File

@ -121,7 +121,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.Markup("<p>").Accepts(AcceptedCharacters.None)),
new MarkupTagBlock(
Factory.Markup("</p>").Accepts(AcceptedCharacters.None))),
Factory.Span(SpanKind.Code, new CSharpSymbol(Factory.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown))
Factory.Span(SpanKind.Code, new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown))
.With(new StatementChunkGenerator())
),
new RazorError(

26
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpRazorCommentsTest.cs
View File

@ -19,10 +19,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
.Accepts(AcceptedCharacters.None),
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
Factory.Span(
SpanKind.Comment,
new HtmlSymbol(
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any))),
new RazorError(
LegacyResources.ParseError_RazorComment_Not_Terminated,
@ -42,7 +43,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -70,7 +70,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new CSharpSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
CSharpSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -102,7 +101,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new CSharpSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
CSharpSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any)))),
@ -142,7 +140,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -183,9 +180,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
.Accepts(AcceptedCharacters.None),
Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new CSharpSymbol(Factory.LocationTracker.CurrentLocation,
string.Empty,
CSharpSymbolType.Unknown))
Factory.Span(SpanKind.Comment, new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any)))),
new RazorError(
LegacyResources.ParseError_RazorComment_Not_Terminated,
@ -215,7 +210,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -248,7 +242,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -263,7 +256,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -294,7 +286,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -310,7 +301,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
string.Empty,
HtmlSymbolType.Unknown))
.Accepts(AcceptedCharacters.Any),
@ -341,7 +331,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
" hello ",
HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any),
@ -356,7 +345,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
" world ",
HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any),
@ -390,7 +378,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
" content ",
HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any),
@ -405,7 +392,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar)
.Accepts(AcceptedCharacters.None),
Factory.Span(SpanKind.Comment, new HtmlSymbol(
Factory.LocationTracker.CurrentLocation,
Environment.NewLine + "content" + Environment.NewLine,
HtmlSymbolType.RazorComment))
.Accepts(AcceptedCharacters.Any),

51
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerCommentTest.cs
View File

@ -12,80 +12,83 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer("@* Foo * Bar * Baz *",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz *", CSharpSymbolType.RazorComment));
TestTokenizer(
"@* Foo * Bar * Baz *",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(" Foo * Bar * Baz *", CSharpSymbolType.RazorComment));
}
[Fact]
public void Next_Ignores_Star_Without_Trailing_At()
{
TestTokenizer("@* Foo * Bar * Baz *@",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol(19, 0, 19, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(20, 0, 20, "@", CSharpSymbolType.RazorCommentTransition));
TestTokenizer(
"@* Foo * Bar * Baz *@",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(" Foo * Bar * Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{
TestTokenizer("@* Foo Bar Baz *@",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(2, 0, 2, " Foo Bar Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol(15, 0, 15, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(16, 0, 16, "@", CSharpSymbolType.RazorCommentTransition));
TestTokenizer(
"@* Foo Bar Baz *@",
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(" Foo Bar Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{
TestTokenizer("// Foo Bar Baz", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment));
TestTokenizer("// Foo Bar Baz", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment));
}
[Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline()
{
TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining);
TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol(0, 0, 0, "// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining);
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol("// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment));
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment));
}
[Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{
TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol("/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol(0, 0, 0, "/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol("/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining);
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol("/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining);
}
}
}

26
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerIdentifierTest.cs
View File

@ -10,73 +10,73 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Simple_Identifier_Is_Recognized()
{
TestTokenizer("foo", new CSharpSymbol(0, 0, 0, "foo", CSharpSymbolType.Identifier));
TestTokenizer("foo", new CSharpSymbol("foo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized()
{
TestTokenizer("_foo", new CSharpSymbol(0, 0, 0, "_foo", CSharpSymbolType.Identifier));
TestTokenizer("_foo", new CSharpSymbol("_foo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Digits()
{
TestTokenizer("foo4", new CSharpSymbol(0, 0, 0, "foo4", CSharpSymbolType.Identifier));
TestTokenizer("foo4", new CSharpSymbol("foo4", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Titlecase_Letter()
{
TestTokenizer("ῼfoo", new CSharpSymbol(0, 0, 0, "ῼfoo", CSharpSymbolType.Identifier));
TestTokenizer("ῼfoo", new CSharpSymbol("ῼfoo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Letter_Modifier()
{
TestTokenizer("ᵊfoo", new CSharpSymbol(0, 0, 0, "ᵊfoo", CSharpSymbolType.Identifier));
TestTokenizer("ᵊfoo", new CSharpSymbol("ᵊfoo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Other_Letter()
{
TestTokenizer("ƻfoo", new CSharpSymbol(0, 0, 0, "ƻfoo", CSharpSymbolType.Identifier));
TestTokenizer("ƻfoo", new CSharpSymbol("ƻfoo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Number_Letter()
{
TestTokenizer("Ⅽool", new CSharpSymbol(0, 0, 0, "Ⅽool", CSharpSymbolType.Identifier));
TestTokenizer("Ⅽool", new CSharpSymbol("Ⅽool", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark()
{
TestTokenizer("foo\u0300", new CSharpSymbol(0, 0, 0, "foo\u0300", CSharpSymbolType.Identifier));
TestTokenizer("foo\u0300", new CSharpSymbol("foo\u0300", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark()
{
TestTokenizer("fooः", new CSharpSymbol(0, 0, 0, "fooः", CSharpSymbolType.Identifier));
TestTokenizer("fooः", new CSharpSymbol("fooः", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_English_Digit()
{
TestTokenizer("foo١", new CSharpSymbol(0, 0, 0, "foo١", CSharpSymbolType.Identifier));
TestTokenizer("foo١", new CSharpSymbol("foo١", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Connector_Punctuation()
{
TestTokenizer("foo‿bar", new CSharpSymbol(0, 0, 0, "foo‿bar", CSharpSymbolType.Identifier));
TestTokenizer("foo‿bar", new CSharpSymbol("foo‿bar", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Format_Character()
{
TestTokenizer("foo؃bar", new CSharpSymbol(0, 0, 0, "foo؃bar", CSharpSymbolType.Identifier));
TestTokenizer("foo؃bar", new CSharpSymbol("foo؃bar", CSharpSymbolType.Identifier));
}
[Fact]
@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private void TestKeyword(string keyword, CSharpKeyword keywordType)
{
TestTokenizer(keyword, new CSharpSymbol(0, 0, 0, keyword, CSharpSymbolType.Keyword) { Keyword = keywordType });
TestTokenizer(keyword, new CSharpSymbol(keyword, CSharpSymbolType.Keyword) { Keyword = keywordType });
}
}
}

36
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerLiteralTest.cs
View File

@ -41,7 +41,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{
TestTokenizer("42a", new CSharpSymbol(0, 0, 0, "42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
TestTokenizer("42a", new CSharpSymbol("42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
@ -75,25 +75,25 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{
TestTokenizer("0xDEADBEEFz", new CSharpSymbol(0, 0, 0, "0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
TestTokenizer("0xDEADBEEFz", new CSharpSymbol("0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{
TestTokenizer("3.a", new CSharpSymbol(0, 0, 0, "3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
TestTokenizer("3.a", new CSharpSymbol("3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Simple_Real_Literal_Is_Recognized()
{
TestTokenizer("3.14159", new CSharpSymbol(0, 0, 0, "3.14159", CSharpSymbolType.RealLiteral));
TestTokenizer("3.14159", new CSharpSymbol("3.14159", CSharpSymbolType.RealLiteral));
}
[Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{
TestTokenizer(".14159", new CSharpSymbol(0, 0, 0, ".14159", CSharpSymbolType.RealLiteral));
TestTokenizer(".14159", new CSharpSymbol(".14159", CSharpSymbolType.RealLiteral));
}
[Fact]
@ -173,37 +173,37 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
TestTokenizer("'foo\n", new CSharpSymbol(0, 0, 0, "'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\n", new CSharpSymbol("'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("'foo\\\n", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\\n", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer("'foo\\\nflarg", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\\nflarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Allows_Escaped_Escape()
{
TestTokenizer("'foo\\\\'blah", new CSharpSymbol(0, 0, 0, "'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
TestTokenizer("'foo\\\\'blah", new CSharpSymbol("'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
@ -227,37 +227,37 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
TestTokenizer("\"foo\n", new CSharpSymbol(0, 0, 0, "\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\n", new CSharpSymbol("\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("\"foo\\\n", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\\n", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer("\"foo\\\nflarg", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\\nflarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash()
{
TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff()
{
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Allows_Escaped_Escape()
{
TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol(0, 0, 0, "\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol("\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
@ -275,7 +275,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{
TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol(0, 0, 0, "@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol("@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]

8
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerOperatorsTest.cs
View File

@ -125,8 +125,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void LeftShift_Is_Not_Specially_Recognized()
{
TestTokenizer("<<",
new CSharpSymbol(new SourceLocation(0, 0, 0), "<", CSharpSymbolType.LessThan),
new CSharpSymbol(new SourceLocation(1, 0, 1), "<", CSharpSymbolType.LessThan));
new CSharpSymbol("<", CSharpSymbolType.LessThan),
new CSharpSymbol("<", CSharpSymbolType.LessThan));
}
[Fact]
@ -283,8 +283,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void RightShift_Is_Not_Specially_Recognized()
{
TestTokenizer(">>",
new CSharpSymbol(0, 0, 0, ">", CSharpSymbolType.GreaterThan),
new CSharpSymbol(1, 0, 1, ">", CSharpSymbolType.GreaterThan));
new CSharpSymbol(">", CSharpSymbolType.GreaterThan),
new CSharpSymbol(">", CSharpSymbolType.GreaterThan));
}
[Fact]

70
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTest.cs
View File

@ -18,67 +18,74 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Next_Returns_Newline_Token_For_Single_CR()
{
TestTokenizer("\r\ra",
new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine),
IgnoreRemaining);
TestTokenizer(
"\r\ra",
new CSharpSymbol("\r", CSharpSymbolType.NewLine),
new CSharpSymbol("\r", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_LF()
{
TestTokenizer("\n\na",
new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
TestTokenizer(
"\n\na",
new CSharpSymbol("\n", CSharpSymbolType.NewLine),
new CSharpSymbol("\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_NEL()
{
// NEL: Unicode "Next Line" U+0085
TestTokenizer("\u0085\u0085a",
new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine),
IgnoreRemaining);
TestTokenizer(
"\u0085\u0085a",
new CSharpSymbol("\u0085", CSharpSymbolType.NewLine),
new CSharpSymbol("\u0085", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Line_Separator()
{
// Unicode "Line Separator" U+2028
TestTokenizer("\u2028\u2028a",
new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine),
IgnoreRemaining);
TestTokenizer(
"\u2028\u2028a",
new CSharpSymbol("\u2028", CSharpSymbolType.NewLine),
new CSharpSymbol("\u2028", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
{
// Unicode "Paragraph Separator" U+2029
TestTokenizer("\u2029\u2029a",
new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine),
IgnoreRemaining);
TestTokenizer(
"\u2029\u2029a",
new CSharpSymbol("\u2029", CSharpSymbolType.NewLine),
new CSharpSymbol("\u2029", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Single_Newline_Token_For_CRLF()
{
TestTokenizer("\r\n\r\na",
new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine),
new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
TestTokenizer(
"\r\n\r\na",
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
new CSharpSymbol("\r\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Token_For_Whitespace_Characters()
{
TestTokenizer(" \f\t\u000B \n ",
new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine),
new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace));
TestTokenizer(
" \f\t\u000B \n ",
new CSharpSymbol(" \f\t\u000B ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol("\n", CSharpSymbolType.NewLine),
new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace));
}
[Fact]
@ -90,9 +97,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Transition_Is_Recognized_As_SingleCharacter()
{
TestTokenizer("@(",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition),
new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis));
TestTokenizer(
"@(",
new CSharpSymbol("@", CSharpSymbolType.Transition),
new CSharpSymbol("(", CSharpSymbolType.LeftParenthesis));
}
}
}

4
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTestBase.cs
View File

@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
public abstract class CSharpTokenizerTestBase : TokenizerTestBase
{
private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(0, 0, 0, string.Empty, CSharpSymbolType.Unknown);
private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown);
internal override object IgnoreRemaining
{
@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
internal void TestSingleToken(string text, CSharpSymbolType expectedSymbolType)
{
TestTokenizer(text, new CSharpSymbol(0, 0, 0, text, expectedSymbolType));
TestTokenizer(text, new CSharpSymbol(text, expectedSymbolType));
}
internal void TestTokenizer(string input, params CSharpSymbol[] expectedSymbols)

45
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTest.cs
View File

@ -17,23 +17,23 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public void Text_Is_Recognized()
{
TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384",
new HtmlSymbol(0, 0, 0, "foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text));
new HtmlSymbol("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text));
}
[Fact]
public void Whitespace_Is_Recognized()
{
TestTokenizer(" \t\f ",
new HtmlSymbol(0, 0, 0, " \t\f ", HtmlSymbolType.WhiteSpace));
new HtmlSymbol(" \t\f ", HtmlSymbolType.WhiteSpace));
}
[Fact]
public void Newline_Is_Recognized()
{
TestTokenizer("\n\r\r\n",
new HtmlSymbol(0, 0, 0, "\n", HtmlSymbolType.NewLine),
new HtmlSymbol(1, 1, 0, "\r", HtmlSymbolType.NewLine),
new HtmlSymbol(2, 2, 0, "\r\n", HtmlSymbolType.NewLine));
new HtmlSymbol("\n", HtmlSymbolType.NewLine),
new HtmlSymbol("\r", HtmlSymbolType.NewLine),
new HtmlSymbol("\r\n", HtmlSymbolType.NewLine));
}
[Fact]
@ -129,32 +129,35 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer("@* Foo * Bar * Baz *",
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz *", HtmlSymbolType.RazorComment));
TestTokenizer(
"@* Foo * Bar * Baz *",
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(" Foo * Bar * Baz *", HtmlSymbolType.RazorComment));
}
[Fact]
public void Next_Ignores_Star_Without_Trailing_At()
{
TestTokenizer("@* Foo * Bar * Baz *@",
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol(19, 0, 19, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(20, 0, 20, "@", HtmlSymbolType.RazorCommentTransition));
TestTokenizer(
"@* Foo * Bar * Baz *@",
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(" Foo * Bar * Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{
TestTokenizer("@* Foo Bar Baz *@",
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(2, 0, 2, " Foo Bar Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol(15, 0, 15, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(16, 0, 16, "@", HtmlSymbolType.RazorCommentTransition));
TestTokenizer(
"@* Foo Bar Baz *@",
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(" Foo Bar Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition));
}
}
}

4
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTestBase.cs
View File

@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
public abstract class HtmlTokenizerTestBase : TokenizerTestBase
{
private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(0, 0, 0, string.Empty, HtmlSymbolType.Unknown);
private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown);
internal override object IgnoreRemaining
{
@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
internal void TestSingleToken(string text, HtmlSymbolType expectedSymbolType)
{
TestTokenizer(text, new HtmlSymbol(0, 0, 0, text, expectedSymbolType));
TestTokenizer(text, new HtmlSymbol(text, expectedSymbolType));
}
internal void TestTokenizer(string input, params HtmlSymbol[] expectedSymbols)

9
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RawTextSymbol.cs
View File

@ -1,21 +1,24 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Globalization;
namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
{
internal class RawTextSymbol : ISymbol
{
public SourceLocation Start { get; private set; }
public string Content { get; }
public RawTextSymbol(SourceLocation start, string content)
{
Start = start;
Content = content;
}
public SourceLocation Start { get; private set; }
public string Content { get; }
public Span Parent { get; set; }
public int Offset { get; set; }
public override bool Equals(object obj)
{
var other = obj as RawTextSymbol;

47
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TestSpanBuilder.cs
View File

@ -14,7 +14,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return new UnclassifiedCodeSpanConstructor(
self.Span(
SpanKind.Code,
new CSharpSymbol(self.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown)));
new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown)));
}
public static SpanConstructor EmptyHtml(this SpanFactory self)
@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return self
.Span(
SpanKind.Markup,
new HtmlSymbol(self.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown))
new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown))
.With(new MarkupChunkGenerator());
}
@ -178,34 +178,41 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public SpanConstructor Span(SpanKind kind, string content, CSharpSymbolType type)
{
return CreateSymbolSpan(kind, content, st => new CSharpSymbol(st, content, type));
return CreateSymbolSpan(kind, content, () => new CSharpSymbol(content, type));
}
public SpanConstructor Span(SpanKind kind, string content, HtmlSymbolType type)
{
return CreateSymbolSpan(kind, content, st => new HtmlSymbol(st, content, type));
return CreateSymbolSpan(kind, content, () => new HtmlSymbol(content, type));
}
public SpanConstructor Span(SpanKind kind, string content, bool markup)
{
return new SpanConstructor(kind, Tokenize(new[] { content }, markup));
return new SpanConstructor(kind, LocationTracker.CurrentLocation, Tokenize(new[] { content }, markup));
}
public SpanConstructor Span(SpanKind kind, string[] content, bool markup)
{
return new SpanConstructor(kind, Tokenize(content, markup));
return new SpanConstructor(kind, LocationTracker.CurrentLocation, Tokenize(content, markup));
}
public SpanConstructor Span(SpanKind kind, params ISymbol[] symbols)
{
return new SpanConstructor(kind, symbols);
var start = LocationTracker.CurrentLocation;
foreach (var symbol in symbols)
{
LocationTracker.UpdateLocation(symbol.Content);
}
return new SpanConstructor(kind, start, symbols);
}
private SpanConstructor CreateSymbolSpan(SpanKind kind, string content, Func<SourceLocation, ISymbol> ctor)
private SpanConstructor CreateSymbolSpan(SpanKind kind, string content, Func<ISymbol> ctor)
{
var start = LocationTracker.CurrentLocation;
LocationTracker.UpdateLocation(content);
return new SpanConstructor(kind, new[] { ctor(start) });
return new SpanConstructor(kind, start, new[] { ctor() });
}
public void Reset()
@ -220,15 +227,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
private IEnumerable<ISymbol> Tokenize(string content, bool markup)
{
var tok = MakeTokenizer(markup, new SeekableTextReader(content));
ISymbol sym;
var tokenizer = MakeTokenizer(markup, new SeekableTextReader(content));
ISymbol symbol;
ISymbol last = null;
while ((sym = tok.NextSymbol()) != null)
while ((symbol = tokenizer.NextSymbol()) != null)
{
OffsetStart(sym, LocationTracker.CurrentLocation);
last = sym;
yield return sym;
last = symbol;
yield return symbol;
}
LocationTracker.UpdateLocation(content);
}
@ -243,11 +251,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return CodeTokenizerFactory(seekableTextReader);
}
}
private void OffsetStart(ISymbol sym, SourceLocation sourceLocation)
{
sym.OffsetStart(sourceLocation);
}
}
internal static class SpanConstructorExtensions
@ -359,9 +362,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
yield return new RawTextSymbol(SourceLocation.Zero, str);
}
public SpanConstructor(SpanKind kind, IEnumerable<ISymbol> symbols)
public SpanConstructor(SpanKind kind, SourceLocation location, IEnumerable<ISymbol> symbols)
{
Builder = new SpanBuilder();
Builder = new SpanBuilder(location);
Builder.Kind = kind;
Builder.EditHandler = SpanEditHandler.CreateDefault((content) => SpanConstructor.TestTokenizer(content));
foreach (ISymbol sym in symbols)

1
test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs
View File

@ -104,7 +104,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
}
protected override CSharpSymbol CreateSymbol(
SourceLocation start,
string content,
CSharpSymbolType type,
IReadOnlyList<RazorError> errors)