diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpCodeParser.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpCodeParser.cs index 37f933e23b..a643a579db 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpCodeParser.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpCodeParser.cs @@ -165,6 +165,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy throw new InvalidOperationException(LegacyResources.Parser_Context_Not_Set); } + Span.Start = CurrentLocation; + // Unless changed, the block is a statement block using (Context.Builder.StartBlock(BlockType.Statement)) { @@ -179,7 +181,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { var split = Language.SplitSymbol(CurrentSymbol, 1, CSharpSymbolType.Transition); current = split.Item1; - Context.Source.Position = split.Item2.Start.AbsoluteIndex; + + // Back up to the end of the transition + Context.Source.Position -= split.Item2.Content.Length; NextToken(); } else if (At(CSharpSymbolType.Transition)) @@ -201,6 +205,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // No "@" => Jump straight to AfterTransition AfterTransition(); } + Output(SpanKind.Code); } } @@ -258,7 +263,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy StringComparison.Ordinal)) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatParseError_HelperDirectiveNotAvailable( SyntaxConstants.CSharp.HelperKeyword), CurrentSymbol.Content.Length); @@ -297,21 +302,21 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (At(CSharpSymbolType.WhiteSpace) || At(CSharpSymbolType.NewLine)) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.ParseError_Unexpected_WhiteSpace_At_Start_Of_CodeBlock_CS, CurrentSymbol.Content.Length); } else if (EndOfFile) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.ParseError_Unexpected_EndOfFile_At_Start_Of_CodeBlock, length: 1 /* end of file */); } else { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatParseError_Unexpected_Character_At_Start_Of_CodeBlock_CS( CurrentSymbol.Content), CurrentSymbol.Content.Length); @@ -328,7 +333,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void VerbatimBlock() { Assert(CSharpSymbolType.LeftBrace); - var block = new Block(LegacyResources.BlockName_Code, CurrentLocation); + var block = new Block(LegacyResources.BlockName_Code, CurrentStart); AcceptAndMoveNext(); // Set up the "{" span and output @@ -567,7 +572,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void ExplicitExpression() { - var block = new Block(LegacyResources.BlockName_ExplicitExpression, CurrentLocation); + var block = new Block(LegacyResources.BlockName_ExplicitExpression, CurrentStart); Assert(CSharpSymbolType.LeftParenthesis); AcceptAndMoveNext(); Span.EditHandler.AcceptedCharacters = AcceptedCharacters.None; @@ -595,7 +600,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // If necessary, put an empty-content marker symbol here if (Span.Symbols.Count == 0) { - Accept(new CSharpSymbol(CurrentLocation, string.Empty, CSharpSymbolType.Unknown)); + Accept(new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown)); } // Output the content span and then capture the ")" @@ -617,7 +622,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (Context.Builder.ActiveBlocks.Any(block => block.Type == BlockType.Template)) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.ParseError_InlineMarkup_Blocks_Cannot_Be_Nested, length: 1 /* @ */); } @@ -643,12 +648,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void NestedBlock() { Output(SpanKind.Code); + var wasNested = IsNested; IsNested = true; using (PushSpanConfig()) { ParseBlock(); } + + Span.Start = CurrentLocation; Initialize(Span); IsNested = wasNested; NextToken(); @@ -683,12 +691,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // as C#; it should be handled as a period because it's wrapped in markup. var wasNested = IsNested; IsNested = false; + using (PushSpanConfig()) { parseAction(HtmlParser); } + + Span.Start = CurrentLocation; Initialize(Span); + IsNested = wasNested; + NextToken(); } @@ -712,7 +725,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy protected virtual void ReservedDirective(bool topLevel) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatParseError_ReservedWord(CurrentSymbol.Content), CurrentSymbol.Content.Length); AcceptAndMoveNext(); @@ -780,7 +793,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void UsingKeyword(bool topLevel) { Assert(CSharpKeyword.Using); - var block = new Block(CurrentSymbol); + var block = new Block(CurrentSymbol, CurrentStart); AcceptAndMoveNext(); AcceptWhile(IsSpacingToken(includeNewLines: false, includeComments: true)); @@ -817,6 +830,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Set block type to directive Context.Builder.CurrentBlock.Type = BlockType.Directive; + var start = CurrentStart; if (At(CSharpSymbolType.Identifier)) { // non-static using @@ -849,8 +863,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } Span.EditHandler.AcceptedCharacters = AcceptedCharacters.AnyExceptNewline; - Span.ChunkGenerator = new AddImportChunkGenerator( - Span.GetContent(symbols => symbols.Skip(1))); + Span.ChunkGenerator = new AddImportChunkGenerator(new LocationTagged( + string.Concat(Span.Symbols.Skip(1).Select(s => s.Content)), + start)); // Optional ";" if (EnsureCurrent()) @@ -990,7 +1005,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { return; } - var block = new Block(CurrentSymbol); + var block = new Block(CurrentSymbol, CurrentStart); AcceptAndMoveNext(); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); @@ -1016,7 +1031,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (!At(CSharpSymbolType.LeftBrace)) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatParseError_SingleLine_ControlFlowStatements_Not_Allowed( Language.GetSample(CSharpSymbolType.LeftBrace), CurrentSymbol.Content), @@ -1031,7 +1046,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void UnconditionalBlock() { Assert(CSharpSymbolType.Keyword); - var block = new Block(CurrentSymbol); + var block = new Block(CurrentSymbol, CurrentStart); AcceptAndMoveNext(); AcceptWhile(IsSpacingToken(includeNewLines: true, includeComments: true)); ExpectCodeBlock(block); @@ -1041,7 +1056,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { Assert(CSharpKeyword.Catch); - var block = new Block(CurrentSymbol); + var block = new Block(CurrentSymbol, CurrentStart); // Accept "catch" AcceptAndMoveNext(); @@ -1075,7 +1090,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void ConditionalBlock(bool topLevel) { Assert(CSharpSymbolType.Keyword); - var block = new Block(CurrentSymbol); + var block = new Block(CurrentSymbol, CurrentStart); ConditionalBlock(block); if (topLevel) { @@ -1125,8 +1140,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Accept whitespace but always keep the last whitespace node so we can put it back if necessary var lastWhitespace = AcceptWhiteSpaceInLines(); - Debug.Assert(lastWhitespace == null || - (lastWhitespace.Start.AbsoluteIndex + lastWhitespace.Content.Length == CurrentLocation.AbsoluteIndex)); if (EndOfFile) { @@ -1138,7 +1151,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } var type = CurrentSymbol.Type; - var loc = CurrentLocation; + var loc = CurrentStart; // Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text var isSingleLineMarkup = type == CSharpSymbolType.Transition && @@ -1208,7 +1221,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy break; case CSharpSymbolType.LeftBrace: // Verbatim Block - block = block ?? new Block(LegacyResources.BlockName_Code, CurrentLocation); + block = block ?? new Block(LegacyResources.BlockName_Code, CurrentStart); AcceptAndMoveNext(); CodeBlock(block); break; @@ -1260,7 +1273,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (At(CSharpSymbolType.LeftBrace)) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.ParseError_Unexpected_Nested_CodeBlock, length: 1 /* { */); } @@ -1280,7 +1293,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { while (!EndOfFile) { - var bookmark = CurrentLocation.AbsoluteIndex; + var bookmark = CurrentStart.AbsoluteIndex; IEnumerable read = ReadWhile(sym => sym.Type != CSharpSymbolType.Semicolon && sym.Type != CSharpSymbolType.RazorCommentTransition && sym.Type != CSharpSymbolType.Transition && @@ -1479,7 +1492,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (EndOfFile) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, tokenDescriptor.Kind.ToString().ToLowerInvariant()), length: 1); return; @@ -1492,7 +1505,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (!NamespaceOrTypeName()) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatDirectiveExpectsTypeName(descriptor.Name), CurrentSymbol.Content.Length); @@ -1509,7 +1522,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatDirectiveExpectsIdentifier(descriptor.Name), CurrentSymbol.Content.Length); return; @@ -1528,7 +1541,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, tokenDescriptor.Value), CurrentSymbol.Content.Length); return; @@ -1555,7 +1568,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else if (!EndOfFile) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, Environment.NewLine), CurrentSymbol.Content.Length); } @@ -1573,12 +1586,17 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // as C#; it should be handled as a period because it's wrapped in markup. var wasNested = IsNested; IsNested = false; + using (PushSpanConfig()) { HtmlParser.ParseRazorBlock(Tuple.Create("{", "}"), caseSensitive: true); } + + Span.Start = CurrentLocation; Initialize(Span); + IsNested = wasNested; + NextToken(); }); break; @@ -1602,14 +1620,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (EndOfFile) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatUnexpectedEOFAfterDirective(descriptor.Name, "{"), length: 1 /* { */); } else if (!At(CSharpSymbolType.LeftBrace)) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.FormatUnexpectedDirectiveLiteral(descriptor.Name, "{"), CurrentSymbol.Content.Length); } @@ -1617,7 +1635,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { var editHandler = new AutoCompleteEditHandler(Language.TokenizeString, autoCompleteAtEndOfSpan: true); Span.EditHandler = editHandler; - var startingBraceLocation = CurrentLocation; + var startingBraceLocation = CurrentStart; Accept(CurrentSymbol); Span.ChunkGenerator = SpanChunkGenerator.Null; Output(SpanKind.MetaCode, AcceptedCharacters.None); @@ -1678,7 +1696,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Set the block type Context.Builder.CurrentBlock.Type = BlockType.Directive; - var keywordLength = Span.GetContent().Value.Length; + var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex; // Accept whitespace var remainingWhitespace = AcceptSingleWhiteSpaceCharacter(); @@ -1714,7 +1732,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } // Pull out the type name - string baseType = Span.GetContent(); + string baseType = string.Concat(Span.Symbols.Select(s => s.Content)); // Set up chunk generation Span.ChunkGenerator = createChunkGenerator(baseType.Trim()); @@ -1727,7 +1745,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void TagHelperDirective(string keyword, Func chunkGeneratorFactory) { AssertDirective(keyword); - var keywordStartLocation = CurrentLocation; + var keywordStartLocation = CurrentStart; // Accept the directive name AcceptAndMoveNext(); @@ -1735,7 +1753,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Set the block type Context.Builder.CurrentBlock.Type = BlockType.Directive; - var keywordLength = Span.GetContent().Value.Length; + var keywordLength = Span.End.AbsoluteIndex - Span.Start.AbsoluteIndex; var foundWhitespace = At(CSharpSymbolType.WhiteSpace); AcceptWhile(CSharpSymbolType.WhiteSpace); @@ -1757,14 +1775,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else { // Need to grab the current location before we accept until the end of the line. - var startLocation = CurrentLocation; + var startLocation = CurrentStart; // Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code // etc. AcceptUntil(CSharpSymbolType.NewLine); // Pull out the value and remove whitespaces and optional quotes - var rawValue = Span.GetContent().Value.Trim(); + var rawValue = string.Concat(Span.Symbols.Select(s => s.Content)).Trim(); var startsWithQuote = rawValue.StartsWith("\"", StringComparison.Ordinal); var endsWithQuote = rawValue.EndsWith("\"", StringComparison.Ordinal); @@ -1807,8 +1825,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Start = start; } - public Block(CSharpSymbol symbol) - : this(GetName(symbol), symbol.Start) + public Block(CSharpSymbol symbol, SourceLocation start) + : this(GetName(symbol), start) { } diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpLanguageCharacteristics.cs index 0aa0fedd7e..8af54a2c35 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpLanguageCharacteristics.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpLanguageCharacteristics.cs @@ -74,9 +74,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return new CSharpTokenizer(source); } - protected override CSharpSymbol CreateSymbol(SourceLocation location, string content, CSharpSymbolType type, IReadOnlyList errors) + protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList errors) { - return new CSharpSymbol(location, content, type, errors); + return new CSharpSymbol(content, type, errors); } public override string GetSample(CSharpSymbolType type) @@ -111,9 +111,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return sample; } - public override CSharpSymbol CreateMarkerSymbol(SourceLocation location) + public override CSharpSymbol CreateMarkerSymbol() { - return new CSharpSymbol(location, string.Empty, CSharpSymbolType.Unknown); + return new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown); } public override CSharpSymbolType GetKnownSymbolType(KnownSymbolType type) diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpSymbol.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpSymbol.cs index 2a4f3f1d59..10cd31470b 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpSymbol.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpSymbol.cs @@ -8,17 +8,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { internal class CSharpSymbol : SymbolBase { - public CSharpSymbol(int absoluteIndex, int lineIndex, int characterIndex, string content, CSharpSymbolType type) - : this(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, RazorError.EmptyArray) - { - if (content == null) - { - throw new ArgumentNullException(nameof(content)); - } - } - - public CSharpSymbol(SourceLocation start, string content, CSharpSymbolType type) - : this(start, content, type, RazorError.EmptyArray) + public CSharpSymbol( + string content, + CSharpSymbolType type) + : base(content, type, RazorError.EmptyArray) { if (content == null) { @@ -27,26 +20,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } public CSharpSymbol( - int offset, - int line, - int column, string content, CSharpSymbolType type, IReadOnlyList errors) - : base(new SourceLocation(offset, line, column), content, type, errors) - { - if (content == null) - { - throw new ArgumentNullException(nameof(content)); - } - } - - public CSharpSymbol( - SourceLocation start, - string content, - CSharpSymbolType type, - IReadOnlyList errors) - : base(start, content, type, errors) + : base(content, type, errors) { if (content == null) { diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpTokenizer.cs index 2f55429170..f21804d7d9 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpTokenizer.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/CSharpTokenizer.cs @@ -168,9 +168,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } } - protected override CSharpSymbol CreateSymbol(SourceLocation start, string content, CSharpSymbolType type, IReadOnlyList errors) + protected override CSharpSymbol CreateSymbol(string content, CSharpSymbolType type, IReadOnlyList errors) { - return new CSharpSymbol(start, content, type, errors); + return new CSharpSymbol(content, type, errors); } private StateResult Data() @@ -548,13 +548,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { type = CSharpSymbolType.Keyword; } - - symbol = new CSharpSymbol(CurrentStart, symbolContent, type) + + symbol = new CSharpSymbol(symbolContent, type) { Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null, }; + + Buffer.Clear(); + CurrentErrors.Clear(); } - StartSymbol(); + return Stay(symbol); } diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ConditionalAttributeCollapser.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ConditionalAttributeCollapser.cs index 4ba54c662c..5f0a76408a 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ConditionalAttributeCollapser.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ConditionalAttributeCollapser.cs @@ -39,7 +39,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } // Create a new span containing this content - var span = new SpanBuilder(); + var span = new SpanBuilder(block.Children[0].Start); span.EditHandler = SpanEditHandler.CreateDefault(HtmlLanguageCharacteristics.Instance.TokenizeString); Debug.Assert(block.Children.Count > 0); diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlLanguageCharacteristics.cs index db0126990b..bf445413d3 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlLanguageCharacteristics.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlLanguageCharacteristics.cs @@ -95,9 +95,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } } - public override HtmlSymbol CreateMarkerSymbol(SourceLocation location) + public override HtmlSymbol CreateMarkerSymbol() { - return new HtmlSymbol(location, string.Empty, HtmlSymbolType.Unknown); + return new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown); } public override HtmlSymbolType GetKnownSymbolType(KnownSymbolType type) @@ -125,9 +125,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } } - protected override HtmlSymbol CreateSymbol(SourceLocation location, string content, HtmlSymbolType type, IReadOnlyList errors) + protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList errors) { - return new HtmlSymbol(location, content, type, errors); + return new HtmlSymbol(content, type, errors); } } } diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlMarkupParser.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlMarkupParser.cs index 017eec80e7..31dc029f46 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlMarkupParser.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlMarkupParser.cs @@ -219,10 +219,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { AddMarkerSymbolIfNecessary(); Output(SpanKind.Markup); + using (PushSpanConfig()) { CodeParser.ParseBlock(); } + + Span.Start = CurrentLocation; Initialize(Span); NextToken(); } @@ -271,6 +274,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { using (Context.Builder.StartBlock(BlockType.Markup)) { + Span.Start = CurrentLocation; + if (!NextToken()) { return; @@ -305,7 +310,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else { Context.ErrorSink.OnError( - CurrentSymbol.Start, + CurrentStart, LegacyResources.ParseError_MarkupBlock_Must_Start_With_Tag, CurrentSymbol.Content.Length); } @@ -393,10 +398,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else { _bufferedOpenAngle = null; - _lastTagStart = CurrentLocation; + _lastTagStart = CurrentStart; Assert(HtmlSymbolType.OpenAngle); _bufferedOpenAngle = CurrentSymbol; - var tagStart = CurrentLocation; + var tagStart = CurrentStart; if (!NextToken()) { Accept(_bufferedOpenAngle); @@ -467,7 +472,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (tags.Count == 0) { Context.ErrorSink.OnError( - CurrentLocation, + CurrentStart, LegacyResources.ParseError_OuterTagMissingName, length: 1 /* end of file */); } @@ -620,7 +625,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Accept(_bufferedOpenAngle); Accept(solidus); - var textLocation = CurrentLocation; + var textLocation = CurrentStart; Assert(HtmlSymbolType.Text); AcceptAndMoveNext(); @@ -789,8 +794,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy IEnumerable whitespaceAfterAttributeName) { // First, determine if this is a 'data-' attribute (since those can't use conditional attributes) - var name = nameSymbols.GetContent(Span.Start); - var attributeCanBeConditional = !name.Value.StartsWith("data-", StringComparison.OrdinalIgnoreCase); + var name = string.Concat(nameSymbols.Select(s => s.Content)); + var attributeCanBeConditional = !name.StartsWith("data-", StringComparison.OrdinalIgnoreCase); // Accept the whitespace and name Accept(whitespace); @@ -818,7 +823,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } // We now have the prefix: (i.e. ' foo="') - var prefix = Span.GetContent(); + var prefix = new LocationTagged(string.Concat(Span.Symbols.Select(s => s.Content)), Span.Start); if (attributeCanBeConditional) { @@ -837,10 +842,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } // Capture the suffix - var suffix = new LocationTagged(string.Empty, CurrentLocation); + var suffix = new LocationTagged(string.Empty, CurrentStart); if (quote != HtmlSymbolType.Unknown && At(quote)) { - suffix = CurrentSymbol.GetContent(); + suffix = new LocationTagged(CurrentSymbol.Content, CurrentStart); AcceptAndMoveNext(); } @@ -881,7 +886,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void AttributeValue(HtmlSymbolType quote) { - var prefixStart = CurrentLocation; + var prefixStart = CurrentStart; var prefix = ReadWhile(sym => sym.Type == HtmlSymbolType.WhiteSpace || sym.Type == HtmlSymbolType.NewLine); if (At(HtmlSymbolType.Transition)) @@ -895,8 +900,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Render a single "@" in place of "@@". Span.ChunkGenerator = new LiteralAttributeChunkGenerator( - prefix.GetContent(prefixStart), - new LocationTagged(CurrentSymbol.GetContent(), CurrentLocation)); + new LocationTagged(string.Concat(prefix.Select(s => s.Content)), prefixStart), + new LocationTagged(CurrentSymbol.Content, CurrentStart)); AcceptAndMoveNext(); Output(SpanKind.Markup, AcceptedCharacters.None); @@ -908,7 +913,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy else { Accept(prefix); - var valueStart = CurrentLocation; + var valueStart = CurrentStart; PutCurrentBack(); // Output the prefix but as a null-span. DynamicAttributeBlockChunkGenerator will render it @@ -918,7 +923,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy using (Context.Builder.StartBlock(BlockType.Markup)) { Context.Builder.CurrentBlock.ChunkGenerator = - new DynamicAttributeBlockChunkGenerator(prefix.GetContent(prefixStart), valueStart); + new DynamicAttributeBlockChunkGenerator( + new LocationTagged(string.Concat(prefix.Select(s => s.Content)), prefixStart), + valueStart); OtherParserBlock(); } @@ -931,6 +938,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Literal value // 'quote' should be "Unknown" if not quoted and symbols coming from the tokenizer should never have // "Unknown" type. + var valueStart = CurrentStart; var value = ReadWhile(sym => // These three conditions find separators which break the attribute value into portions sym.Type != HtmlSymbolType.WhiteSpace && @@ -941,8 +949,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy !IsEndOfAttributeValue(quote, sym)); Accept(value); Span.ChunkGenerator = new LiteralAttributeChunkGenerator( - prefix.GetContent(prefixStart), - value.GetContent(prefixStart)); + new LocationTagged(string.Concat(prefix.Select(s => s.Content)), prefixStart), + new LocationTagged(string.Concat(value.Select(s => s.Content)), valueStart)); } Output(SpanKind.Markup); } @@ -1037,11 +1045,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (potentialTagNameSymbol == null || potentialTagNameSymbol.Type != HtmlSymbolType.Text) { - tagName = new HtmlSymbol(potentialTagNameSymbol.Start, string.Empty, HtmlSymbolType.Unknown); + tagName = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown); } else if (bangSymbol != null) { - tagName = new HtmlSymbol(bangSymbol.Start, "!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text); + tagName = new HtmlSymbol("!" + potentialTagNameSymbol.Content, HtmlSymbolType.Text); } else { @@ -1059,12 +1067,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Span.ChunkGenerator = SpanChunkGenerator.Null; Accept(_bufferedOpenAngle); - var textLocation = CurrentLocation; + var textLocation = CurrentStart; Assert(HtmlSymbolType.Text); AcceptAndMoveNext(); - var bookmark = CurrentLocation.AbsoluteIndex; + var bookmark = CurrentStart.AbsoluteIndex; IEnumerable tokens = ReadWhile(IsSpacingToken(includeNewLines: true)); var empty = At(HtmlSymbolType.ForwardSlash); if (empty) @@ -1072,7 +1080,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Accept(tokens); Assert(HtmlSymbolType.ForwardSlash); AcceptAndMoveNext(); - bookmark = CurrentLocation.AbsoluteIndex; + bookmark = CurrentStart.AbsoluteIndex; tokens = ReadWhile(IsSpacingToken(includeNewLines: true)); } @@ -1151,7 +1159,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Technically, void elements like "meta" are not allowed to have end tags. Just in case they do, // we need to look ahead at the next set of tokens. If we see "<", "/", tag name, accept it and the ">" following it // Place a bookmark - var bookmark = CurrentLocation.AbsoluteIndex; + var bookmark = CurrentStart.AbsoluteIndex; // Skip whitespace IEnumerable whiteSpace = ReadWhile(IsSpacingToken(includeNewLines: true)); @@ -1230,7 +1238,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy while (!seenEndScript && !EndOfFile) { SkipToAndParseCode(HtmlSymbolType.OpenAngle); - var tagStart = CurrentLocation; + var tagStart = CurrentStart; if (NextIs(HtmlSymbolType.ForwardSlash)) { @@ -1438,6 +1446,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { using (Context.Builder.StartBlock(BlockType.Markup)) { + Span.Start = CurrentLocation; + NextToken(); while (!EndOfFile) { @@ -1586,6 +1596,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy using (PushSpanConfig(DefaultMarkupSpan)) { + Span.Start = CurrentLocation; + using (Context.Builder.StartBlock(BlockType.Markup)) { NextToken(); @@ -1652,7 +1664,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy EnsureCurrent(); if (string.Equals(CurrentSymbol.Content, nestingSequenceComponents[0], Comparison)) { - var bookmark = CurrentSymbol.Start.AbsoluteIndex; + var bookmark = Context.Source.Position - CurrentSymbol.Content.Length; try { foreach (string component in nestingSequenceComponents) @@ -1706,7 +1718,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy if (string.Equals(possibleStart, sequence, Comparison)) { // Capture the current symbol and "put it back" (really we just want to clear CurrentSymbol) - var bookmark = Context.Source.Position; + var bookmark = CurrentStart; var sym = CurrentSymbol; PutCurrentBack(); @@ -1717,6 +1729,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy pair = Language.SplitSymbol(pair.Item2, sequence.Length, HtmlSymbolType.Text); var sequenceToken = pair.Item1; var postSequence = pair.Item2; + var postSequenceBookmark = bookmark.AbsoluteIndex + preSequence.Content.Length + pair.Item1.Content.Length; // Accept the first chunk (up to the nesting sequence we just saw) if (!string.IsNullOrEmpty(preSequence.Content)) @@ -1729,22 +1742,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // This is 'popping' the final entry on the stack of nesting sequences // A caller higher in the parsing stack will accept the sequence token, so advance // to it - Context.Source.Position = sequenceToken.Start.AbsoluteIndex; + Context.Source.Position = bookmark.AbsoluteIndex + preSequence.Content.Length; } else { // This isn't the end of the last nesting sequence, accept the token and keep going Accept(sequenceToken); - // Position at the start of the postSequence symbol - if (postSequence != null) - { - Context.Source.Position = postSequence.Start.AbsoluteIndex; - } - else - { - Context.Source.Position = bookmark; - } + // Position at the start of the postSequence symbol, which might be null. + Context.Source.Position = postSequenceBookmark; } // Return the value we were asked to return if matched, since we found a nesting sequence diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlSymbol.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlSymbol.cs index f5714e4c59..e9bd8d70f3 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlSymbol.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlSymbol.cs @@ -8,17 +8,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { internal class HtmlSymbol : SymbolBase { - public HtmlSymbol(int absoluteIndex, int lineIndex, int characterIndex, string content, HtmlSymbolType type) - : this(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, RazorError.EmptyArray) - { - if (content == null) - { - throw new ArgumentNullException(nameof(content)); - } - } - - public HtmlSymbol(SourceLocation start, string content, HtmlSymbolType type) - : base(start, content, type, RazorError.EmptyArray) + public HtmlSymbol(string content, HtmlSymbolType type) + : base(content, type, RazorError.EmptyArray) { if (content == null) { @@ -27,26 +18,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } public HtmlSymbol( - int absoluteIndex, - int lineIndex, - int characterIndex, string content, HtmlSymbolType type, IReadOnlyList errors) - : base(new SourceLocation(absoluteIndex, lineIndex, characterIndex), content, type, errors) - { - if (content == null) - { - throw new ArgumentNullException(nameof(content)); - } - } - - public HtmlSymbol( - SourceLocation start, - string content, - HtmlSymbolType type, - IReadOnlyList errors) - : base(start, content, type, errors) + : base(content, type, errors) { if (content == null) { diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlTokenizer.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlTokenizer.cs index d736b18580..324d1e0540 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlTokenizer.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/HtmlTokenizer.cs @@ -36,9 +36,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy get { return HtmlSymbolType.RazorCommentStar; } } - protected override HtmlSymbol CreateSymbol(SourceLocation start, string content, HtmlSymbolType type, IReadOnlyList errors) + protected override HtmlSymbol CreateSymbol(string content, HtmlSymbolType type, IReadOnlyList errors) { - return new HtmlSymbol(start, content, type, errors); + return new HtmlSymbol(content, type, errors); } protected override StateResult Dispatch() diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ISymbol.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ISymbol.cs index e3bea968db..f1dfbcb8df 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ISymbol.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/ISymbol.cs @@ -5,10 +5,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { internal interface ISymbol { - SourceLocation Start { get; } + Span Parent { get; set; } + string Content { get; } - void OffsetStart(SourceLocation documentStart); - void ChangeStart(SourceLocation newStart); + SourceLocation Start { get; } } } diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LanguageCharacteristics.cs index f2011d6030..92df79a27c 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LanguageCharacteristics.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LanguageCharacteristics.cs @@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public abstract string GetSample(TSymbolType type); public abstract TTokenizer CreateTokenizer(ITextDocument source); public abstract TSymbolType FlipBracket(TSymbolType bracket); - public abstract TSymbol CreateMarkerSymbol(SourceLocation location); + public abstract TSymbol CreateMarkerSymbol(); public virtual IEnumerable TokenizeString(string content) { @@ -30,7 +30,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy TSymbol sym; while ((sym = tok.NextSymbol()) != null) { - sym.OffsetStart(start); yield return sym; } } @@ -88,12 +87,14 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public virtual Tuple SplitSymbol(TSymbol symbol, int splitAt, TSymbolType leftType) { - var left = CreateSymbol(symbol.Start, symbol.Content.Substring(0, splitAt), leftType, RazorError.EmptyArray); + var left = CreateSymbol(symbol.Content.Substring(0, splitAt), leftType, RazorError.EmptyArray); + TSymbol right = null; if (splitAt < symbol.Content.Length) { - right = CreateSymbol(SourceLocationTracker.CalculateNewLocation(symbol.Start, left.Content), symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors); + right = CreateSymbol(symbol.Content.Substring(splitAt), symbol.Type, symbol.Errors); } + return Tuple.Create(left, right); } @@ -104,6 +105,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return type == KnownSymbolType.Unknown || !Equals(GetKnownSymbolType(type), GetKnownSymbolType(KnownSymbolType.Unknown)); } - protected abstract TSymbol CreateSymbol(SourceLocation location, string content, TSymbolType type, IReadOnlyList errors); + protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList errors); } } diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanBuilder.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanBuilder.cs index 07ed152db7..b68286a331 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanBuilder.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanBuilder.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Linq; @@ -8,26 +9,43 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { internal class SpanBuilder { + private SourceLocation _start; private List _symbols; - private SourceLocationTracker _tracker = new SourceLocationTracker(); + private SourceLocationTracker _tracker; public SpanBuilder(Span original) { Kind = original.Kind; - _symbols = new List(original.Symbols); EditHandler = original.EditHandler; - Start = original.Start; + _start = original.Start; ChunkGenerator = original.ChunkGenerator; + + _symbols = new List(original.Symbols); + _tracker = new SourceLocationTracker(original.Start); } - public SpanBuilder() + public SpanBuilder(SourceLocation location) { + _tracker = new SourceLocationTracker(); + Reset(); + + Start = location; } public ISpanChunkGenerator ChunkGenerator { get; set; } - public SourceLocation Start { get; set; } + public SourceLocation Start + { + get { return _start; } + set + { + _start = value; + _tracker.CurrentLocation = value; + } + } + + public SourceLocation End => _tracker.CurrentLocation; public SpanKind Kind { get; set; } @@ -51,15 +69,24 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Need to potentially allocate a new list because Span.ReplaceWith takes ownership // of the original list. _symbols = null; + _symbols = new List(); EditHandler = SpanEditHandler.CreateDefault((content) => Enumerable.Empty()); ChunkGenerator = SpanChunkGenerator.Null; - Start = SourceLocation.Zero; + Start = SourceLocation.Undefined; } public Span Build() { - return new Span(this); + var span = new Span(this); + + for (var i = 0; i < span.Symbols.Count; i++) + { + var symbol = span.Symbols[i]; + symbol.Parent = span; + } + + return span; } public void ClearSymbols() @@ -74,15 +101,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return; } - if (Symbols.Count == 0) + if (Start.Equals(SourceLocation.Undefined)) { - Start = symbol.Start; - symbol.ChangeStart(SourceLocation.Zero); - _tracker.CurrentLocation = SourceLocation.Zero; - } - else - { - symbol.ChangeStart(_tracker.CurrentLocation); + throw new InvalidOperationException("SpanBuilder must have a valid location"); } _symbols.Add(symbol); diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanEditHandler.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanEditHandler.cs index 0dd052c059..2af2952864 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanEditHandler.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SpanEditHandler.cs @@ -72,7 +72,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy newSpan.ClearSymbols(); foreach (ISymbol sym in Tokenizer(newContent)) { - sym.OffsetStart(target.Start); newSpan.Accept(sym); } if (target.Next != null) diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolBase.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolBase.cs index be0d330760..180b163852 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolBase.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolBase.cs @@ -11,7 +11,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy internal abstract class SymbolBase : ISymbol where TType : struct { protected SymbolBase( - SourceLocation start, string content, TType type, IReadOnlyList errors) @@ -21,13 +20,12 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy throw new ArgumentNullException(nameof(content)); } - Start = start; Content = content; Type = type; Errors = errors; } - public SourceLocation Start { get; private set; } + public Span Parent { get; set; } public IReadOnlyList Errors { get; } @@ -35,11 +33,35 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public TType Type { get; } + public SourceLocation Start + { + get + { + if (Parent == null) + { + return SourceLocation.Undefined; + } + + var tracker = new SourceLocationTracker(Parent.Start); + for (var i = 0; i < Parent.Symbols.Count; i++) + { + var symbol = Parent.Symbols[i]; + if (object.ReferenceEquals(this, symbol)) + { + break; + } + + tracker.UpdateLocation(symbol.Content); + } + + return tracker.CurrentLocation; + } + } + public override bool Equals(object obj) { var other = obj as SymbolBase; return other != null && - Start.Equals(other.Start) && string.Equals(Content, other.Content, StringComparison.Ordinal) && Type.Equals(other.Type); } @@ -47,26 +69,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public override int GetHashCode() { // Hash code should include only immutable properties. - var hashCodeCombiner = HashCodeCombiner.Start(); - hashCodeCombiner.Add(Content, StringComparer.Ordinal); - hashCodeCombiner.Add(Type); + var hash = HashCodeCombiner.Start(); + hash.Add(Content, StringComparer.Ordinal); + hash.Add(Type); - return hashCodeCombiner; + return hash; } public override string ToString() { - return string.Format(CultureInfo.InvariantCulture, "{0} {1} - [{2}]", Start, Type, Content); - } - - public void OffsetStart(SourceLocation documentStart) - { - Start = documentStart + Start; - } - - public void ChangeStart(SourceLocation newStart) - { - Start = newStart; + return string.Format(CultureInfo.InvariantCulture, "{0} [{1}]", Type, Content); } } } diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolExtensions.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolExtensions.cs deleted file mode 100644 index a8b6db9fc1..0000000000 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SymbolExtensions.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.AspNetCore.Razor.Evolution.Legacy -{ - internal static class SymbolExtensions - { - public static LocationTagged GetContent(this SpanBuilder span) - { - return GetContent(span, e => e); - } - - public static LocationTagged GetContent(this SpanBuilder span, Func, IEnumerable> filter) - { - return GetContent(filter(span.Symbols), span.Start); - } - - public static LocationTagged GetContent(this IEnumerable symbols, SourceLocation spanStart) - { - if (symbols.Any()) - { - return new LocationTagged(string.Concat(symbols.Select(s => s.Content)), spanStart + symbols.First().Start); - } - else - { - return new LocationTagged(string.Empty, spanStart); - } - } - - public static LocationTagged GetContent(this ISymbol symbol) - { - return new LocationTagged(symbol.Content, symbol.Start); - } - } -} diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TagHelperBlockRewriter.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TagHelperBlockRewriter.cs index 9e43f7c2cb..3f28433f08 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TagHelperBlockRewriter.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TagHelperBlockRewriter.cs @@ -149,7 +149,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy ErrorSink errorSink) { var afterEquals = false; - var builder = new SpanBuilder + var builder = new SpanBuilder(span.Start) { ChunkGenerator = span.ChunkGenerator, EditHandler = span.EditHandler, @@ -190,7 +190,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { capturedAttributeValueStart = true; - attributeValueStartLocation = span.Start + symbol.Start; + attributeValueStartLocation = symbol.Start; } builder.Accept(symbol); @@ -260,7 +260,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } attributeValueStartLocation = - span.Start + symbolStartLocation + new SourceLocation(absoluteIndex: 1, lineIndex: 0, characterIndex: 1); @@ -280,7 +279,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } // After all symbols have been added we need to set the builders start position so we do not indirectly - // modify each symbol's Start location. + // modify the span's start location. builder.Start = attributeValueStartLocation; if (name == null) @@ -613,7 +612,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy .OfType() .First(sym => sym.Type != HtmlSymbolType.WhiteSpace && sym.Type != HtmlSymbolType.NewLine); - return nodeStart + firstNonWhitespaceSymbol.Start; + return firstNonWhitespaceSymbol.Start; } private static Span CreateMarkupAttribute(SpanBuilder builder, bool isBoundNonStringAttribute) diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/Tokenizer.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/Tokenizer.cs index 95fdced87d..21a56c985d 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/Tokenizer.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/Tokenizer.cs @@ -60,14 +60,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } } - protected SourceLocation CurrentLocation - { - get { return Source.Location; } - } + public SourceLocation CurrentLocation => Source.Location; - protected SourceLocation CurrentStart { get; private set; } + public SourceLocation CurrentStart { get; private set; } - protected abstract TSymbol CreateSymbol(SourceLocation start, string content, TSymbolType type, IReadOnlyList errors); + protected abstract TSymbol CreateSymbol(string content, TSymbolType type, IReadOnlyList errors); protected abstract StateResult Dispatch(); @@ -92,6 +89,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy // Post-Condition: Buffer should be empty at the end of Next() Debug.Assert(Buffer.Length == 0); + // Post-Condition: Token should be non-zero length unless we're at EOF. + Debug.Assert(EndOfFile || !CurrentStart.Equals(CurrentLocation)); + return symbol; } @@ -193,19 +193,15 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy protected void StartSymbol() { - Buffer.Clear(); + Debug.Assert(Buffer.Length == 0); + Debug.Assert(CurrentErrors.Count == 0); + CurrentStart = CurrentLocation; - CurrentErrors.Clear(); } protected TSymbol EndSymbol(TSymbolType type) { - return EndSymbol(CurrentStart, type); - } - - protected TSymbol EndSymbol(SourceLocation start, TSymbolType type) - { - TSymbol sym = null; + TSymbol symbol = null; if (HaveContent) { // Perf: Don't allocate a new errors array unless necessary. @@ -215,10 +211,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy errors[i] = CurrentErrors[i]; } - sym = CreateSymbol(start, GetSymbolContent(type), type, errors); + symbol = CreateSymbol(GetSymbolContent(type), type, errors); + + Buffer.Clear(); + CurrentErrors.Clear(); } - StartSymbol(); - return sym; + + return symbol; } protected virtual string GetSymbolContent(TSymbolType type) diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerBackedParser.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerBackedParser.cs index cf24afe40b..afb9f07d82 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerBackedParser.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerBackedParser.cs @@ -18,10 +18,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy protected TokenizerBackedParser(LanguageCharacteristics language, ParserContext context) : base(context) { - Span = new SpanBuilder(); Language = language; + var languageTokenizer = Language.CreateTokenizer(Context.Source); _tokenizer = new TokenizerView(languageTokenizer); + Span = new SpanBuilder(CurrentLocation); } protected SpanBuilder Span { get; private set; } @@ -35,10 +36,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy protected TSymbol PreviousSymbol { get; private set; } - protected SourceLocation CurrentLocation - { - get { return (EndOfFile || CurrentSymbol == null) ? Context.Source.Location : CurrentSymbol.Start; } - } + protected SourceLocation CurrentLocation => _tokenizer.Tokenizer.CurrentLocation; + + protected SourceLocation CurrentStart => _tokenizer.Tokenizer.CurrentStart; protected bool EndOfFile { @@ -163,7 +163,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { var left = CurrentSymbol.Type; var right = Language.FlipBracket(left); - var start = CurrentLocation; + var start = CurrentStart; AcceptAndMoveNext(); if (EndOfFile && ((mode & BalancingModes.NoErrorOnFailure) != BalancingModes.NoErrorOnFailure)) { @@ -180,7 +180,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start) { - var startPosition = CurrentLocation.AbsoluteIndex; + var startPosition = CurrentStart.AbsoluteIndex; var nesting = 1; if (!EndOfFile) { @@ -196,7 +196,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy HandleEmbeddedTransition(); // Reset backtracking since we've already outputted some spans. - startPosition = CurrentLocation.AbsoluteIndex; + startPosition = CurrentStart.AbsoluteIndex; } if (At(left)) { @@ -328,15 +328,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } protected internal void AddMarkerSymbolIfNecessary() - { - AddMarkerSymbolIfNecessary(CurrentLocation); - } - - protected internal void AddMarkerSymbolIfNecessary(SourceLocation location) { if (Span.Symbols.Count == 0 && Context.Builder.LastAcceptedCharacters != AcceptedCharacters.Any) { - Accept(Language.CreateMarkerSymbol(location)); + Accept(Language.CreateMarkerSymbol()); } } @@ -362,9 +357,18 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { if (Span.Symbols.Count > 0) { + var nextStart = Span.End; + var builtSpan = Span.Build(); Context.Builder.Add(builtSpan); Initialize(Span); + + // Ensure spans are contiguous. + // + // Note: Using Span.End here to avoid CurrentLocation. CurrentLocation will + // vary depending on what tokens have been read. We often read a token and *then* + // make a decision about whether to include it in the current span. + Span.Start = nextStart; } } @@ -464,7 +468,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy errorLength = Math.Max(CurrentSymbol.Content.Length, 1); } - Context.ErrorSink.OnError(CurrentLocation, errorBase(error), errorLength); + Context.ErrorSink.OnError(CurrentStart, errorBase(error), errorLength); } return found; } @@ -475,6 +479,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { return NextToken(); } + return true; } @@ -612,7 +617,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy using (Context.Builder.StartBlock(BlockType.Comment)) { Context.Builder.CurrentBlock.ChunkGenerator = new RazorCommentChunkGenerator(); - var start = CurrentLocation; + var start = CurrentStart; Expected(KnownSymbolType.CommentStart); Output(SpanKind.Transition, AcceptedCharacters.None); diff --git a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerView.cs b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerView.cs index 12f806c8ad..eaded56ec5 100644 --- a/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerView.cs +++ b/src/Microsoft.AspNetCore.Razor.Evolution/Legacy/TokenizerView.cs @@ -34,15 +34,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public void PutBack(TSymbol symbol) { - Debug.Assert(Source.Position == symbol.Start.AbsoluteIndex + symbol.Content.Length); - if (Source.Position != symbol.Start.AbsoluteIndex + symbol.Content.Length) - { - // We've already passed this symbol - throw new InvalidOperationException( - LegacyResources.FormatTokenizerView_CannotPutBack( - symbol.Start.AbsoluteIndex + symbol.Content.Length, - Source.Position)); - } Source.Position -= symbol.Content.Length; Current = null; EndOfFile = Source.Position >= Source.Length; diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/BlockTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/BlockTest.cs index 423f50fa71..ec35e5d440 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/BlockTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/BlockTest.cs @@ -13,7 +13,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { // Arrange var builder = new BlockBuilder() { Type = BlockType.Comment }; - var span = new SpanBuilder() { Kind = SpanKind.Code }.Build(); + var span = new SpanBuilder(SourceLocation.Undefined) { Kind = SpanKind.Code }.Build(); builder.Children.Add(span); // Act @@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public void ConstructorTransfersChildrenFromBlockBuilder() { // Arrange - var expected = new SpanBuilder() { Kind = SpanKind.Code }.Build(); + var expected = new SpanBuilder(SourceLocation.Undefined) { Kind = SpanKind.Code }.Build(); var builder = new BlockBuilder() { Type = BlockType.Functions diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpAutoCompleteTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpAutoCompleteTest.cs index 8741508333..b3477f810c 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpAutoCompleteTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpAutoCompleteTest.cs @@ -121,7 +121,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.Markup("

").Accepts(AcceptedCharacters.None)), new MarkupTagBlock( Factory.Markup("

").Accepts(AcceptedCharacters.None))), - Factory.Span(SpanKind.Code, new CSharpSymbol(Factory.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown)) + Factory.Span(SpanKind.Code, new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown)) .With(new StatementChunkGenerator()) ), new RazorError( diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpRazorCommentsTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpRazorCommentsTest.cs index f83f61df05..7262353f34 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpRazorCommentsTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpRazorCommentsTest.cs @@ -19,10 +19,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy .Accepts(AcceptedCharacters.None), Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), - Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, - string.Empty, - HtmlSymbolType.Unknown)) + Factory.Span( + SpanKind.Comment, + new HtmlSymbol( + string.Empty, + HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any))), new RazorError( LegacyResources.ParseError_RazorComment_Not_Terminated, @@ -42,7 +43,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -70,7 +70,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new CSharpSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -102,7 +101,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new CSharpSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any)))), @@ -142,7 +140,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -183,9 +180,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy .Accepts(AcceptedCharacters.None), Factory.MetaCode("*", CSharpSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), - Factory.Span(SpanKind.Comment, new CSharpSymbol(Factory.LocationTracker.CurrentLocation, - string.Empty, - CSharpSymbolType.Unknown)) + Factory.Span(SpanKind.Comment, new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any)))), new RazorError( LegacyResources.ParseError_RazorComment_Not_Terminated, @@ -215,7 +210,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -248,7 +242,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -263,7 +256,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -294,7 +286,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -310,7 +301,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) .Accepts(AcceptedCharacters.Any), @@ -341,7 +331,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, " hello ", HtmlSymbolType.RazorComment)) .Accepts(AcceptedCharacters.Any), @@ -356,7 +345,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, " world ", HtmlSymbolType.RazorComment)) .Accepts(AcceptedCharacters.Any), @@ -390,7 +378,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, " content ", HtmlSymbolType.RazorComment)) .Accepts(AcceptedCharacters.Any), @@ -405,7 +392,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy Factory.MetaMarkup("*", HtmlSymbolType.RazorCommentStar) .Accepts(AcceptedCharacters.None), Factory.Span(SpanKind.Comment, new HtmlSymbol( - Factory.LocationTracker.CurrentLocation, Environment.NewLine + "content" + Environment.NewLine, HtmlSymbolType.RazorComment)) .Accepts(AcceptedCharacters.Any), diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerCommentTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerCommentTest.cs index 2eb353b3a7..3fe23a9741 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerCommentTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerCommentTest.cs @@ -12,80 +12,83 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Next_Ignores_Star_At_EOF_In_RazorComment() { - TestTokenizer("@* Foo * Bar * Baz *", - new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition), - new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar), - new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz *", CSharpSymbolType.RazorComment)); + TestTokenizer( + "@* Foo * Bar * Baz *", + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol(" Foo * Bar * Baz *", CSharpSymbolType.RazorComment)); } [Fact] public void Next_Ignores_Star_Without_Trailing_At() { - TestTokenizer("@* Foo * Bar * Baz *@", - new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition), - new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar), - new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz ", CSharpSymbolType.RazorComment), - new CSharpSymbol(19, 0, 19, "*", CSharpSymbolType.RazorCommentStar), - new CSharpSymbol(20, 0, 20, "@", CSharpSymbolType.RazorCommentTransition)); + TestTokenizer( + "@* Foo * Bar * Baz *@", + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol(" Foo * Bar * Baz ", CSharpSymbolType.RazorComment), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition)); } [Fact] public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment() { - TestTokenizer("@* Foo Bar Baz *@", - new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition), - new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar), - new CSharpSymbol(2, 0, 2, " Foo Bar Baz ", CSharpSymbolType.RazorComment), - new CSharpSymbol(15, 0, 15, "*", CSharpSymbolType.RazorCommentStar), - new CSharpSymbol(16, 0, 16, "@", CSharpSymbolType.RazorCommentTransition)); + TestTokenizer( + "@* Foo Bar Baz *@", + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol(" Foo Bar Baz ", CSharpSymbolType.RazorComment), + new CSharpSymbol("*", CSharpSymbolType.RazorCommentStar), + new CSharpSymbol("@", CSharpSymbolType.RazorCommentTransition)); } [Fact] public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment() { - TestTokenizer("// Foo Bar Baz", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment)); + TestTokenizer("// Foo Bar Baz", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment)); } [Fact] public void Single_Line_Comment_Is_Terminated_By_Newline() { - TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining); + TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol("// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining); } [Fact] public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect() { - TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol(0, 0, 0, "// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining); + TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol("// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining); } [Fact] public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment() { - TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment)); + TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment)); } [Fact] public void Multi_Line_Comment_Is_Terminated_By_End_Sequence() { - TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining); + TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol("/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining); } [Fact] public void Unterminated_Multi_Line_Comment_Captures_To_EOF() { - TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining); + TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol("/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining); } [Fact] public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence() { - TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol(0, 0, 0, "/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining); + TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol("/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining); } [Fact] public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence() { - TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining); + TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol("/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining); } } } diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerIdentifierTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerIdentifierTest.cs index 03327b58ee..fb60cba01d 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerIdentifierTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerIdentifierTest.cs @@ -10,73 +10,73 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Simple_Identifier_Is_Recognized() { - TestTokenizer("foo", new CSharpSymbol(0, 0, 0, "foo", CSharpSymbolType.Identifier)); + TestTokenizer("foo", new CSharpSymbol("foo", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Starting_With_Underscore_Is_Recognized() { - TestTokenizer("_foo", new CSharpSymbol(0, 0, 0, "_foo", CSharpSymbolType.Identifier)); + TestTokenizer("_foo", new CSharpSymbol("_foo", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Contain_Digits() { - TestTokenizer("foo4", new CSharpSymbol(0, 0, 0, "foo4", CSharpSymbolType.Identifier)); + TestTokenizer("foo4", new CSharpSymbol("foo4", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Start_With_Titlecase_Letter() { - TestTokenizer("ῼfoo", new CSharpSymbol(0, 0, 0, "ῼfoo", CSharpSymbolType.Identifier)); + TestTokenizer("ῼfoo", new CSharpSymbol("ῼfoo", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Start_With_Letter_Modifier() { - TestTokenizer("ᵊfoo", new CSharpSymbol(0, 0, 0, "ᵊfoo", CSharpSymbolType.Identifier)); + TestTokenizer("ᵊfoo", new CSharpSymbol("ᵊfoo", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Start_With_Other_Letter() { - TestTokenizer("ƻfoo", new CSharpSymbol(0, 0, 0, "ƻfoo", CSharpSymbolType.Identifier)); + TestTokenizer("ƻfoo", new CSharpSymbol("ƻfoo", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Start_With_Number_Letter() { - TestTokenizer("Ⅽool", new CSharpSymbol(0, 0, 0, "Ⅽool", CSharpSymbolType.Identifier)); + TestTokenizer("Ⅽool", new CSharpSymbol("Ⅽool", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Contain_Non_Spacing_Mark() { - TestTokenizer("foo\u0300", new CSharpSymbol(0, 0, 0, "foo\u0300", CSharpSymbolType.Identifier)); + TestTokenizer("foo\u0300", new CSharpSymbol("foo\u0300", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Contain_Spacing_Combining_Mark() { - TestTokenizer("fooः", new CSharpSymbol(0, 0, 0, "fooः", CSharpSymbolType.Identifier)); + TestTokenizer("fooः", new CSharpSymbol("fooः", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Contain_Non_English_Digit() { - TestTokenizer("foo١", new CSharpSymbol(0, 0, 0, "foo١", CSharpSymbolType.Identifier)); + TestTokenizer("foo١", new CSharpSymbol("foo١", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Contain_Connector_Punctuation() { - TestTokenizer("foo‿bar", new CSharpSymbol(0, 0, 0, "foo‿bar", CSharpSymbolType.Identifier)); + TestTokenizer("foo‿bar", new CSharpSymbol("foo‿bar", CSharpSymbolType.Identifier)); } [Fact] public void Identifier_Can_Contain_Format_Character() { - TestTokenizer("foo؃bar", new CSharpSymbol(0, 0, 0, "foo؃bar", CSharpSymbolType.Identifier)); + TestTokenizer("foo؃bar", new CSharpSymbol("foo؃bar", CSharpSymbolType.Identifier)); } [Fact] @@ -164,7 +164,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private void TestKeyword(string keyword, CSharpKeyword keywordType) { - TestTokenizer(keyword, new CSharpSymbol(0, 0, 0, keyword, CSharpSymbolType.Keyword) { Keyword = keywordType }); + TestTokenizer(keyword, new CSharpSymbol(keyword, CSharpSymbolType.Keyword) { Keyword = keywordType }); } } } diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerLiteralTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerLiteralTest.cs index b40d88a73e..4863a4f6c6 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerLiteralTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerLiteralTest.cs @@ -41,7 +41,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix() { - TestTokenizer("42a", new CSharpSymbol(0, 0, 0, "42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); + TestTokenizer("42a", new CSharpSymbol("42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); } [Fact] @@ -75,25 +75,25 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix() { - TestTokenizer("0xDEADBEEFz", new CSharpSymbol(0, 0, 0, "0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); + TestTokenizer("0xDEADBEEFz", new CSharpSymbol("0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); } [Fact] public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal() { - TestTokenizer("3.a", new CSharpSymbol(0, 0, 0, "3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); + TestTokenizer("3.a", new CSharpSymbol("3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining); } [Fact] public void Simple_Real_Literal_Is_Recognized() { - TestTokenizer("3.14159", new CSharpSymbol(0, 0, 0, "3.14159", CSharpSymbolType.RealLiteral)); + TestTokenizer("3.14159", new CSharpSymbol("3.14159", CSharpSymbolType.RealLiteral)); } [Fact] public void Real_Literal_Between_Zero_And_One_Is_Recognized() { - TestTokenizer(".14159", new CSharpSymbol(0, 0, 0, ".14159", CSharpSymbolType.RealLiteral)); + TestTokenizer(".14159", new CSharpSymbol(".14159", CSharpSymbolType.RealLiteral)); } [Fact] @@ -173,37 +173,37 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated() { - TestTokenizer("'foo\n", new CSharpSymbol(0, 0, 0, "'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); + TestTokenizer("'foo\n", new CSharpSymbol("'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); } [Fact] public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() { - TestTokenizer("'foo\\\n", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); + TestTokenizer("'foo\\\n", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); } [Fact] public void Character_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() { - TestTokenizer("'foo\\\nflarg", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); + TestTokenizer("'foo\\\nflarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); } [Fact] public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() { - TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); + TestTokenizer("'foo\\" + Environment.NewLine, new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); } [Fact] public void Character_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() { - TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol(0, 0, 0, "'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); + TestTokenizer($"'foo\\{Environment.NewLine}flarg", new CSharpSymbol("'foo\\", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); } [Fact] public void Character_Literal_Allows_Escaped_Escape() { - TestTokenizer("'foo\\\\'blah", new CSharpSymbol(0, 0, 0, "'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); + TestTokenizer("'foo\\\\'blah", new CSharpSymbol("'foo\\\\'", CSharpSymbolType.CharacterLiteral), IgnoreRemaining); } [Fact] @@ -227,37 +227,37 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void String_Literal_Is_Terminated_By_EOL_If_Unterminated() { - TestTokenizer("\"foo\n", new CSharpSymbol(0, 0, 0, "\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer("\"foo\n", new CSharpSymbol("\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash() { - TestTokenizer("\"foo\\\n", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer("\"foo\\\n", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] public void String_Literal_Terminated_By_EOL_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() { - TestTokenizer("\"foo\\\nflarg", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer("\"foo\\\nflarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash() { - TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer("\"foo\\" + Environment.NewLine, new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] public void String_Literal_Terminated_By_CRLF_Even_When_Last_Char_Is_Slash_And_Followed_By_Stuff() { - TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol(0, 0, 0, "\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer($"\"foo\\{Environment.NewLine}flarg", new CSharpSymbol("\"foo\\", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] public void String_Literal_Allows_Escaped_Escape() { - TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol(0, 0, 0, "\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer("\"foo\\\\\"blah", new CSharpSymbol("\"foo\\\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] @@ -275,7 +275,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote() { - TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol(0, 0, 0, "@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); + TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol("@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining); } [Fact] diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerOperatorsTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerOperatorsTest.cs index 477bf26c06..69482a8e53 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerOperatorsTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerOperatorsTest.cs @@ -125,8 +125,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public void LeftShift_Is_Not_Specially_Recognized() { TestTokenizer("<<", - new CSharpSymbol(new SourceLocation(0, 0, 0), "<", CSharpSymbolType.LessThan), - new CSharpSymbol(new SourceLocation(1, 0, 1), "<", CSharpSymbolType.LessThan)); + new CSharpSymbol("<", CSharpSymbolType.LessThan), + new CSharpSymbol("<", CSharpSymbolType.LessThan)); } [Fact] @@ -283,8 +283,8 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public void RightShift_Is_Not_Specially_Recognized() { TestTokenizer(">>", - new CSharpSymbol(0, 0, 0, ">", CSharpSymbolType.GreaterThan), - new CSharpSymbol(1, 0, 1, ">", CSharpSymbolType.GreaterThan)); + new CSharpSymbol(">", CSharpSymbolType.GreaterThan), + new CSharpSymbol(">", CSharpSymbolType.GreaterThan)); } [Fact] diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTest.cs index 49b321d2df..5902686123 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTest.cs @@ -18,67 +18,74 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Next_Returns_Newline_Token_For_Single_CR() { - TestTokenizer("\r\ra", - new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine), - new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine), - IgnoreRemaining); + TestTokenizer( + "\r\ra", + new CSharpSymbol("\r", CSharpSymbolType.NewLine), + new CSharpSymbol("\r", CSharpSymbolType.NewLine), + IgnoreRemaining); } [Fact] public void Next_Returns_Newline_Token_For_Single_LF() { - TestTokenizer("\n\na", - new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine), - new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine), - IgnoreRemaining); + TestTokenizer( + "\n\na", + new CSharpSymbol("\n", CSharpSymbolType.NewLine), + new CSharpSymbol("\n", CSharpSymbolType.NewLine), + IgnoreRemaining); } [Fact] public void Next_Returns_Newline_Token_For_Single_NEL() { // NEL: Unicode "Next Line" U+0085 - TestTokenizer("\u0085\u0085a", - new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine), - new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine), - IgnoreRemaining); + TestTokenizer( + "\u0085\u0085a", + new CSharpSymbol("\u0085", CSharpSymbolType.NewLine), + new CSharpSymbol("\u0085", CSharpSymbolType.NewLine), + IgnoreRemaining); } [Fact] public void Next_Returns_Newline_Token_For_Single_Line_Separator() { // Unicode "Line Separator" U+2028 - TestTokenizer("\u2028\u2028a", - new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine), - new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine), - IgnoreRemaining); + TestTokenizer( + "\u2028\u2028a", + new CSharpSymbol("\u2028", CSharpSymbolType.NewLine), + new CSharpSymbol("\u2028", CSharpSymbolType.NewLine), + IgnoreRemaining); } [Fact] public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator() { // Unicode "Paragraph Separator" U+2029 - TestTokenizer("\u2029\u2029a", - new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine), - new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine), - IgnoreRemaining); + TestTokenizer( + "\u2029\u2029a", + new CSharpSymbol("\u2029", CSharpSymbolType.NewLine), + new CSharpSymbol("\u2029", CSharpSymbolType.NewLine), + IgnoreRemaining); } [Fact] public void Next_Returns_Single_Newline_Token_For_CRLF() { - TestTokenizer("\r\n\r\na", - new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine), - new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine), - IgnoreRemaining); + TestTokenizer( + "\r\n\r\na", + new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), + new CSharpSymbol("\r\n", CSharpSymbolType.NewLine), + IgnoreRemaining); } [Fact] public void Next_Returns_Token_For_Whitespace_Characters() { - TestTokenizer(" \f\t\u000B \n ", - new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace), - new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine), - new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace)); + TestTokenizer( + " \f\t\u000B \n ", + new CSharpSymbol(" \f\t\u000B ", CSharpSymbolType.WhiteSpace), + new CSharpSymbol("\n", CSharpSymbolType.NewLine), + new CSharpSymbol(" ", CSharpSymbolType.WhiteSpace)); } [Fact] @@ -90,9 +97,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Transition_Is_Recognized_As_SingleCharacter() { - TestTokenizer("@(", - new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition), - new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis)); + TestTokenizer( + "@(", + new CSharpSymbol("@", CSharpSymbolType.Transition), + new CSharpSymbol("(", CSharpSymbolType.LeftParenthesis)); } } } diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTestBase.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTestBase.cs index 913bda63f7..213384a313 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTestBase.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/CSharpTokenizerTestBase.cs @@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { public abstract class CSharpTokenizerTestBase : TokenizerTestBase { - private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(0, 0, 0, string.Empty, CSharpSymbolType.Unknown); + private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown); internal override object IgnoreRemaining { @@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy internal void TestSingleToken(string text, CSharpSymbolType expectedSymbolType) { - TestTokenizer(text, new CSharpSymbol(0, 0, 0, text, expectedSymbolType)); + TestTokenizer(text, new CSharpSymbol(text, expectedSymbolType)); } internal void TestTokenizer(string input, params CSharpSymbol[] expectedSymbols) diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTest.cs index 67b1c3e292..75a436c294 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTest.cs @@ -17,23 +17,23 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public void Text_Is_Recognized() { TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384", - new HtmlSymbol(0, 0, 0, "foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text)); + new HtmlSymbol("foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text)); } [Fact] public void Whitespace_Is_Recognized() { TestTokenizer(" \t\f ", - new HtmlSymbol(0, 0, 0, " \t\f ", HtmlSymbolType.WhiteSpace)); + new HtmlSymbol(" \t\f ", HtmlSymbolType.WhiteSpace)); } [Fact] public void Newline_Is_Recognized() { TestTokenizer("\n\r\r\n", - new HtmlSymbol(0, 0, 0, "\n", HtmlSymbolType.NewLine), - new HtmlSymbol(1, 1, 0, "\r", HtmlSymbolType.NewLine), - new HtmlSymbol(2, 2, 0, "\r\n", HtmlSymbolType.NewLine)); + new HtmlSymbol("\n", HtmlSymbolType.NewLine), + new HtmlSymbol("\r", HtmlSymbolType.NewLine), + new HtmlSymbol("\r\n", HtmlSymbolType.NewLine)); } [Fact] @@ -129,32 +129,35 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy [Fact] public void Next_Ignores_Star_At_EOF_In_RazorComment() { - TestTokenizer("@* Foo * Bar * Baz *", - new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition), - new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar), - new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz *", HtmlSymbolType.RazorComment)); + TestTokenizer( + "@* Foo * Bar * Baz *", + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol(" Foo * Bar * Baz *", HtmlSymbolType.RazorComment)); } [Fact] public void Next_Ignores_Star_Without_Trailing_At() { - TestTokenizer("@* Foo * Bar * Baz *@", - new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition), - new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar), - new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz ", HtmlSymbolType.RazorComment), - new HtmlSymbol(19, 0, 19, "*", HtmlSymbolType.RazorCommentStar), - new HtmlSymbol(20, 0, 20, "@", HtmlSymbolType.RazorCommentTransition)); + TestTokenizer( + "@* Foo * Bar * Baz *@", + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol(" Foo * Bar * Baz ", HtmlSymbolType.RazorComment), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition)); } [Fact] public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment() { - TestTokenizer("@* Foo Bar Baz *@", - new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition), - new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar), - new HtmlSymbol(2, 0, 2, " Foo Bar Baz ", HtmlSymbolType.RazorComment), - new HtmlSymbol(15, 0, 15, "*", HtmlSymbolType.RazorCommentStar), - new HtmlSymbol(16, 0, 16, "@", HtmlSymbolType.RazorCommentTransition)); + TestTokenizer( + "@* Foo Bar Baz *@", + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol(" Foo Bar Baz ", HtmlSymbolType.RazorComment), + new HtmlSymbol("*", HtmlSymbolType.RazorCommentStar), + new HtmlSymbol("@", HtmlSymbolType.RazorCommentTransition)); } } } diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTestBase.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTestBase.cs index fcc30a8749..13c8c8ff01 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTestBase.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/HtmlTokenizerTestBase.cs @@ -5,7 +5,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { public abstract class HtmlTokenizerTestBase : TokenizerTestBase { - private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(0, 0, 0, string.Empty, HtmlSymbolType.Unknown); + private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown); internal override object IgnoreRemaining { @@ -19,7 +19,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy internal void TestSingleToken(string text, HtmlSymbolType expectedSymbolType) { - TestTokenizer(text, new HtmlSymbol(0, 0, 0, text, expectedSymbolType)); + TestTokenizer(text, new HtmlSymbol(text, expectedSymbolType)); } internal void TestTokenizer(string input, params HtmlSymbol[] expectedSymbols) diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RawTextSymbol.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RawTextSymbol.cs index b0ab640a46..deabe0a66c 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RawTextSymbol.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RawTextSymbol.cs @@ -1,21 +1,24 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Globalization; namespace Microsoft.AspNetCore.Razor.Evolution.Legacy { internal class RawTextSymbol : ISymbol { - public SourceLocation Start { get; private set; } - public string Content { get; } - public RawTextSymbol(SourceLocation start, string content) { Start = start; Content = content; } + public SourceLocation Start { get; private set; } + public string Content { get; } + public Span Parent { get; set; } + public int Offset { get; set; } + public override bool Equals(object obj) { var other = obj as RawTextSymbol; diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TestSpanBuilder.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TestSpanBuilder.cs index 63767e4d62..5775fd227d 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TestSpanBuilder.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TestSpanBuilder.cs @@ -14,7 +14,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return new UnclassifiedCodeSpanConstructor( self.Span( SpanKind.Code, - new CSharpSymbol(self.LocationTracker.CurrentLocation, string.Empty, CSharpSymbolType.Unknown))); + new CSharpSymbol(string.Empty, CSharpSymbolType.Unknown))); } public static SpanConstructor EmptyHtml(this SpanFactory self) @@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return self .Span( SpanKind.Markup, - new HtmlSymbol(self.LocationTracker.CurrentLocation, string.Empty, HtmlSymbolType.Unknown)) + new HtmlSymbol(string.Empty, HtmlSymbolType.Unknown)) .With(new MarkupChunkGenerator()); } @@ -178,34 +178,41 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy public SpanConstructor Span(SpanKind kind, string content, CSharpSymbolType type) { - return CreateSymbolSpan(kind, content, st => new CSharpSymbol(st, content, type)); + return CreateSymbolSpan(kind, content, () => new CSharpSymbol(content, type)); } public SpanConstructor Span(SpanKind kind, string content, HtmlSymbolType type) { - return CreateSymbolSpan(kind, content, st => new HtmlSymbol(st, content, type)); + return CreateSymbolSpan(kind, content, () => new HtmlSymbol(content, type)); } public SpanConstructor Span(SpanKind kind, string content, bool markup) { - return new SpanConstructor(kind, Tokenize(new[] { content }, markup)); + return new SpanConstructor(kind, LocationTracker.CurrentLocation, Tokenize(new[] { content }, markup)); } public SpanConstructor Span(SpanKind kind, string[] content, bool markup) { - return new SpanConstructor(kind, Tokenize(content, markup)); + return new SpanConstructor(kind, LocationTracker.CurrentLocation, Tokenize(content, markup)); } public SpanConstructor Span(SpanKind kind, params ISymbol[] symbols) { - return new SpanConstructor(kind, symbols); + var start = LocationTracker.CurrentLocation; + foreach (var symbol in symbols) + { + LocationTracker.UpdateLocation(symbol.Content); + } + + return new SpanConstructor(kind, start, symbols); } - private SpanConstructor CreateSymbolSpan(SpanKind kind, string content, Func ctor) + private SpanConstructor CreateSymbolSpan(SpanKind kind, string content, Func ctor) { var start = LocationTracker.CurrentLocation; LocationTracker.UpdateLocation(content); - return new SpanConstructor(kind, new[] { ctor(start) }); + + return new SpanConstructor(kind, start, new[] { ctor() }); } public void Reset() @@ -220,15 +227,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy private IEnumerable Tokenize(string content, bool markup) { - var tok = MakeTokenizer(markup, new SeekableTextReader(content)); - ISymbol sym; + var tokenizer = MakeTokenizer(markup, new SeekableTextReader(content)); + ISymbol symbol; ISymbol last = null; - while ((sym = tok.NextSymbol()) != null) + + while ((symbol = tokenizer.NextSymbol()) != null) { - OffsetStart(sym, LocationTracker.CurrentLocation); - last = sym; - yield return sym; + last = symbol; + yield return symbol; } + LocationTracker.UpdateLocation(content); } @@ -243,11 +251,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy return CodeTokenizerFactory(seekableTextReader); } } - - private void OffsetStart(ISymbol sym, SourceLocation sourceLocation) - { - sym.OffsetStart(sourceLocation); - } } internal static class SpanConstructorExtensions @@ -359,9 +362,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy yield return new RawTextSymbol(SourceLocation.Zero, str); } - public SpanConstructor(SpanKind kind, IEnumerable symbols) + public SpanConstructor(SpanKind kind, SourceLocation location, IEnumerable symbols) { - Builder = new SpanBuilder(); + Builder = new SpanBuilder(location); Builder.Kind = kind; Builder.EditHandler = SpanEditHandler.CreateDefault((content) => SpanConstructor.TestTokenizer(content)); foreach (ISymbol sym in symbols) diff --git a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs index 38364ea4d0..7008606c66 100644 --- a/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs +++ b/test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs @@ -104,7 +104,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy } protected override CSharpSymbol CreateSymbol( - SourceLocation start, string content, CSharpSymbolType type, IReadOnlyList errors)