Add indexability to the RazorSourceDocument.

- Removed the `CreateReader` API in favor of a `CopyTo`, `Length`, `[index]` and `Encoding`.
- Updated existing APIs to react to the change.
- Added tests.

src/Microsoft.AspNetCore.Razor.Evolution/DefaultRazorSourceDocument.cs

@@ -2,38 +2,67 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.IO;
 using System.Text;
 
 namespace Microsoft.AspNetCore.Razor.Evolution
 {
     internal class DefaultRazorSourceDocument : RazorSourceDocument
     {
-        private MemoryStream _stream;
+        private readonly string _content;
 
-        public DefaultRazorSourceDocument(MemoryStream stream, Encoding encoding, string filename)
+        public DefaultRazorSourceDocument(string content, Encoding encoding, string filename)
         {
-            if (stream == null)
+            if (content == null)
             {
-                throw new ArgumentNullException(nameof(stream));
+                throw new ArgumentNullException(nameof(content));
             }
 
-            _stream = stream;
+            if (encoding == null)
+            {
+                throw new ArgumentNullException(nameof(encoding));
+            }
+
+            _content = content;
             Encoding = encoding;
             Filename = filename;
         }
 
-        public Encoding Encoding { get; }
+        public override char this[int position] => _content[position];
+
+        public override Encoding Encoding { get; }
 
         public override string Filename { get; }
 
-        public override TextReader CreateReader()
+        public override int Length => _content.Length;
-        {
-            var copy = new MemoryStream(_stream.ToArray());
 
-            return Encoding == null
+        public override void CopyTo(int sourceIndex, char[] destination, int destinationIndex, int count)
-                ? new StreamReader(copy, detectEncodingFromByteOrderMarks: true)
+        {
-                : new StreamReader(copy, Encoding);
+            if (destination == null)
+            {
+                throw new ArgumentNullException(nameof(destination));
+            }
+
+            if (sourceIndex < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(sourceIndex));
+            }
+
+            if (destinationIndex < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(destinationIndex));
+            }
+
+            if (count < 0 || count > Length - sourceIndex || count > destination.Length - destinationIndex)
+            {
+                throw new ArgumentOutOfRangeException(nameof(count));
+            }
+
+            if (count == 0)
+            {
+                return;
+            }
+
+            _content.CopyTo(sourceIndex, destination, destinationIndex, count);
         }
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/Legacy/LineTrackingStringBuffer.cs

@@ -15,6 +15,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         private TextLine _endLine;
 
         public LineTrackingStringBuffer(string content)
+            : this(content.ToCharArray())
+        {
+        }
+
+        public LineTrackingStringBuffer(char[] content)
         {
             _endLine = new TextLine(0, 0);
             _lines = new List<TextLine>() { _endLine };
@@ -43,7 +48,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
             return new CharacterReference(line.Content[idx], new SourceLocation(absoluteIndex, line.Index, idx));
         }
 
-        private void Append(string content)
+        private void Append(char[] content)
         {
             for (int i = 0; i < content.Length; i++)
             {

src/Microsoft.AspNetCore.Razor.Evolution/Legacy/RazorParser.cs

@@ -13,17 +13,13 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
 
         public bool DesignTimeMode { get; set; }
 
-        public virtual RazorSyntaxTree Parse(TextReader input)
+        public virtual RazorSyntaxTree Parse(TextReader input) => Parse(input.ReadToEnd());
-        {
-            var reader = new SeekableTextReader(input);
 
-            return Parse((ITextDocument)reader);
+        public virtual RazorSyntaxTree Parse(string input) => Parse(((ITextDocument)new SeekableTextReader(input)));
-        }
 
-        public virtual RazorSyntaxTree Parse(ITextDocument input)
+        public virtual RazorSyntaxTree Parse(char[] input) => Parse(((ITextDocument)new SeekableTextReader(input)));
-        {
+
-            return ParseCore(input);
+        public virtual RazorSyntaxTree Parse(ITextDocument input) => ParseCore(input);
-        }
 
         private RazorSyntaxTree ParseCore(ITextDocument input)
         {

src/Microsoft.AspNetCore.Razor.Evolution/Legacy/SeekableTextReader.cs

@@ -13,12 +13,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         private SourceLocation _location = SourceLocation.Zero;
         private char? _current;
 
-        public SeekableTextReader(TextReader source)
+        public SeekableTextReader(string source) : this(source.ToCharArray()) { }
-            : this(source.ReadToEnd())
-        {
-        }
 
-        public SeekableTextReader(string source)
+        public SeekableTextReader(char[] source)
         {
             if (source == null)
             {

src/Microsoft.AspNetCore.Razor.Evolution/Properties/Resources.Designer.cs

@@ -26,6 +26,22 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             return string.Format(CultureInfo.CurrentCulture, GetString("IRBuilder_PopInvalid"), p0);
         }
 
+        /// <summary>
+        /// The specified encoding '{0}' does not match the content's encoding '{1}'.
+        /// </summary>
+        internal static string MismatchedContentEncoding
+        {
+            get { return GetString("MismatchedContentEncoding"); }
+        }
+
+        /// <summary>
+        /// The specified encoding '{0}' does not match the content's encoding '{1}'.
+        /// </summary>
+        internal static string FormatMismatchedContentEncoding(object p0, object p1)
+        {
+            return string.Format(CultureInfo.CurrentCulture, GetString("MismatchedContentEncoding"), p0, p1);
+        }
+
         /// <summary>
         /// The '{0}' phase requires a '{1}' provided by the '{2}'.
         /// </summary>

src/Microsoft.AspNetCore.Razor.Evolution/RazorSourceDocument.cs

@@ -9,6 +9,16 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 {
     public abstract class RazorSourceDocument
     {
+        public abstract Encoding Encoding { get; }
+
+        public abstract string Filename { get; }
+
+        public abstract char this[int position] { get; }
+
+        public abstract int Length { get; }
+
+        public abstract void CopyTo(int sourceIndex, char[] destination, int destinationIndex, int count);
+
         public static RazorSourceDocument ReadFrom(Stream stream, string filename)
         {
             if (stream == null)
@@ -36,14 +46,31 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 
         private static RazorSourceDocument ReadFromInternal(Stream stream, string filename, Encoding encoding)
         {
-            var memoryStream = new MemoryStream();
+            var reader = new StreamReader(
-            stream.CopyTo(memoryStream);
+                stream,
+                encoding ?? Encoding.UTF8,
+                detectEncodingFromByteOrderMarks: true,
+                bufferSize: (int)stream.Length,
+                leaveOpen: true);
 
-            return new DefaultRazorSourceDocument(memoryStream, encoding, filename);
+            using (reader)
+            {
+                var content = reader.ReadToEnd();
+
+                if (encoding == null)
+                {
+                    encoding = reader.CurrentEncoding;
+                }
+                else if (encoding != reader.CurrentEncoding)
+                {
+                    throw new InvalidOperationException(
+                        Resources.FormatMismatchedContentEncoding(
+                            encoding.EncodingName,
+                            reader.CurrentEncoding.EncodingName));
+                }
+
+                return new DefaultRazorSourceDocument(content, encoding, filename);
+            }
         }
-
-        public abstract string Filename { get; }
-
-        public abstract TextReader CreateReader();
     }
 }

src/Microsoft.AspNetCore.Razor.Evolution/RazorSyntaxTree.cs

@@ -32,11 +32,10 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             }
 
             var parser = new RazorParser();
+            var sourceContent = new char[source.Length];
+            source.CopyTo(0, sourceContent, 0, source.Length);
 
-            using (var reader = source.CreateReader())
+            return parser.Parse(sourceContent);
-            {
-                return parser.Parse(reader);
-            }
         }
 
         internal abstract IReadOnlyList<RazorError> Diagnostics { get; }

src/Microsoft.AspNetCore.Razor.Evolution/Resources.resx

@@ -120,6 +120,9 @@
   <data name="IRBuilder_PopInvalid" xml:space="preserve">
     <value>The '{0}' operation is not valid when the builder is empty.</value>
   </data>
+  <data name="MismatchedContentEncoding" xml:space="preserve">
+    <value>The specified encoding '{0}' does not match the content's encoding '{1}'.</value>
+  </data>
   <data name="PhaseDependencyMissing" xml:space="preserve">
     <value>The '{0}' phase requires a '{1}' provided by the '{2}'.</value>
   </data>

test/Microsoft.AspNetCore.Razor.Evolution.Test/DefaultRazorSourceDocumentTest.cs

@@ -9,11 +9,41 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 {
     public class DefaultRazorSourceDocumentTest
     {
+        [Fact]
+        public void Indexer_ProvidesCharacterAccessToContent()
+        {
+            // Arrange
+            var expectedContent = "Hello, World!";
+            var indexerBuffer = new char[expectedContent.Length];
+            var document = new DefaultRazorSourceDocument(expectedContent, Encoding.UTF8, filename: "file.cshtml");
+
+            // Act
+            for (var i = 0; i < document.Length; i++)
+            {
+                indexerBuffer[i] = document[i];
+            }
+
+            // Assert
+            var output = new string(indexerBuffer);
+            Assert.Equal(expectedContent, output);
+        }
+
+        [Fact]
+        public void Length()
+        {
+            // Arrange
+            var expectedContent = "Hello, World!";
+            var document = new DefaultRazorSourceDocument(expectedContent, Encoding.UTF8, filename: "file.cshtml");
+
+            // Act & Assert
+            Assert.Equal(expectedContent.Length, document.Length);
+        }
+
         [Fact]
         public void Filename()
         {
             // Arrange
-            var content = CreateContent();
+            var content = "Hello, World!";
 
             // Act
             var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: "file.cshtml");
@@ -26,7 +56,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
         public void Filename_Null()
         {
             // Arrange
-            var content = CreateContent();
+            var content = "Hello, World!";
 
             // Act
             var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: null);
@@ -36,64 +66,90 @@ namespace Microsoft.AspNetCore.Razor.Evolution
         }
 
         [Fact]
-        public void CreateReader_WithEncoding()
+        public void CopyTo_PartialCopyFromStart()
         {
             // Arrange
-            var content = CreateContent("Hi", encoding: Encoding.UTF8);
+            var content = "Hello, World!";
             var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: null);
+            var expectedContent = "Hello";
+            var charBuffer = new char[expectedContent.Length];
 
             // Act
-            using (var reader = document.CreateReader())
+            document.CopyTo(0, charBuffer, 0, expectedContent.Length);
-            {
+
-                // Assert
+            // Assert
-                Assert.Equal("Hi", reader.ReadToEnd());
+            var copiedContent = new string(charBuffer);
-            }
+            Assert.Equal(expectedContent, copiedContent);
         }
 
         [Fact]
-        public void CreateReader_Null_DetectsEncoding()
+        public void CopyTo_PartialCopyDestinationOffset()
         {
             // Arrange
-            var content = CreateContent("Hi", encoding: Encoding.UTF32);
+            var content = "Hello, World!";
-            var document = new DefaultRazorSourceDocument(content, encoding: null, filename: null);
+            var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: null);
+            var expectedContent = "$Hello";
+            var charBuffer = new char[expectedContent.Length];
+            charBuffer[0] = '$';
 
             // Act
-            using (var reader = document.CreateReader())
+            document.CopyTo(0, charBuffer, 1, "Hello".Length);
-            {
+
-                // Assert
+            // Assert
-                Assert.Equal("Hi", reader.ReadToEnd());
+            var copiedContent = new string(charBuffer);
-            }
+            Assert.Equal(expectedContent, copiedContent);
         }
 
         [Fact]
-        public void CreateReader_DisposeReader_DoesNotDirtyDocument()
+        public void CopyTo_PartialCopySourceOffset()
         {
             // Arrange
-            var content = CreateContent("Hi", encoding: Encoding.UTF32);
+            var content = "Hello, World!";
-            var document = new DefaultRazorSourceDocument(content, encoding: null, filename: null);
+            var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: null);
+            var expectedContent = "World";
+            var charBuffer = new char[expectedContent.Length];
+
+            // Act
+            document.CopyTo(7, charBuffer, 0, expectedContent.Length);
+
+            // Assert
+            var copiedContent = new string(charBuffer);
+            Assert.Equal(expectedContent, copiedContent);
+        }
+
+        [Fact]
+        public void CopyTo_WithEncoding()
+        {
+            // Arrange
+            var content = "Hi";
+            var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: null);
+            var charBuffer = new char[2];
+
+            // Act
+            document.CopyTo(0, charBuffer, 0, 2);
+
+            // Assert
+            var copiedContent = new string(charBuffer);
+            Assert.Equal("Hi", copiedContent);
+        }
+
+        [Fact]
+        public void CopyTo_CanCopyMultipleTimes()
+        {
+            // Arrange
+            var content = "Hi";
+            var document = new DefaultRazorSourceDocument(content, Encoding.UTF8, filename: null);
 
             // Act & Assert
             //
             // (we should be able to do this twice to prove that the underlying data isn't disposed)
             for (var i = 0; i < 2; i++)
             {
-                using (var reader = document.CreateReader())
+                var charBuffer = new char[2];
-                {
+                document.CopyTo(0, charBuffer, 0, 2);
-                    // Assert
+                var copiedContent = new string(charBuffer);
-                    Assert.Equal("Hi", reader.ReadToEnd());
+                Assert.Equal("Hi", copiedContent);
-                }
             }
         }
-
-        private static MemoryStream CreateContent(string content = "Hello, World!", Encoding encoding = null)
-        {
-            var stream = new MemoryStream();
-            using (var writer = new StreamWriter(stream, encoding ?? Encoding.UTF8, bufferSize: 1024, leaveOpen: true))
-            {
-                writer.Write(content);
-            }
-
-            return stream;
-        }
     }
 }

test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/RazorParserTest.cs

@@ -11,11 +11,11 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         [Fact]
         public void CanParseStuff()
         {
-            
             var parser = new RazorParser();
             var sourceDocument = TestRazorSourceDocument.CreateResource("TestFiles/Source/BasicMarkup.cshtml");
-            var documentReader = sourceDocument.CreateReader();
+            var sourceContent = new char[sourceDocument.Length];
-            var output = parser.Parse(documentReader);
+            sourceDocument.CopyTo(0, sourceContent, 0, sourceDocument.Length);
+            var output = parser.Parse(sourceContent);
 
             Assert.NotNull(output);
         }

test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerLookaheadTest.cs

@@ -59,7 +59,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
         private class ExposedTokenizer : Tokenizer<CSharpSymbol, CSharpSymbolType>
         {
             public ExposedTokenizer(string input)
-                : base(new SeekableTextReader(new StringReader(input)))
+                : base(new SeekableTextReader(input))
             {
             }
 

test/Microsoft.AspNetCore.Razor.Evolution.Test/Legacy/TokenizerTestBase.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Diagnostics;
-using System.IO;
 using System.Text;
 using Xunit;
 
@@ -21,45 +20,42 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
             // Arrange
             var success = true;
             var output = new StringBuilder();
-            using (StringReader reader = new StringReader(input))
+            using (var source = new SeekableTextReader(input))
             {
-                using (SeekableTextReader source = new SeekableTextReader(reader))
+                var tokenizer = (Tokenizer<TSymbol, TSymbolType>)CreateTokenizer(source);
+                var counter = 0;
+                TSymbol current = null;
+                while ((current = tokenizer.NextSymbol()) != null)
                 {
-                    var tokenizer = (Tokenizer<TSymbol, TSymbolType>)CreateTokenizer(source);
+                    if (counter >= expectedSymbols.Length)
-                    var counter = 0;
-                    TSymbol current = null;
-                    while ((current = tokenizer.NextSymbol()) != null)
                     {
-                        if (counter >= expectedSymbols.Length)
+                        output.AppendLine(string.Format("F: Expected: << Nothing >>; Actual: {0}", current));
+                        success = false;
+                    }
+                    else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
+                    {
+                        output.AppendLine(string.Format("P: Ignored {0}", current));
+                    }
+                    else
+                    {
+                        if (!Equals(expectedSymbols[counter], current))
                         {
-                            output.AppendLine(string.Format("F: Expected: << Nothing >>; Actual: {0}", current));
+                            output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
                             success = false;
                         }
-                        else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
-                        {
-                            output.AppendLine(string.Format("P: Ignored {0}", current));
-                        }
                         else
                         {
-                            if (!Equals(expectedSymbols[counter], current))
+                            output.AppendLine(string.Format("P: Expected: {0}", expectedSymbols[counter]));
-                            {
-                                output.AppendLine(string.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
-                                success = false;
-                            }
-                            else
-                            {
-                                output.AppendLine(string.Format("P: Expected: {0}", expectedSymbols[counter]));
-                            }
-                            counter++;
                         }
+                        counter++;
                     }
-                    if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
+                }
+                if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
+                {
+                    success = false;
+                    for (; counter < expectedSymbols.Length; counter++)
                     {
-                        success = false;
+                        output.AppendLine(string.Format("F: Expected: {0}; Actual: << None >>", expectedSymbols[counter]));
-                        for (; counter < expectedSymbols.Length; counter++)
-                        {
-                            output.AppendLine(string.Format("F: Expected: {0}; Actual: << None >>", expectedSymbols[counter]));
-                        }
                     }
                 }
             }

test/Microsoft.AspNetCore.Razor.Evolution.Test/RazorSourceDocumentTest.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Text;
 using Xunit;
 
@@ -12,21 +13,22 @@ namespace Microsoft.AspNetCore.Razor.Evolution
         public void Create()
         {
             // Arrange
-            var content = TestRazorSourceDocument.CreateContent();
+            var content = TestRazorSourceDocument.CreateStreamContent();
 
             // Act
             var document = RazorSourceDocument.ReadFrom(content, "file.cshtml");
 
             // Assert
+            Assert.IsType<DefaultRazorSourceDocument>(document);
             Assert.Equal("file.cshtml", document.Filename);
-            Assert.Null(Assert.IsType<DefaultRazorSourceDocument>(document).Encoding);
+            Assert.Same(Encoding.UTF8, document.Encoding);
         }
 
         [Fact]
         public void Create_WithEncoding()
         {
             // Arrange
-            var content = TestRazorSourceDocument.CreateContent(encoding: Encoding.UTF32);
+            var content = TestRazorSourceDocument.CreateStreamContent(encoding: Encoding.UTF32);
 
             // Act
             var document = RazorSourceDocument.ReadFrom(content, "file.cshtml", Encoding.UTF32);
@@ -35,5 +37,33 @@ namespace Microsoft.AspNetCore.Razor.Evolution
             Assert.Equal("file.cshtml", document.Filename);
             Assert.Same(Encoding.UTF32, Assert.IsType<DefaultRazorSourceDocument>(document).Encoding);
         }
+
+        [Fact]
+        public void ReadFrom_DetectsEncoding()
+        {
+            // Arrange
+            var content = TestRazorSourceDocument.CreateStreamContent(encoding: Encoding.UTF32);
+
+            // Act
+            var document = RazorSourceDocument.ReadFrom(content, "file.cshtml");
+
+            // Assert
+            Assert.IsType<DefaultRazorSourceDocument>(document);
+            Assert.Equal("file.cshtml", document.Filename);
+            Assert.Equal(Encoding.UTF32, document.Encoding);
+        }
+
+        [Fact]
+        public void ReadFrom_FailsOnMismatchedEncoding()
+        {
+            // Arrange
+            var content = TestRazorSourceDocument.CreateStreamContent(encoding: Encoding.UTF32);
+            var expectedMessage = Resources.FormatMismatchedContentEncoding(Encoding.UTF8.EncodingName, Encoding.UTF32.EncodingName);
+
+            // Act & Assert
+            var exception = Assert.Throws<InvalidOperationException>(
+                () => RazorSourceDocument.ReadFrom(content, "file.cshtml", Encoding.UTF8));
+            Assert.Equal(expectedMessage, exception.Message);
+        }
     }
 }

test/Microsoft.AspNetCore.Razor.Evolution.Test/TestRazorSourceDocument.cs

@@ -9,25 +9,29 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 {
     internal class TestRazorSourceDocument : DefaultRazorSourceDocument
     {
+        private TestRazorSourceDocument(string content, Encoding encoding, string filename)
+            : base(content, encoding, filename)
+        {
+        }
+
         public static RazorSourceDocument CreateResource(string path, Encoding encoding = null)
         {
             var file = TestFile.Create(path);
 
-            var stream = new MemoryStream();
             using (var input = file.OpenRead())
+            using (var reader = new StreamReader(input))
             {
-                input.CopyTo(stream);
+                var content = reader.ReadToEnd();
+
+                return new TestRazorSourceDocument(content, encoding ?? Encoding.UTF8, path);
             }
-
-            stream.Seek(0L, SeekOrigin.Begin);
-
-            return new TestRazorSourceDocument(stream, encoding ?? Encoding.UTF8, path);
         }
 
-        public static MemoryStream CreateContent(string content = "Hello, World!", Encoding encoding = null)
+        public static MemoryStream CreateStreamContent(string content = "Hello, World!", Encoding encoding = null)
         {
             var stream = new MemoryStream();
-            using (var writer = new StreamWriter(stream, encoding ?? Encoding.UTF8, bufferSize: 1024, leaveOpen: true))
+            encoding = encoding ?? Encoding.UTF8;
+            using (var writer = new StreamWriter(stream, encoding, bufferSize: 1024, leaveOpen: true))
             {
                 writer.Write(content);
             }
@@ -39,13 +43,7 @@ namespace Microsoft.AspNetCore.Razor.Evolution
 
         public static RazorSourceDocument Create(string content = "Hello, world!", Encoding encoding = null)
         {
-            var stream = CreateContent(content, encoding);
+            return new TestRazorSourceDocument(content, encoding ?? Encoding.UTF8, "test.cshtml");
-            return new TestRazorSourceDocument(stream, encoding ?? Encoding.UTF8, "test.cshtml");
-        }
-
-        public TestRazorSourceDocument(MemoryStream stream, Encoding encoding, string filename) 
-            : base(stream, encoding, filename)
-        {
         }
     }
 }