Add integration testing

This change adds a basic framework for doing baselined integration tests.
This is very similar to what we do elsewhere with generated files and
tests that read them from resources.

What's here now is the support to do this kind of baselining with IR in a
pretty readable serialization format.

This is a building block and the intent is that we'd do something similar
in the future for syntax nodes and C# source.

Looking at the code of the tests in particular, we'll also build the
ability to capture the documents at key points (such as before/after a
targeted phase) and then verify them in the same manner.
This commit is contained in:
Ryan Nowak 2016-12-05 23:32:15 -08:00
parent 2f54c12b82
commit 853c28e568
13 changed files with 672 additions and 17 deletions

View File

@ -5,10 +5,58 @@ using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
using Microsoft.AspNetCore.Razor.Evolution.Legacy;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Evolution
namespace Microsoft.AspNetCore.Razor.Evolution.IntegrationTests
{
public class IntegrationTest
public class BasicIntegrationTest : IntegrationTestBase
{
[Fact]
public void Empty()
{
// Arrange
var engine = RazorEngine.Create();
var document = CreateCodeDocument();
// Act
engine.Process(document);
// Assert
AssertIRMatchesBaseline(document.GetIRDocument());
}
[Fact]
public void HelloWorld()
{
// Arrange
var engine = RazorEngine.Create();
var document = CreateCodeDocument();
// Act
engine.Process(document);
// Assert
AssertIRMatchesBaseline(document.GetIRDocument());
}
[Fact]
public void CustomDirective()
{
// Arrange
var engine = RazorEngine.Create(b =>
{
b.AddDirective(DirectiveDescriptorBuilder.Create("test_directive").Build());
});
var document = CreateCodeDocument();
// Act
engine.Process(document);
// Assert
AssertIRMatchesBaseline(document.GetIRDocument());
}
[Fact]
public void BuildEngine_CallProcess()
{

View File

@ -0,0 +1,94 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
#if NET451
using System.Runtime.Remoting;
using System.Runtime.Remoting.Messaging;
#else
using System.Threading;
#endif
using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
using Xunit;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Razor.Evolution.IntegrationTests
{
[IntializeTestFile]
public abstract class IntegrationTestBase
{
#if GENERATE_BASELINES
private static readonly bool GenerateBaselines = true;
#else
private static readonly bool GenerateBaselines = false;
#endif
#if !NET451
private static readonly AsyncLocal<string> _filename = new AsyncLocal<string>();
#endif
// Used by the test framework to set the 'base' name for test files.
public static string Filename
{
#if NET451
get
{
var handle = (ObjectHandle)CallContext.LogicalGetData("IntegrationTestBase_Filename");
return (string)handle.Unwrap();
}
set
{
CallContext.LogicalSetData("IntegrationTestBase_Filename", new ObjectHandle(value));
}
#else
get { return _filename.Value; }
set { _filename.Value = value; }
#endif
}
protected RazorCodeDocument CreateCodeDocument()
{
if (Filename == null)
{
var message = $"{nameof(CreateCodeDocument)} should only be called from an integration test ({nameof(Filename)} is null).";
throw new InvalidOperationException(message);
}
var sourceFilename = Path.ChangeExtension(Filename, ".cshtml");
var testFile = TestFile.Create(sourceFilename);
if (!testFile.Exists())
{
throw new XunitException($"The resource {sourceFilename} was not found.");
}
return RazorCodeDocument.Create(TestRazorSourceDocument.CreateResource(sourceFilename));
}
protected void AssertIRMatchesBaseline(DocumentIRNode document)
{
if (Filename == null)
{
var message = $"{nameof(AssertIRMatchesBaseline)} should only be called from an integration test ({nameof(Filename)} is null).";
throw new InvalidOperationException(message);
}
var baselineFilename = Path.ChangeExtension(Filename, ".ir.txt");
if (GenerateBaselines)
{
File.WriteAllText(baselineFilename, RazorIRNodeSerializer.Serialize(document));
return;
}
var testFile = TestFile.Create(baselineFilename);
if (!testFile.Exists())
{
throw new XunitException($"The resource {baselineFilename} was not found.");
}
var baseline = testFile.ReadAllText().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
RazorIRNodeVerifier.Verify(document, baseline);
}
}
}

View File

@ -0,0 +1,28 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Reflection;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Razor.Evolution.IntegrationTests
{
public class IntializeTestFileAttribute : BeforeAfterTestAttribute
{
public override void Before(MethodInfo methodUnderTest)
{
if (typeof(IntegrationTestBase).IsAssignableFrom(methodUnderTest.DeclaringType))
{
var typeName = methodUnderTest.DeclaringType.Name;
IntegrationTestBase.Filename = $"TestFiles/IntegrationTests/{typeName}/{methodUnderTest.Name}";
}
}
public override void After(MethodInfo methodUnderTest)
{
if (typeof(IntegrationTestBase).IsAssignableFrom(methodUnderTest.DeclaringType))
{
IntegrationTestBase.Filename = null;
}
}
}
}

View File

@ -0,0 +1,46 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.IO;
using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
namespace Microsoft.AspNetCore.Razor.Evolution.IntegrationTests
{
public static class RazorIRNodeSerializer
{
public static string Serialize(RazorIRNode node)
{
using (var writer = new StringWriter())
{
var walker = new Walker(writer);
walker.Visit(node);
return writer.ToString();
}
}
private class Walker : RazorIRNodeWalker
{
private readonly RazorIRNodeWriter _visitor;
private readonly TextWriter _writer;
public Walker(TextWriter writer)
{
_visitor = new RazorIRNodeWriter(writer);
_writer = writer;
}
public TextWriter Writer { get; }
public override void VisitDefault(RazorIRNode node)
{
_visitor.Visit(node);
_writer.WriteLine();
_visitor.Depth++;
base.VisitDefault(node);
_visitor.Depth--;
}
}
}
}

View File

@ -0,0 +1,265 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
using System.Text;
using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
using Xunit;
using Xunit.Sdk;
namespace Microsoft.AspNetCore.Razor.Evolution.IntegrationTests
{
public static class RazorIRNodeVerifier
{
public static void Verify(RazorIRNode node, string[] baseline)
{
var walker = new Walker(baseline);
walker.Visit(node);
}
private class Walker : RazorIRNodeWalker
{
private readonly string[] _baseline;
private readonly RazorIRNodeWriter _visitor;
private readonly StringWriter _writer;
private int _index;
public Walker(string[] baseline)
{
_writer = new StringWriter();
_visitor = new RazorIRNodeWriter(_writer);
_baseline = baseline;
}
public TextWriter Writer { get; }
public override void VisitDefault(RazorIRNode node)
{
var expected = _index < _baseline.Length ? _baseline[_index++] : null;
// Write the node as text for comparison
_writer.GetStringBuilder().Clear();
_visitor.Visit(node);
var actual = _writer.GetStringBuilder().ToString();
AssertNodeEquals(node, expected, actual);
_visitor.Depth++;
base.VisitDefault(node);
_visitor.Depth--;
}
private void AssertNodeEquals(RazorIRNode node, string expected, string actual)
{
if (string.Equals(expected, actual))
{
// YAY!!! everything is great.
return;
}
if (expected == null)
{
var message = "The node is missing from baseline.";
throw new IRBaselineException(node, expected, actual, message);
}
int charsVerified = 0;
AssertNestingEqual(node, expected, actual, ref charsVerified);
AssertNameEqual(node, expected, actual, ref charsVerified);
AssertDelimiter(node, expected, actual, true, ref charsVerified);
AssertLocationEqual(node, expected, actual, ref charsVerified);
AssertDelimiter(node, expected, actual, false, ref charsVerified);
AssertContentEqual(node, expected, actual, ref charsVerified);
throw new InvalidOperationException("We can't figure out HOW these two things are different. This is a bug.");
}
private void AssertNestingEqual(RazorIRNode node, string expected, string actual, ref int charsVerified)
{
var i = 0;
for (; i < expected.Length; i++)
{
if (expected[i] != ' ')
{
break;
}
}
var failed = false;
var j = 0;
for (; j < i; j++)
{
if (actual.Length <= j || actual[j] != ' ')
{
failed = true;
break;
}
}
if (actual.Length <= j + 1 || actual[j] == ' ')
{
failed = true;
}
if (failed)
{
var message = "The node is at the wrong level of nesting. This usually means a child is missing.";
throw new IRBaselineException(node, expected, actual, message);
}
charsVerified = j;
}
private void AssertNameEqual(RazorIRNode node, string expected, string actual, ref int charsVerified)
{
var expectedName = GetName(expected, charsVerified);
var actualName = GetName(actual, charsVerified);
if (!string.Equals(expectedName, actualName))
{
var message = $"Node names are not equal.";
throw new IRBaselineException(node, expected, actual, message);
}
charsVerified += expectedName.Length;
}
// Either both strings need to have a delimiter next or neither should.
private void AssertDelimiter(RazorIRNode node, string expected, string actual, bool required, ref int charsVerified)
{
if (charsVerified == expected.Length && required)
{
throw new InvalidOperationException($"Baseline text is not well-formed: '{expected}'.");
}
if (charsVerified == actual.Length && required)
{
throw new InvalidOperationException($"Baseline text is not well-formed: '{actual}'.");
}
if (charsVerified == expected.Length && charsVerified == actual.Length)
{
return;
}
var expectedDelimiter = expected.IndexOf(" - ", charsVerified);
if (expectedDelimiter != charsVerified && expectedDelimiter != -1)
{
throw new InvalidOperationException($"Baseline text is not well-formed: '{actual}'.");
}
var actualDelimiter = actual.IndexOf(" - ", charsVerified);
if (actualDelimiter != charsVerified && actualDelimiter != -1)
{
throw new InvalidOperationException($"Baseline text is not well-formed: '{actual}'.");
}
Assert.Equal(expectedDelimiter, actualDelimiter);
charsVerified += 3;
}
private void AssertLocationEqual(RazorIRNode node, string expected, string actual, ref int charsVerified)
{
var expectedLocation = GetLocation(expected, charsVerified);
var actualLocation = GetLocation(actual, charsVerified);
if (!string.Equals(expectedLocation, actualLocation))
{
var message = $"Locations are not equal.";
throw new IRBaselineException(node, expected, actual, message);
}
charsVerified += expectedLocation.Length;
}
private void AssertContentEqual(RazorIRNode node, string expected, string actual, ref int charsVerified)
{
var expectedContent = GetContent(expected, charsVerified);
var actualContent = GetContent(actual, charsVerified);
if (!string.Equals(expectedContent, actualContent))
{
var message = $"Contents are not equal.";
throw new IRBaselineException(node, expected, actual, message);
}
charsVerified += expectedContent.Length;
}
private string GetName(string text, int start)
{
var delimiter = text.IndexOf(" - ", start);
if (delimiter == -1)
{
throw new InvalidOperationException($"Baseline text is not well-formed: '{text}'.");
}
return text.Substring(start, delimiter - start);
}
private string GetLocation(string text, int start)
{
var delimiter = text.IndexOf(" - ", start);
return delimiter == -1 ? text.Substring(start) : text.Substring(start, delimiter - start);
}
private string GetContent(string text, int start)
{
return start == text.Length ? string.Empty : text.Substring(start);
}
private class IRBaselineException : XunitException
{
public IRBaselineException(RazorIRNode node, string expected, string actual, string userMessage)
: base(Format(node, expected, actual, userMessage))
{
Node = node;
Expected = expected;
Actual = actual;
}
public RazorIRNode Node { get; }
public string Actual { get; }
public string Expected { get; }
private static string Format(RazorIRNode node, string expected, string actual, string userMessage)
{
var builder = new StringBuilder();
builder.AppendLine(userMessage);
builder.AppendLine();
if (expected != null)
{
builder.Append("Expected: ");
builder.AppendLine(expected);
}
if (actual != null)
{
builder.Append("Actual: ");
builder.AppendLine(actual);
}
builder.AppendLine();
builder.AppendLine("Path:");
var current = node;
do
{
builder.AppendLine(current.ToString());
}
while ((current = current.Parent) != null);
return builder.ToString();
}
}
}
}
}

View File

@ -0,0 +1,165 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Collections.Generic;
using System.IO;
using Microsoft.AspNetCore.Razor.Evolution.Intermediate;
namespace Microsoft.AspNetCore.Razor.Evolution.IntegrationTests
{
// Serializes single IR nodes (shallow).
public class RazorIRNodeWriter : RazorIRNodeVisitor
{
private readonly TextWriter _writer;
public RazorIRNodeWriter(TextWriter writer)
{
_writer = writer;
}
public int Depth { get; set; }
public override void VisitDefault(RazorIRNode node)
{
WriteBasicNode(node);
}
public override void VisitClass(ClassDeclarationIRNode node)
{
WriteContentNode(node, node.AccessModifier, node.Name, node.BaseType, string.Join(", ", node.Interfaces ?? new List<string>()));
}
public override void VisitCSharpAttributeValue(CSharpAttributeValueIRNode node)
{
WriteContentNode(node, node.Prefix, node.Content.ToString()); // This is broken.
}
public override void VisitCSharpExpression(CSharpExpressionIRNode node)
{
WriteContentNode(node, node.Content.ToString()); // This is broken
}
public override void VisitCSharpStatement(CSharpStatementIRNode node)
{
WriteContentNode(node, node.Content);
}
public override void VisitCSharpToken(CSharpTokenIRNode node)
{
WriteContentNode(node, node.Content);
}
public override void VisitDirective(DirectiveIRNode node)
{
WriteContentNode(node, node.Name);
}
public override void VisitDirectiveToken(DirectiveTokenIRNode node)
{
WriteContentNode(node, node.Content);
}
public override void VisitHtml(HtmlContentIRNode node)
{
WriteContentNode(node, node.Content);
}
public override void VisitHtmlAttribute(HtmlAttributeIRNode node)
{
WriteContentNode(node, node.Prefix, node.Value.ToString(), node.Suffix);
}
public override void VisitHtmlAttributeValue(HtmlAttributeValueIRNode node)
{
WriteContentNode(node, node.Prefix, node.Content);
}
public override void VisitNamespace(NamespaceDeclarationIRNode node)
{
WriteContentNode(node, node.Content);
}
public override void VisitRazorMethodDeclaration(RazorMethodDeclarationIRNode node)
{
WriteContentNode(node, node.AccessModifier, string.Join(", ", node.Modifiers ?? new List<string>()), node.ReturnType, node.Name);
}
public override void VisitUsingStatement(UsingStatementIRNode node)
{
WriteContentNode(node, node.Content);
}
protected void WriteBasicNode(RazorIRNode node)
{
WriteIndent();
WriteName(node);
WriteSeparator();
WriteLocation(node);
}
protected void WriteContentNode(RazorIRNode node, params string[] content)
{
WriteIndent();
WriteName(node);
WriteSeparator();
WriteLocation(node);
for (var i = 0; i < content.Length; i++)
{
WriteSeparator();
WriteContent(content[i]);
}
}
protected void WriteIndent()
{
for (var i = 0; i < Depth; i++)
{
for (var j = 0; j < 4; j++)
{
_writer.Write(' ');
}
}
}
protected void WriteSeparator()
{
_writer.Write(" - ");
}
protected void WriteNewLine()
{
_writer.WriteLine();
}
protected void WriteName(RazorIRNode node)
{
var typeName = node.GetType().Name;
if (typeName.EndsWith("IRNode"))
{
_writer.Write(typeName.Substring(0, typeName.Length - "IRNode".Length));
}
else
{
_writer.Write(typeName);
}
}
protected void WriteLocation(RazorIRNode node)
{
_writer.Write(node.SourceLocation.ToString());
}
protected void WriteContent(string content)
{
if (content == null)
{
return;
}
// We explicitly escape newlines in node content so that the IR can be compared line-by-line.
// Also, escape our separator so we can search for ` - `to find delimiters.
_writer.Write(content.Replace("\r", "\\r").Replace("\n", "\\n").Replace("-", "\\-"));
}
}
}

View File

@ -6,11 +6,11 @@ using System.IO;
using System.Reflection;
using Xunit;
namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
namespace Microsoft.AspNetCore.Razor.Evolution
{
public class TestFile
{
public TestFile(string resourceName, Assembly assembly)
private TestFile(string resourceName, Assembly assembly)
{
Assembly = assembly;
ResourceName = Assembly.GetName().Name + "." + resourceName.Replace('/', '.');
@ -20,9 +20,9 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
public string ResourceName { get; }
public static TestFile Create(string localResourceName)
public static TestFile Create(string resourceName)
{
return new TestFile(localResourceName, typeof(TestFile).GetTypeInfo().Assembly);
return new TestFile(resourceName, typeof(TestFile).GetTypeInfo().Assembly);
}
public Stream OpenRead()
@ -51,17 +51,6 @@ namespace Microsoft.AspNetCore.Razor.Evolution.Legacy
return false;
}
public byte[] ReadAllBytes()
{
using (var stream = OpenRead())
{
var buffer = new byte[stream.Length];
stream.Read(buffer, 0, buffer.Length);
return buffer;
}
}
public string ReadAllText()
{
using (var reader = new StreamReader(OpenRead()))

View File

@ -0,0 +1,7 @@
Document - (0:0,0)
NamespaceDeclaration - (0:0,0) -
ClassDeclaration - (0:0,0) - - - -
RazorMethodDeclaration - (0:0,0) - - - -
HtmlContent - (0:0,0) -
Directive - (0:0,0) - test_directive
HtmlContent - (15:0,15) -

View File

@ -0,0 +1,5 @@
Document - (0:0,0)
NamespaceDeclaration - (0:0,0) -
ClassDeclaration - (0:0,0) - - - -
RazorMethodDeclaration - (0:0,0) - - - -
HtmlContent - (0:0,0) -

View File

@ -0,0 +1,5 @@
Document - (0:0,0)
NamespaceDeclaration - (0:0,0) -
ClassDeclaration - (0:0,0) - - - -
RazorMethodDeclaration - (0:0,0) - - - -
HtmlContent - (0:0,0) - Hello, World!