diff --git a/.vsts/builds/e2e-tests.yml b/.vsts/builds/e2e-tests.yml new file mode 100644 index 0000000000..014f5481ba --- /dev/null +++ b/.vsts/builds/e2e-tests.yml @@ -0,0 +1,19 @@ +trigger: none +queue: + name: DotNetCore-Windows + timeoutInMinutes: 120 +steps: +- task: NodeTool@0 + displayName: Install Node 10.x + inputs: + versionSpec: 10.x +- powershell: | + test/Cli.FunctionalTests/run-tests.ps1 -ci -ProdConManifestUrl $env:ProdConManifestUrl + condition: ne(variables['PB_SkipTests'], 'true') + displayName: Run E2E tests +- task: PublishTestResults@2 + displayName: Publish test results + condition: always() + inputs: + testRunner: vstest + testResultsFiles: 'artifacts/logs/**/*.trx' diff --git a/build/artifacts.props b/build/artifacts.props index abaa4ce285..e90d320e82 100644 --- a/build/artifacts.props +++ b/build/artifacts.props @@ -274,6 +274,7 @@ + diff --git a/test/Cli.FunctionalTests/AssemblyInfo.cs b/test/Cli.FunctionalTests/AssemblyInfo.cs new file mode 100644 index 0000000000..cbe539e977 --- /dev/null +++ b/test/Cli.FunctionalTests/AssemblyInfo.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Cli.FunctionalTests.Util; +using NUnit.Framework; + +// Run all test cases in parallel +[assembly: Parallelizable(ParallelScope.Children)] + +[SetUpFixture] +public class AssemblySetUp +{ + public static string TempDir { get; private set; } + + [OneTimeSetUp] + public void SetUp() + { + TempDir = IOUtil.GetTempDir(); + } + + [OneTimeTearDown] + public void TearDown() + { + IOUtil.DeleteDir(TempDir); + } +} diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj new file mode 100644 index 0000000000..385a9dde2a --- /dev/null +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp2.1 + false + latest + + https://api.nuget.org/v3/index.json; + $(DotNetRestoreSources) + + + + + + + + + + + + diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.sln b/test/Cli.FunctionalTests/Cli.FunctionalTests.sln new file mode 100644 index 0000000000..6aff0f5478 --- /dev/null +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28016.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cli.FunctionalTests", "Cli.FunctionalTests.csproj", "{D44EA496-EF83-4D47-8C45-4DAF5A1B0070}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {70432DA7-DCE4-4F73-A00C-E1AB180DDD6A} + EndGlobalSection +EndGlobal diff --git a/test/Cli.FunctionalTests/Directory.Build.props b/test/Cli.FunctionalTests/Directory.Build.props new file mode 100644 index 0000000000..6d087b3032 --- /dev/null +++ b/test/Cli.FunctionalTests/Directory.Build.props @@ -0,0 +1,2 @@ + + diff --git a/test/Cli.FunctionalTests/Directory.Build.rsp b/test/Cli.FunctionalTests/Directory.Build.rsp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/Cli.FunctionalTests/Directory.Build.targets b/test/Cli.FunctionalTests/Directory.Build.targets new file mode 100644 index 0000000000..6d087b3032 --- /dev/null +++ b/test/Cli.FunctionalTests/Directory.Build.targets @@ -0,0 +1,2 @@ + + diff --git a/test/Cli.FunctionalTests/NuGet.config b/test/Cli.FunctionalTests/NuGet.config new file mode 100644 index 0000000000..4bb3170917 --- /dev/null +++ b/test/Cli.FunctionalTests/NuGet.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/test/Cli.FunctionalTests/NuGetPackageSource.cs b/test/Cli.FunctionalTests/NuGetPackageSource.cs new file mode 100644 index 0000000000..30545f8ca5 --- /dev/null +++ b/test/Cli.FunctionalTests/NuGetPackageSource.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; + +namespace Cli.FunctionalTests +{ + public class NuGetPackageSource + { + public static NuGetPackageSource None { get; } = new NuGetPackageSource + { + Name = nameof(None), + SourceArgumentLazy = new Lazy(string.Empty), + }; + + public static NuGetPackageSource NuGetOrg { get; } = new NuGetPackageSource + { + Name = nameof(NuGetOrg), + SourceArgumentLazy = new Lazy("--source https://api.nuget.org/v3/index.json"), + }; + + public static NuGetPackageSource DotNetCore { get; } = new NuGetPackageSource + { + Name = nameof(DotNetCore), + SourceArgumentLazy = new Lazy("--source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json"), + }; + + public static NuGetPackageSource EnvironmentVariable { get; } = new NuGetPackageSource + { + Name = nameof(EnvironmentVariable), + SourceArgumentLazy = new Lazy(() => GetSourceArgumentFromEnvironment()), + }; + + public static NuGetPackageSource EnvironmentVariableAndNuGetOrg { get; } = new NuGetPackageSource + { + Name = nameof(EnvironmentVariableAndNuGetOrg), + SourceArgumentLazy = new Lazy(() => string.Join(" ", EnvironmentVariable.SourceArgument, NuGetOrg.SourceArgument)), + }; + + private NuGetPackageSource() { } + + public string Name { get; private set; } + public string SourceArgument => SourceArgumentLazy.Value; + private Lazy SourceArgumentLazy { get; set; } + + public override string ToString() => Name; + + private static string GetSourceArgumentFromEnvironment() + { + var sourceString = Environment.GetEnvironmentVariable("NUGET_PACKAGE_SOURCE") ?? + throw new InvalidOperationException("Environment variable NUGET_PACKAGE_SOURCE is required but not set"); + + return string.Join(" ", sourceString.Split(',').Select(s => $"--source {s}")); + } + } +} diff --git a/test/Cli.FunctionalTests/README.md b/test/Cli.FunctionalTests/README.md new file mode 100644 index 0000000000..4587c251c1 --- /dev/null +++ b/test/Cli.FunctionalTests/README.md @@ -0,0 +1,5 @@ +# Cli.FunctionalTests + +This folder contains tests for ASP.NET Core scenarios in the .NET Core CLI. + +This tests in this folder is meant to be kept in isolation from the rest of the repo, and are not invoked during the course of a regular build. diff --git a/test/Cli.FunctionalTests/RuntimeIdentifier.cs b/test/Cli.FunctionalTests/RuntimeIdentifier.cs new file mode 100644 index 0000000000..7e638a6935 --- /dev/null +++ b/test/Cli.FunctionalTests/RuntimeIdentifier.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Cli.FunctionalTests +{ + // https://docs.microsoft.com/en-us/dotnet/core/rid-catalog + public class RuntimeIdentifier + { + public static RuntimeIdentifier None = new RuntimeIdentifier() { + Name = "none", + OSPlatforms = new[] { OSPlatform.Linux, OSPlatform.OSX, OSPlatform.Windows, }, + }; + + public static RuntimeIdentifier Linux_x64 = new RuntimeIdentifier() { + Name = "linux-x64", + OSPlatforms = new[] { OSPlatform.Linux, }, + ExecutableFileExtension = string.Empty, + }; + + public static RuntimeIdentifier OSX_x64 = new RuntimeIdentifier() + { + Name = "osx-x64", + OSPlatforms = new[] { OSPlatform.OSX, }, + ExecutableFileExtension = string.Empty, + }; + + public static RuntimeIdentifier Win_x64 = new RuntimeIdentifier() + { + Name = "win-x64", + OSPlatforms = new[] { OSPlatform.Windows, }, + ExecutableFileExtension = ".exe", + }; + + public static IEnumerable All = new[] + { + RuntimeIdentifier.None, + RuntimeIdentifier.Linux_x64, + RuntimeIdentifier.OSX_x64, + RuntimeIdentifier.Win_x64, + }; + + private RuntimeIdentifier() { } + + public string Name { get; private set; } + public string RuntimeArgument => (this == None) ? string.Empty : $"--runtime {Name}"; + public string Path => (this == None) ? string.Empty : Name; + public IEnumerable OSPlatforms { get; private set; } + public string ExecutableFileExtension { get; private set; } + + public override string ToString() => Name; + } +} diff --git a/test/Cli.FunctionalTests/TemplateTests.cs b/test/Cli.FunctionalTests/TemplateTests.cs new file mode 100644 index 0000000000..7e8143114f --- /dev/null +++ b/test/Cli.FunctionalTests/TemplateTests.cs @@ -0,0 +1,220 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Runtime.InteropServices; +using Cli.FunctionalTests.Templates; +using Cli.FunctionalTests.Util; +using NuGet.Versioning; +using NUnit.Framework; + +namespace Cli.FunctionalTests +{ + [TestFixture] + public class TemplateTests + { + [Test] + [TestCaseSource(nameof(RestoreData))] + public void _1_Restore(Template template) + { + var expected = template.ExpectedObjFilesAfterRestore; + var actual = template.ObjFilesAfterRestore; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(RestoreData))] + public void _2_RestoreIncremental(Template template) + { + var expected = template.ExpectedObjFilesAfterRestore; + var actual = template.ObjFilesAfterRestoreIncremental; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(BuildData))] + public void _3_Build(Template template) + { + var expectedObj = template.ExpectedObjFilesAfterBuild; + var actualObj = template.ObjFilesAfterBuild; + CollectionAssert.AreEquivalent(expectedObj, actualObj); + + var expectedBin = template.ExpectedBinFilesAfterBuild; + var actualBin = template.BinFilesAfterBuild; + CollectionAssert.AreEquivalent(expectedBin, actualBin); + } + + [Test] + [TestCaseSource(nameof(BuildData))] + public void _4_BuildIncremental(Template template) + { + var expectedObj = template.ExpectedObjFilesAfterBuild; + var actualObj = template.ObjFilesAfterBuildIncremental; + CollectionAssert.AreEquivalent(expectedObj, actualObj); + + var expectedBin = template.ExpectedBinFilesAfterBuild; + var actualBin = template.BinFilesAfterBuildIncremental; + CollectionAssert.AreEquivalent(expectedBin, actualBin); + } + + [Test] + [TestCaseSource(nameof(RunData))] + public void _5_Run(Template template) + { + var statusCode = template.HttpResponseAfterRun.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterRun, template.ServerErrorAfterRun)); + + statusCode = template.HttpsResponseAfterRun.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterRun, template.ServerErrorAfterRun)); + } + + [NonParallelizable] + [Test] + [TestCaseSource(nameof(RunNonParallelizableData))] + public void _5_RunNonParallelizable(Template template) + { + _5_Run(template); + } + + [Test] + [TestCaseSource(nameof(PublishData))] + public void _6_Publish(Template template) + { + var expected = template.ExpectedFilesAfterPublish; + var actual = template.FilesAfterPublish; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(PublishData))] + public void _7_PublishIncremental(Template template) + { + var expected = template.ExpectedFilesAfterPublish; + var actual = template.FilesAfterPublishIncremental; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(ExecData))] + public void _8_Exec(Template template) + { + var statusCode = template.HttpResponseAfterExec.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterExec, template.ServerErrorAfterExec)); + + statusCode = template.HttpsResponseAfterExec.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterExec, template.ServerErrorAfterExec)); + } + + private static string GetMessage(HttpStatusCode statusCode, string serverOutput, string serverError) + { + return String.Join(Environment.NewLine, + $"StatusCode: {statusCode}", + string.Empty, + "ServerOutput", + "------------", + serverOutput, + string.Empty, + "ServerError", + "------------", + serverError); + } + + private static IEnumerable