From 1635034fee5622bb850ed0e52efd41b059f31cce Mon Sep 17 00:00:00 2001 From: "ASP.NET CI" Date: Wed, 26 Sep 2018 02:44:53 +0000 Subject: [PATCH 1/3] Updating submodule(s) Common => a1aa6090a60f51419a20e8e23dd5ee0b08c0f8e3 [auto-updated: submodules] --- modules/Common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Common b/modules/Common index 6812ed2ea7..a1aa6090a6 160000 --- a/modules/Common +++ b/modules/Common @@ -1 +1 @@ -Subproject commit 6812ed2ea751ac0c8ad0977b2e38706032b7f468 +Subproject commit a1aa6090a60f51419a20e8e23dd5ee0b08c0f8e3 From a624afdf8e6019dd4c23ce84f58147714b2bd652 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Wed, 26 Sep 2018 10:51:03 -0700 Subject: [PATCH 2/3] Functional tests for .NET Core CLI 2.1.X (#1415) --- .vsts/builds/e2e-tests.yml | 58 +++ test/Cli.FunctionalTests/AssemblyInfo.cs | 26 ++ .../Cli.FunctionalTests.csproj | 21 + .../Cli.FunctionalTests.sln | 30 ++ .../Cli.FunctionalTests/Directory.Build.props | 2 + test/Cli.FunctionalTests/Directory.Build.rsp | 0 .../Directory.Build.targets | 2 + test/Cli.FunctionalTests/NuGet.config | 6 + .../Cli.FunctionalTests/NuGetPackageSource.cs | 57 +++ test/Cli.FunctionalTests/README.md | 5 + test/Cli.FunctionalTests/RuntimeIdentifier.cs | 55 +++ test/Cli.FunctionalTests/TemplateTests.cs | 220 ++++++++++ .../Templates/AngularTemplate.cs | 46 +++ .../Templates/ClassLibraryTemplate.cs | 48 +++ .../Templates/ConsoleApplicationTemplate.cs | 375 ++++++++++++++++++ .../Templates/MvcTemplate.cs | 35 ++ .../Templates/RazorApplicationBaseTemplate.cs | 61 +++ .../Templates/RazorBootstrapJQueryTemplate.cs | 60 +++ .../Templates/RazorClassLibraryTemplate.cs | 359 +++++++++++++++++ .../Templates/RazorTemplate.cs | 35 ++ .../Templates/RazorUtil.cs | 39 ++ .../Templates/ReactReduxTemplate.cs | 12 + .../Templates/ReactTemplate.cs | 45 +++ .../Templates/SpaBaseTemplate.cs | 21 + .../Cli.FunctionalTests/Templates/Template.cs | 248 ++++++++++++ .../Templates/TemplateType.cs | 12 + .../Templates/WebApiTemplate.cs | 25 ++ .../Templates/WebTemplate.cs | 227 +++++++++++ .../Util/ConcurrentStringBuilder.cs | 37 ++ test/Cli.FunctionalTests/Util/DotNetUtil.cs | 243 ++++++++++++ test/Cli.FunctionalTests/Util/IOUtil.cs | 85 ++++ .../Cli.FunctionalTests/Util/ProcessHelper.cs | 113 ++++++ test/Cli.FunctionalTests/run-tests.ps1 | 169 ++++++++ test/Cli.FunctionalTests/test.sh | 3 + 34 files changed, 2780 insertions(+) create mode 100644 .vsts/builds/e2e-tests.yml create mode 100644 test/Cli.FunctionalTests/AssemblyInfo.cs create mode 100644 test/Cli.FunctionalTests/Cli.FunctionalTests.csproj create mode 100644 test/Cli.FunctionalTests/Cli.FunctionalTests.sln create mode 100644 test/Cli.FunctionalTests/Directory.Build.props create mode 100644 test/Cli.FunctionalTests/Directory.Build.rsp create mode 100644 test/Cli.FunctionalTests/Directory.Build.targets create mode 100644 test/Cli.FunctionalTests/NuGet.config create mode 100644 test/Cli.FunctionalTests/NuGetPackageSource.cs create mode 100644 test/Cli.FunctionalTests/README.md create mode 100644 test/Cli.FunctionalTests/RuntimeIdentifier.cs create mode 100644 test/Cli.FunctionalTests/TemplateTests.cs create mode 100644 test/Cli.FunctionalTests/Templates/AngularTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/ClassLibraryTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/ConsoleApplicationTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/MvcTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/RazorApplicationBaseTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/RazorBootstrapJQueryTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/RazorClassLibraryTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/RazorTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/RazorUtil.cs create mode 100644 test/Cli.FunctionalTests/Templates/ReactReduxTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/ReactTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/SpaBaseTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/Template.cs create mode 100644 test/Cli.FunctionalTests/Templates/TemplateType.cs create mode 100644 test/Cli.FunctionalTests/Templates/WebApiTemplate.cs create mode 100644 test/Cli.FunctionalTests/Templates/WebTemplate.cs create mode 100644 test/Cli.FunctionalTests/Util/ConcurrentStringBuilder.cs create mode 100644 test/Cli.FunctionalTests/Util/DotNetUtil.cs create mode 100644 test/Cli.FunctionalTests/Util/IOUtil.cs create mode 100644 test/Cli.FunctionalTests/Util/ProcessHelper.cs create mode 100644 test/Cli.FunctionalTests/run-tests.ps1 create mode 100644 test/Cli.FunctionalTests/test.sh diff --git a/.vsts/builds/e2e-tests.yml b/.vsts/builds/e2e-tests.yml new file mode 100644 index 0000000000..6019a47184 --- /dev/null +++ b/.vsts/builds/e2e-tests.yml @@ -0,0 +1,58 @@ +trigger: none +phases: +- phase: Host_Windows + queue: + name: DotNetCore-Windows + parallel: 4 + matrix: + Portable: + Test.RuntimeIdentifier: none + SelfContainedWindows: + Test.RuntimeIdentifier: win-x64 + SelfContainedLinux: + Test.RuntimeIdentifier: linux-x64 + SelfContainedMacOs: + Test.RuntimeIdentifier: osx-x64 + steps: + - task: NodeTool@0 + displayName: Install Node 10.x + inputs: + versionSpec: 10.x + - powershell: | + test/Cli.FunctionalTests/run-tests.ps1 -ci -ProdConManifestUrl $env:PRODCONMANIFESTURL -TestRuntimeIdentifier $(Test.RuntimeIdentifier) + condition: ne(variables['PB_SkipTests'], 'true') + displayName: Run E2E tests + - task: PublishTestResults@2 + displayName: Publish test results + condition: always() + inputs: + testRunner: vstest + testResultsFiles: 'artifacts/logs/**/*.trx' +- phase: Host_macOS + queue: + name: Hosted macOS Preview + parallel: 4 + matrix: + Portable: + Test.RuntimeIdentifier: none + SelfContainedWindows: + Test.RuntimeIdentifier: win-x64 + SelfContainedLinux: + Test.RuntimeIdentifier: linux-x64 + SelfContainedMacOs: + Test.RuntimeIdentifier: osx-x64 + steps: + - task: NodeTool@0 + displayName: Install Node 8.x + inputs: + versionSpec: 8.x + - powershell: | + test/Cli.FunctionalTests/run-tests.ps1 -ci -ProdConManifestUrl $env:PRODCONMANIFESTURL -TestRuntimeIdentifier $(Test.RuntimeIdentifier) + condition: ne(variables['PB_SkipTests'], 'true') + displayName: Run E2E tests + - task: PublishTestResults@2 + displayName: Publish test results + condition: always() + inputs: + testRunner: vstest + testResultsFiles: 'artifacts/logs/**/*.trx' diff --git a/test/Cli.FunctionalTests/AssemblyInfo.cs b/test/Cli.FunctionalTests/AssemblyInfo.cs new file mode 100644 index 0000000000..cbe539e977 --- /dev/null +++ b/test/Cli.FunctionalTests/AssemblyInfo.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Cli.FunctionalTests.Util; +using NUnit.Framework; + +// Run all test cases in parallel +[assembly: Parallelizable(ParallelScope.Children)] + +[SetUpFixture] +public class AssemblySetUp +{ + public static string TempDir { get; private set; } + + [OneTimeSetUp] + public void SetUp() + { + TempDir = IOUtil.GetTempDir(); + } + + [OneTimeTearDown] + public void TearDown() + { + IOUtil.DeleteDir(TempDir); + } +} diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj new file mode 100644 index 0000000000..385a9dde2a --- /dev/null +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp2.1 + false + latest + + https://api.nuget.org/v3/index.json; + $(DotNetRestoreSources) + + + + + + + + + + + + diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.sln b/test/Cli.FunctionalTests/Cli.FunctionalTests.sln new file mode 100644 index 0000000000..42f7f90f69 --- /dev/null +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.sln @@ -0,0 +1,30 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28016.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cli.FunctionalTests", "Cli.FunctionalTests.csproj", "{D44EA496-EF83-4D47-8C45-4DAF5A1B0070}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0EC88B4E-B1F2-4183-9EBF-BF66C45D19D7}" + ProjectSection(SolutionItems) = preProject + ..\..\.vsts\builds\e2e-tests.yml = ..\..\.vsts\builds\e2e-tests.yml + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {70432DA7-DCE4-4F73-A00C-E1AB180DDD6A} + EndGlobalSection +EndGlobal diff --git a/test/Cli.FunctionalTests/Directory.Build.props b/test/Cli.FunctionalTests/Directory.Build.props new file mode 100644 index 0000000000..6d087b3032 --- /dev/null +++ b/test/Cli.FunctionalTests/Directory.Build.props @@ -0,0 +1,2 @@ + + diff --git a/test/Cli.FunctionalTests/Directory.Build.rsp b/test/Cli.FunctionalTests/Directory.Build.rsp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/Cli.FunctionalTests/Directory.Build.targets b/test/Cli.FunctionalTests/Directory.Build.targets new file mode 100644 index 0000000000..6d087b3032 --- /dev/null +++ b/test/Cli.FunctionalTests/Directory.Build.targets @@ -0,0 +1,2 @@ + + diff --git a/test/Cli.FunctionalTests/NuGet.config b/test/Cli.FunctionalTests/NuGet.config new file mode 100644 index 0000000000..4bb3170917 --- /dev/null +++ b/test/Cli.FunctionalTests/NuGet.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/test/Cli.FunctionalTests/NuGetPackageSource.cs b/test/Cli.FunctionalTests/NuGetPackageSource.cs new file mode 100644 index 0000000000..30545f8ca5 --- /dev/null +++ b/test/Cli.FunctionalTests/NuGetPackageSource.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; + +namespace Cli.FunctionalTests +{ + public class NuGetPackageSource + { + public static NuGetPackageSource None { get; } = new NuGetPackageSource + { + Name = nameof(None), + SourceArgumentLazy = new Lazy(string.Empty), + }; + + public static NuGetPackageSource NuGetOrg { get; } = new NuGetPackageSource + { + Name = nameof(NuGetOrg), + SourceArgumentLazy = new Lazy("--source https://api.nuget.org/v3/index.json"), + }; + + public static NuGetPackageSource DotNetCore { get; } = new NuGetPackageSource + { + Name = nameof(DotNetCore), + SourceArgumentLazy = new Lazy("--source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json"), + }; + + public static NuGetPackageSource EnvironmentVariable { get; } = new NuGetPackageSource + { + Name = nameof(EnvironmentVariable), + SourceArgumentLazy = new Lazy(() => GetSourceArgumentFromEnvironment()), + }; + + public static NuGetPackageSource EnvironmentVariableAndNuGetOrg { get; } = new NuGetPackageSource + { + Name = nameof(EnvironmentVariableAndNuGetOrg), + SourceArgumentLazy = new Lazy(() => string.Join(" ", EnvironmentVariable.SourceArgument, NuGetOrg.SourceArgument)), + }; + + private NuGetPackageSource() { } + + public string Name { get; private set; } + public string SourceArgument => SourceArgumentLazy.Value; + private Lazy SourceArgumentLazy { get; set; } + + public override string ToString() => Name; + + private static string GetSourceArgumentFromEnvironment() + { + var sourceString = Environment.GetEnvironmentVariable("NUGET_PACKAGE_SOURCE") ?? + throw new InvalidOperationException("Environment variable NUGET_PACKAGE_SOURCE is required but not set"); + + return string.Join(" ", sourceString.Split(',').Select(s => $"--source {s}")); + } + } +} diff --git a/test/Cli.FunctionalTests/README.md b/test/Cli.FunctionalTests/README.md new file mode 100644 index 0000000000..4587c251c1 --- /dev/null +++ b/test/Cli.FunctionalTests/README.md @@ -0,0 +1,5 @@ +# Cli.FunctionalTests + +This folder contains tests for ASP.NET Core scenarios in the .NET Core CLI. + +This tests in this folder is meant to be kept in isolation from the rest of the repo, and are not invoked during the course of a regular build. diff --git a/test/Cli.FunctionalTests/RuntimeIdentifier.cs b/test/Cli.FunctionalTests/RuntimeIdentifier.cs new file mode 100644 index 0000000000..7e638a6935 --- /dev/null +++ b/test/Cli.FunctionalTests/RuntimeIdentifier.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Cli.FunctionalTests +{ + // https://docs.microsoft.com/en-us/dotnet/core/rid-catalog + public class RuntimeIdentifier + { + public static RuntimeIdentifier None = new RuntimeIdentifier() { + Name = "none", + OSPlatforms = new[] { OSPlatform.Linux, OSPlatform.OSX, OSPlatform.Windows, }, + }; + + public static RuntimeIdentifier Linux_x64 = new RuntimeIdentifier() { + Name = "linux-x64", + OSPlatforms = new[] { OSPlatform.Linux, }, + ExecutableFileExtension = string.Empty, + }; + + public static RuntimeIdentifier OSX_x64 = new RuntimeIdentifier() + { + Name = "osx-x64", + OSPlatforms = new[] { OSPlatform.OSX, }, + ExecutableFileExtension = string.Empty, + }; + + public static RuntimeIdentifier Win_x64 = new RuntimeIdentifier() + { + Name = "win-x64", + OSPlatforms = new[] { OSPlatform.Windows, }, + ExecutableFileExtension = ".exe", + }; + + public static IEnumerable All = new[] + { + RuntimeIdentifier.None, + RuntimeIdentifier.Linux_x64, + RuntimeIdentifier.OSX_x64, + RuntimeIdentifier.Win_x64, + }; + + private RuntimeIdentifier() { } + + public string Name { get; private set; } + public string RuntimeArgument => (this == None) ? string.Empty : $"--runtime {Name}"; + public string Path => (this == None) ? string.Empty : Name; + public IEnumerable OSPlatforms { get; private set; } + public string ExecutableFileExtension { get; private set; } + + public override string ToString() => Name; + } +} diff --git a/test/Cli.FunctionalTests/TemplateTests.cs b/test/Cli.FunctionalTests/TemplateTests.cs new file mode 100644 index 0000000000..7e8143114f --- /dev/null +++ b/test/Cli.FunctionalTests/TemplateTests.cs @@ -0,0 +1,220 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Runtime.InteropServices; +using Cli.FunctionalTests.Templates; +using Cli.FunctionalTests.Util; +using NuGet.Versioning; +using NUnit.Framework; + +namespace Cli.FunctionalTests +{ + [TestFixture] + public class TemplateTests + { + [Test] + [TestCaseSource(nameof(RestoreData))] + public void _1_Restore(Template template) + { + var expected = template.ExpectedObjFilesAfterRestore; + var actual = template.ObjFilesAfterRestore; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(RestoreData))] + public void _2_RestoreIncremental(Template template) + { + var expected = template.ExpectedObjFilesAfterRestore; + var actual = template.ObjFilesAfterRestoreIncremental; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(BuildData))] + public void _3_Build(Template template) + { + var expectedObj = template.ExpectedObjFilesAfterBuild; + var actualObj = template.ObjFilesAfterBuild; + CollectionAssert.AreEquivalent(expectedObj, actualObj); + + var expectedBin = template.ExpectedBinFilesAfterBuild; + var actualBin = template.BinFilesAfterBuild; + CollectionAssert.AreEquivalent(expectedBin, actualBin); + } + + [Test] + [TestCaseSource(nameof(BuildData))] + public void _4_BuildIncremental(Template template) + { + var expectedObj = template.ExpectedObjFilesAfterBuild; + var actualObj = template.ObjFilesAfterBuildIncremental; + CollectionAssert.AreEquivalent(expectedObj, actualObj); + + var expectedBin = template.ExpectedBinFilesAfterBuild; + var actualBin = template.BinFilesAfterBuildIncremental; + CollectionAssert.AreEquivalent(expectedBin, actualBin); + } + + [Test] + [TestCaseSource(nameof(RunData))] + public void _5_Run(Template template) + { + var statusCode = template.HttpResponseAfterRun.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterRun, template.ServerErrorAfterRun)); + + statusCode = template.HttpsResponseAfterRun.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterRun, template.ServerErrorAfterRun)); + } + + [NonParallelizable] + [Test] + [TestCaseSource(nameof(RunNonParallelizableData))] + public void _5_RunNonParallelizable(Template template) + { + _5_Run(template); + } + + [Test] + [TestCaseSource(nameof(PublishData))] + public void _6_Publish(Template template) + { + var expected = template.ExpectedFilesAfterPublish; + var actual = template.FilesAfterPublish; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(PublishData))] + public void _7_PublishIncremental(Template template) + { + var expected = template.ExpectedFilesAfterPublish; + var actual = template.FilesAfterPublishIncremental; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(ExecData))] + public void _8_Exec(Template template) + { + var statusCode = template.HttpResponseAfterExec.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterExec, template.ServerErrorAfterExec)); + + statusCode = template.HttpsResponseAfterExec.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterExec, template.ServerErrorAfterExec)); + } + + private static string GetMessage(HttpStatusCode statusCode, string serverOutput, string serverError) + { + return String.Join(Environment.NewLine, + $"StatusCode: {statusCode}", + string.Empty, + "ServerOutput", + "------------", + serverOutput, + string.Empty, + "ServerError", + "------------", + serverError); + } + + private static IEnumerable