diff --git a/test/Cli.FunctionalTests/.gitignore b/test/Cli.FunctionalTests/.gitignore new file mode 100644 index 0000000000..940794e60f --- /dev/null +++ b/test/Cli.FunctionalTests/.gitignore @@ -0,0 +1,288 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Typescript v1 declaration files +typings/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs diff --git a/test/Cli.FunctionalTests/AssemblyInfo.cs b/test/Cli.FunctionalTests/AssemblyInfo.cs new file mode 100644 index 0000000000..cbe539e977 --- /dev/null +++ b/test/Cli.FunctionalTests/AssemblyInfo.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Cli.FunctionalTests.Util; +using NUnit.Framework; + +// Run all test cases in parallel +[assembly: Parallelizable(ParallelScope.Children)] + +[SetUpFixture] +public class AssemblySetUp +{ + public static string TempDir { get; private set; } + + [OneTimeSetUp] + public void SetUp() + { + TempDir = IOUtil.GetTempDir(); + } + + [OneTimeTearDown] + public void TearDown() + { + IOUtil.DeleteDir(TempDir); + } +} diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj new file mode 100644 index 0000000000..67493a7ea2 --- /dev/null +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp2.1 + false + + + + latest + + + + + + + + + + + diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.sln b/test/Cli.FunctionalTests/Cli.FunctionalTests.sln new file mode 100644 index 0000000000..6aff0f5478 --- /dev/null +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28016.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cli.FunctionalTests", "Cli.FunctionalTests.csproj", "{D44EA496-EF83-4D47-8C45-4DAF5A1B0070}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D44EA496-EF83-4D47-8C45-4DAF5A1B0070}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {70432DA7-DCE4-4F73-A00C-E1AB180DDD6A} + EndGlobalSection +EndGlobal diff --git a/test/Cli.FunctionalTests/NuGetPackageSource.cs b/test/Cli.FunctionalTests/NuGetPackageSource.cs new file mode 100644 index 0000000000..30545f8ca5 --- /dev/null +++ b/test/Cli.FunctionalTests/NuGetPackageSource.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; + +namespace Cli.FunctionalTests +{ + public class NuGetPackageSource + { + public static NuGetPackageSource None { get; } = new NuGetPackageSource + { + Name = nameof(None), + SourceArgumentLazy = new Lazy(string.Empty), + }; + + public static NuGetPackageSource NuGetOrg { get; } = new NuGetPackageSource + { + Name = nameof(NuGetOrg), + SourceArgumentLazy = new Lazy("--source https://api.nuget.org/v3/index.json"), + }; + + public static NuGetPackageSource DotNetCore { get; } = new NuGetPackageSource + { + Name = nameof(DotNetCore), + SourceArgumentLazy = new Lazy("--source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json"), + }; + + public static NuGetPackageSource EnvironmentVariable { get; } = new NuGetPackageSource + { + Name = nameof(EnvironmentVariable), + SourceArgumentLazy = new Lazy(() => GetSourceArgumentFromEnvironment()), + }; + + public static NuGetPackageSource EnvironmentVariableAndNuGetOrg { get; } = new NuGetPackageSource + { + Name = nameof(EnvironmentVariableAndNuGetOrg), + SourceArgumentLazy = new Lazy(() => string.Join(" ", EnvironmentVariable.SourceArgument, NuGetOrg.SourceArgument)), + }; + + private NuGetPackageSource() { } + + public string Name { get; private set; } + public string SourceArgument => SourceArgumentLazy.Value; + private Lazy SourceArgumentLazy { get; set; } + + public override string ToString() => Name; + + private static string GetSourceArgumentFromEnvironment() + { + var sourceString = Environment.GetEnvironmentVariable("NUGET_PACKAGE_SOURCE") ?? + throw new InvalidOperationException("Environment variable NUGET_PACKAGE_SOURCE is required but not set"); + + return string.Join(" ", sourceString.Split(',').Select(s => $"--source {s}")); + } + } +} diff --git a/test/Cli.FunctionalTests/README.md b/test/Cli.FunctionalTests/README.md new file mode 100644 index 0000000000..e1f49f83ac --- /dev/null +++ b/test/Cli.FunctionalTests/README.md @@ -0,0 +1 @@ +# Cli.FunctionalTests diff --git a/test/Cli.FunctionalTests/RuntimeIdentifier.cs b/test/Cli.FunctionalTests/RuntimeIdentifier.cs new file mode 100644 index 0000000000..7e638a6935 --- /dev/null +++ b/test/Cli.FunctionalTests/RuntimeIdentifier.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Cli.FunctionalTests +{ + // https://docs.microsoft.com/en-us/dotnet/core/rid-catalog + public class RuntimeIdentifier + { + public static RuntimeIdentifier None = new RuntimeIdentifier() { + Name = "none", + OSPlatforms = new[] { OSPlatform.Linux, OSPlatform.OSX, OSPlatform.Windows, }, + }; + + public static RuntimeIdentifier Linux_x64 = new RuntimeIdentifier() { + Name = "linux-x64", + OSPlatforms = new[] { OSPlatform.Linux, }, + ExecutableFileExtension = string.Empty, + }; + + public static RuntimeIdentifier OSX_x64 = new RuntimeIdentifier() + { + Name = "osx-x64", + OSPlatforms = new[] { OSPlatform.OSX, }, + ExecutableFileExtension = string.Empty, + }; + + public static RuntimeIdentifier Win_x64 = new RuntimeIdentifier() + { + Name = "win-x64", + OSPlatforms = new[] { OSPlatform.Windows, }, + ExecutableFileExtension = ".exe", + }; + + public static IEnumerable All = new[] + { + RuntimeIdentifier.None, + RuntimeIdentifier.Linux_x64, + RuntimeIdentifier.OSX_x64, + RuntimeIdentifier.Win_x64, + }; + + private RuntimeIdentifier() { } + + public string Name { get; private set; } + public string RuntimeArgument => (this == None) ? string.Empty : $"--runtime {Name}"; + public string Path => (this == None) ? string.Empty : Name; + public IEnumerable OSPlatforms { get; private set; } + public string ExecutableFileExtension { get; private set; } + + public override string ToString() => Name; + } +} diff --git a/test/Cli.FunctionalTests/TemplateTests.cs b/test/Cli.FunctionalTests/TemplateTests.cs new file mode 100644 index 0000000000..7e8143114f --- /dev/null +++ b/test/Cli.FunctionalTests/TemplateTests.cs @@ -0,0 +1,220 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Runtime.InteropServices; +using Cli.FunctionalTests.Templates; +using Cli.FunctionalTests.Util; +using NuGet.Versioning; +using NUnit.Framework; + +namespace Cli.FunctionalTests +{ + [TestFixture] + public class TemplateTests + { + [Test] + [TestCaseSource(nameof(RestoreData))] + public void _1_Restore(Template template) + { + var expected = template.ExpectedObjFilesAfterRestore; + var actual = template.ObjFilesAfterRestore; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(RestoreData))] + public void _2_RestoreIncremental(Template template) + { + var expected = template.ExpectedObjFilesAfterRestore; + var actual = template.ObjFilesAfterRestoreIncremental; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(BuildData))] + public void _3_Build(Template template) + { + var expectedObj = template.ExpectedObjFilesAfterBuild; + var actualObj = template.ObjFilesAfterBuild; + CollectionAssert.AreEquivalent(expectedObj, actualObj); + + var expectedBin = template.ExpectedBinFilesAfterBuild; + var actualBin = template.BinFilesAfterBuild; + CollectionAssert.AreEquivalent(expectedBin, actualBin); + } + + [Test] + [TestCaseSource(nameof(BuildData))] + public void _4_BuildIncremental(Template template) + { + var expectedObj = template.ExpectedObjFilesAfterBuild; + var actualObj = template.ObjFilesAfterBuildIncremental; + CollectionAssert.AreEquivalent(expectedObj, actualObj); + + var expectedBin = template.ExpectedBinFilesAfterBuild; + var actualBin = template.BinFilesAfterBuildIncremental; + CollectionAssert.AreEquivalent(expectedBin, actualBin); + } + + [Test] + [TestCaseSource(nameof(RunData))] + public void _5_Run(Template template) + { + var statusCode = template.HttpResponseAfterRun.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterRun, template.ServerErrorAfterRun)); + + statusCode = template.HttpsResponseAfterRun.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterRun, template.ServerErrorAfterRun)); + } + + [NonParallelizable] + [Test] + [TestCaseSource(nameof(RunNonParallelizableData))] + public void _5_RunNonParallelizable(Template template) + { + _5_Run(template); + } + + [Test] + [TestCaseSource(nameof(PublishData))] + public void _6_Publish(Template template) + { + var expected = template.ExpectedFilesAfterPublish; + var actual = template.FilesAfterPublish; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(PublishData))] + public void _7_PublishIncremental(Template template) + { + var expected = template.ExpectedFilesAfterPublish; + var actual = template.FilesAfterPublishIncremental; + CollectionAssert.AreEquivalent(expected, actual); + } + + [Test] + [TestCaseSource(nameof(ExecData))] + public void _8_Exec(Template template) + { + var statusCode = template.HttpResponseAfterExec.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterExec, template.ServerErrorAfterExec)); + + statusCode = template.HttpsResponseAfterExec.StatusCode; + Assert.AreEqual(HttpStatusCode.OK, statusCode, + GetMessage(statusCode, template.ServerOutputAfterExec, template.ServerErrorAfterExec)); + } + + private static string GetMessage(HttpStatusCode statusCode, string serverOutput, string serverError) + { + return String.Join(Environment.NewLine, + $"StatusCode: {statusCode}", + string.Empty, + "ServerOutput", + "------------", + serverOutput, + string.Empty, + "ServerError", + "------------", + serverError); + } + + private static IEnumerable