From f6fc60a0edd0dc844b14e125dd880b40bbc06b48 Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Wed, 29 Aug 2018 19:37:09 -0700 Subject: [PATCH] Add script to invoke E2E tests on CI using ProdCon results (#1354) --- .vsts/builds/e2e-tests.yml | 25 ++++++ .../Cli.FunctionalTests.csproj | 7 +- .../Cli.FunctionalTests/Directory.Build.props | 2 + test/Cli.FunctionalTests/Directory.Build.rsp | 0 .../Directory.Build.targets | 2 + test/Cli.FunctionalTests/NuGet.config | 6 ++ test/Cli.FunctionalTests/README.md | 4 + test/Cli.FunctionalTests/run-tests.ps1 | 84 +++++++++++++++++++ 8 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 .vsts/builds/e2e-tests.yml create mode 100644 test/Cli.FunctionalTests/Directory.Build.props create mode 100644 test/Cli.FunctionalTests/Directory.Build.rsp create mode 100644 test/Cli.FunctionalTests/Directory.Build.targets create mode 100644 test/Cli.FunctionalTests/NuGet.config create mode 100644 test/Cli.FunctionalTests/run-tests.ps1 diff --git a/.vsts/builds/e2e-tests.yml b/.vsts/builds/e2e-tests.yml new file mode 100644 index 0000000000..6ec1d8bfd0 --- /dev/null +++ b/.vsts/builds/e2e-tests.yml @@ -0,0 +1,25 @@ +phases: +- phase: Windows + queue: + name: DotNetCore-Windows + timeoutInMinutes: 120 + variables: + CI: true + steps: + - task: NodeTool@0 + displayName: Install Node 10.x + inputs: + versionSpec: 10.x + - powershell: < + test/Cli.FunctionalTests/run-tests.ps1 + -AssetRootUrl $(PB_AssetRootUrl) + -RestoreSources $(PB_RestoreSource) + -PackageVersionsFile $(PB_PackageVersionPropsUrl) + condition: ne(variables['PB_SKipTests'], 'true') + displayName: Run e2e tests + - task: PublishTestResults@2 + displayName: Publish test results + condition: always() + inputs: + testRunner: vstest + testResultsFiles: 'artifacts/logs/**/*.trx' diff --git a/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj index 67493a7ea2..385a9dde2a 100644 --- a/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj +++ b/test/Cli.FunctionalTests/Cli.FunctionalTests.csproj @@ -3,10 +3,11 @@ netcoreapp2.1 false - - - latest + + https://api.nuget.org/v3/index.json; + $(DotNetRestoreSources) + diff --git a/test/Cli.FunctionalTests/Directory.Build.props b/test/Cli.FunctionalTests/Directory.Build.props new file mode 100644 index 0000000000..6d087b3032 --- /dev/null +++ b/test/Cli.FunctionalTests/Directory.Build.props @@ -0,0 +1,2 @@ + + diff --git a/test/Cli.FunctionalTests/Directory.Build.rsp b/test/Cli.FunctionalTests/Directory.Build.rsp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/Cli.FunctionalTests/Directory.Build.targets b/test/Cli.FunctionalTests/Directory.Build.targets new file mode 100644 index 0000000000..6d087b3032 --- /dev/null +++ b/test/Cli.FunctionalTests/Directory.Build.targets @@ -0,0 +1,2 @@ + + diff --git a/test/Cli.FunctionalTests/NuGet.config b/test/Cli.FunctionalTests/NuGet.config new file mode 100644 index 0000000000..4bb3170917 --- /dev/null +++ b/test/Cli.FunctionalTests/NuGet.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/test/Cli.FunctionalTests/README.md b/test/Cli.FunctionalTests/README.md index e1f49f83ac..4587c251c1 100644 --- a/test/Cli.FunctionalTests/README.md +++ b/test/Cli.FunctionalTests/README.md @@ -1 +1,5 @@ # Cli.FunctionalTests + +This folder contains tests for ASP.NET Core scenarios in the .NET Core CLI. + +This tests in this folder is meant to be kept in isolation from the rest of the repo, and are not invoked during the course of a regular build. diff --git a/test/Cli.FunctionalTests/run-tests.ps1 b/test/Cli.FunctionalTests/run-tests.ps1 new file mode 100644 index 0000000000..cd2e285c1d --- /dev/null +++ b/test/Cli.FunctionalTests/run-tests.ps1 @@ -0,0 +1,84 @@ +<# +.SYNOPSIS +This script runs the tests in this project on complete build of the .NET Core CLI + +.PARAMETER AssetRootUrl +The blob feed for the .NET Core CLI + +.PARAMETER RestoreSources +A list of additional NuGet feeds + +.PARAMETER SdkVersion +The version of the .NET Core CLI to test. If not specified, the version will be determined automatically if possible. + +.PARAMETER PackageVersionsFile +A URL or filepath to a list of package versions +#> +param( + $AssetRootUrl = 'https://dotnetcli.blob.core.windows.net/dotnet', + $RestoreSources = 'https://dotnet.myget.org/F/dotnet-core/api/v3/index.json', + $SdkVersion = $null, + $PackageVersionsFile = $null +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 1 + +$repoRoot = Resolve-Path "$PSScriptRoot/../../" +Import-Module "$repoRoot/scripts/common.psm1" -Scope Local -Force + +$AssetRootUrl = $AssetRootUrl.TrimEnd('/') + +Push-Location $PSScriptRoot +try { + New-Item -Type Directory "$PSScriptRoot/obj/" -ErrorAction Ignore | Out-Null + + $pkgPropsFile = $PackageVersionsFile + if ($PackageVersionsFile -like 'http*') { + $pkgPropsFile = "$PSScriptRoot/obj/packageversions.props" + Remove-Item $pkgPropsFile -ErrorAction Ignore + Invoke-WebRequest -UseBasicParsing $PackageVersionsFile -OutFile $pkgPropsFile + } + + if (-not $SdkVersion) { + $cliManifestUrl = "$AssetRootUrl/orchestration-metadata/manifests/cli.xml" + Write-Host "No SDK version was specified. Attempting to determine the version from $cliManifestUrl" + $cliXml = "$PSScriptRoot/obj/cli.xml" + Remove-Item $cliXml -ErrorAction Ignore + Invoke-WebRequest -UseBasicParsing $cliManifestUrl -OutFile $cliXml + [xml] $cli = Get-Content $cliXml + $SdkVersion = $cli.Build.ProductVersion + } + + Write-Host "SDK: $SdkVersion" + + @{ sdk = @{ version = $SdkVersion } } | ConvertTo-Json | Set-Content "$PSScriptRoot/global.json" + + $dotnetRoot = "$repoRoot/.dotnet" + $dotnet = "$dotnetRoot/dotnet.exe" + + if (-not (Test-Path "$dotnetRoot/sdk/$SdkVersion/dotnet.dll")) { + Remote-Item -Recurse -Force $dotnetRoot -ErrorAction Ignore | Out-Null + $cliUrl = "$AssetRootUrl/Sdk/$SdkVersion/dotnet-sdk-$SdkVersion-win-x64.zip" + Write-Host "Downloading $cliUrl" + Invoke-WebRequest -UseBasicParsing $cliUrl -OutFile "$PSScriptRoot/obj/dotnet.zip" + Expand-Archive "$PSScriptRoot/obj/dotnet.zip" -DestinationPath $dotnetRoot + } + + # Set a clean test environment + $env:DOTNET_ROOT = $dotnetRoot + $env:DOTNET_MULTILEVEL_LOOKUP = 0 + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 0 + $env:MSBuildSdksPath = '' + $env:PATH="$dotnetRoot;$env:PATH" + + Invoke-Block { & $dotnet test ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFile=$repoRoot/artifacts/logs/e2etests.trx" ` + "-p:DotNetRestoreSources=$RestoreSources" ` + "-p:DotNetPackageVersionPropsPath=$pkgPropsFile" ` + "-bl:$repoRoot/artifacts/logs/e2etests.binlog" } +} +finally { + Pop-Location +}