From 435867e8c5c43f7837468d44c0564effccbb7f07 Mon Sep 17 00:00:00 2001 From: Hao Kung Date: Wed, 6 Mar 2019 23:53:18 -0800 Subject: [PATCH] Add ability to skip tests on specific helix queues (#8231) --- eng/helix/vstest/runtests.cmd | 3 ++- eng/helix/vstest/runtests.sh | 2 +- eng/helix/xunit/runtests.cmd | 2 +- eng/targets/Helix.targets | 4 ++-- src/Shared/test/SkipOnHelixAttribute.cs | 14 ++++++++++---- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/eng/helix/vstest/runtests.cmd b/eng/helix/vstest/runtests.cmd index ae1ebac165..91afad3210 100644 --- a/eng/helix/vstest/runtests.cmd +++ b/eng/helix/vstest/runtests.cmd @@ -1,6 +1,7 @@ set target=%1 set sdkVersion=%2 set runtimeVersion=%3 +set helixQueue=%4 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk set DOTNET_ROOT=%DOTNET_HOME%\x64 @@ -13,7 +14,7 @@ set PATH=%DOTNET_ROOT%;%PATH% powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture x64 -Version %sdkVersion% -InstallDir %DOTNET_ROOT%" powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture x64 -Runtime dotnet -Version %runtimeVersion% -InstallDir %DOTNET_ROOT%" -set HELIX=true +set HELIX=%helixQueue% %DOTNET_ROOT%\dotnet vstest %target% -lt >discovered.txt find /c "Exception thrown" discovered.txt diff --git a/eng/helix/vstest/runtests.sh b/eng/helix/vstest/runtests.sh index 904d12abbf..776db87fe9 100644 --- a/eng/helix/vstest/runtests.sh +++ b/eng/helix/vstest/runtests.sh @@ -58,7 +58,7 @@ export DOTNET_MULTILEVEL_LOOKUP=0 # Avoid contaminating userprofiles export DOTNET_CLI_HOME="$HELIX_CORRELATION_PAYLOAD/home" -export helix="true" +export helix="$4" $DOTNET_ROOT/dotnet vstest $1 -lt >discovered.txt if grep -q "Exception thrown" discovered.txt; then diff --git a/eng/helix/xunit/runtests.cmd b/eng/helix/xunit/runtests.cmd index 350399c1af..2c58f15196 100644 --- a/eng/helix/xunit/runtests.cmd +++ b/eng/helix/xunit/runtests.cmd @@ -1,3 +1,3 @@ set target=%1 -set helix=true +set helix=%4 xunit.console.exe %target% -xml testResults.xml diff --git a/eng/targets/Helix.targets b/eng/targets/Helix.targets index 6860135ac7..d60460b8b4 100644 --- a/eng/targets/Helix.targets +++ b/eng/targets/Helix.targets @@ -64,8 +64,8 @@ $(TargetFileName) @(HelixPreCommand) @(HelixPostCommand) - call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) - ./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) + call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) $(HelixTargetQueue) + ./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) $(HelixTargetQueue) $(HelixTimeout) diff --git a/src/Shared/test/SkipOnHelixAttribute.cs b/src/Shared/test/SkipOnHelixAttribute.cs index f33f1a64d9..2ad4018acc 100644 --- a/src/Shared/test/SkipOnHelixAttribute.cs +++ b/src/Shared/test/SkipOnHelixAttribute.cs @@ -2,12 +2,12 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Linq; namespace Microsoft.AspNetCore.Testing.xunit { /// - /// Skip test if a given environment variable is not enabled. To enable the test, set environment variable - /// to "true" for the test process. + /// Skip test if running on helix (or a particular helix queue). /// [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)] public class SkipOnHelixAttribute : Attribute, ITestCondition @@ -16,10 +16,14 @@ namespace Microsoft.AspNetCore.Testing.xunit { get { - return !OnHelix(); + var skip = OnHelix() && (Queues == null || Queues.ToLowerInvariant().Split(";").Contains(GetTargetHelixQueue().ToLowerInvariant())); + return !skip; } } + // Queues that should be skipped on, i.e. "Windows.10.Amd64.ClientRS4.VS2017.Open;OSX.1012.Amd64.Open" + public string Queues { get; set; } + public string SkipReason { get @@ -28,6 +32,8 @@ namespace Microsoft.AspNetCore.Testing.xunit } } - public static bool OnHelix() => string.Equals(Environment.GetEnvironmentVariable("helix"), "true", StringComparison.OrdinalIgnoreCase); + public static bool OnHelix() => !string.IsNullOrEmpty(GetTargetHelixQueue()); + + public static string GetTargetHelixQueue() => Environment.GetEnvironmentVariable("helix"); } }