[Helix] Split flaky helix tests into separate yml (#18597)

.azure/pipelines/helix-test.yml

@@ -0,0 +1,27 @@
+# Don't run CI for this config yet. We're not ready to move official builds on to Azure Pipelines
+trigger: none
+
+# Run PR validation on all branches
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Helix_quarantine_x64
+    jobDisplayName: 'Tests: Helix Quarantine x64'
+    agentOs: Windows
+    timeoutInMinutes: 240
+    steps:
+    - script: .\restore.cmd -ci
+      displayName: Restore
+    - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+      displayName: Run build.cmd helix target
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
+    artifacts:
+    - name: Helix_logs
+      path: artifacts/log/
+      publishOnError: true

eng/helix/content/runtests.cmd

@@ -8,6 +8,7 @@ set sdkVersion=%3
 set runtimeVersion=%4
 set helixQueue=%5
 set arch=%6
+set quarantined=%7
 
 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
 set DOTNET_ROOT=%DOTNET_HOME%\%arch%
@@ -38,25 +39,25 @@ if not errorlevel 1 (
 
 set exit_code=0
 
-REM Run non-flaky tests first
+set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
+if (%quarantined%==true) (
+    echo Running all tests.
+    %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit
+    if errorlevel 1 (
+        echo Failure in flaky test 1>&2
+        REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
+    )
+) else (
     REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
     REM only puts the explicit filter traits the user provided in
     REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
-set NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
+    echo Running non-quarantined tests.
-echo Running non-flaky tests.
+    %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%NONQUARANTINE_FILTER%
-%DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%NONFLAKY_FILTER%
     if errorlevel 1 (
         echo Failure in non-flaky test 1>&2
         set exit_code=1
         REM DO NOT EXIT
     )
-
-set FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
-echo Running known-flaky tests.
-%DOTNET_ROOT%\dotnet vstest %target% --TestCaseFilter:%FLAKY_FILTER%
-if errorlevel 1 (
-    echo Failure in flaky test 1>&2
-    REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
 )
 
 echo "Copying TestResults\TestResults.xml to ."

eng/helix/content/runtests.sh

@@ -4,6 +4,8 @@ test_binary_path="$1"
 dotnet_sdk_version="$2"
 dotnet_runtime_version="$3"
 helix_queue_name="$4"
+target_arch="$5"
+quarantined="$6"
 
 RESET="\033[0m"
 RED="\033[0;31m"
@@ -87,26 +89,28 @@ if grep -q "Exception thrown" discovered.txt; then
     exit 1
 fi
 
-# Run non-flaky tests first
+exit_code=0
-# We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
+
+# We need to specify all possible quarantined filters that apply to this environment, because the quarantine attribute
 # only puts the explicit filter traits the user provided in the flaky attribute
 # Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
-NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
+NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
-echo "Running non-flaky tests."
+if [ "$quarantined" == true ]; then
-$DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$NONFLAKY_FILTER"
+    echo "Running all tests including quarantined."
-nonflaky_exitcode=$?
+    $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit
-if [ $nonflaky_exitcode != 0 ]; then
+    if [ $? != 0 ]; then
-    echo "Non-flaky tests failed!" 1>&2
+        echo "Quarantined tests failed!" 1>&2
         # DO NOT EXIT
     fi
-
+else
-FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true"
+    echo "Running non-quarantined tests."
-echo "Running known-flaky tests."
+    $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$NONQUARANTINE_FILTER"
-$DOTNET_ROOT/dotnet vstest $test_binary_path --TestCaseFilter:"$FLAKY_FILTER"
+    exit_code=$?
-if [ $? != 0 ]; then
+    if [ $exit_code != 0 ]; then
-    echo "Flaky tests failed!" 1>&2
+        echo "Non-quarantined tests failed!" 1>&2
         # DO NOT EXIT
     fi
+fi
 
 echo "Copying TestResults/TestResults to ."
 cp TestResults/TestResults.xml testResults.xml
@@ -114,4 +118,4 @@ echo "Copying artifacts/logs to $HELIX_WORKITEM_UPLOAD_ROOT/../"
 shopt -s globstar
 cp artifacts/log/**/*.log $HELIX_WORKITEM_UPLOAD_ROOT/../
 cp artifacts/log/**/*.log $HELIX_WORKITEM_UPLOAD_ROOT/
-exit $nonflaky_exitcode
+exit $exit_code

eng/targets/Helix.props

@@ -12,9 +12,12 @@
   <PropertyGroup>
     <CreateHelixPayload>true</CreateHelixPayload>
     <HelixTimeout>00:30:00</HelixTimeout>
+    <RunQuarantinedTests>false</RunQuarantinedTests>
     <IsWindowsHelixQueue>false</IsWindowsHelixQueue>
     <IsWindowsHelixQueue Condition="$(HelixTargetQueue.Contains('Windows')) or $(HelixTargetQueue.Contains('windows'))">true</IsWindowsHelixQueue>
-    <HelixTestName>$(MSBuildProjectName)-$(TargetFramework)</HelixTestName>
+    <_HelixRunType Condition="'$(RunQuarantinedTests)' == 'true'">quarantined</_HelixRunType>
+    <_HelixRunType Condition="'$(RunQuarantinedTests)' != 'true'">unquarantined</_HelixRunType>    
+    <HelixTestName>$(MSBuildProjectName)--$(TargetFramework)--$(_HelixRunType)</HelixTestName>
     <HelixUseArchive>false</HelixUseArchive>
     <LoggingTestingDisableFileLogging Condition="'$(IsHelixJob)' == 'true'">false</LoggingTestingDisableFileLogging>
     <NodeVersion>10.15.3</NodeVersion>

eng/targets/Helix.targets

@@ -83,8 +83,8 @@ Usage: dotnet msbuild /t:Helix src/MyTestProject.csproj
         <TestAssembly>$(TargetFileName)</TestAssembly>
         <PreCommands>@(HelixPreCommand)</PreCommands>
         <PostCommands>@(HelixPostCommand)</PostCommands>
-        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(TargetFrameworkIdentifier) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture)</Command>
+        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(TargetFrameworkIdentifier) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests)</Command>
-        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests)</Command>
         <Timeout>$(HelixTimeout)</Timeout>
       </HelixWorkItem>
     </ItemGroup>