Add a separate test pass to run flaky tests that doesn't fail the build (#8486)

Similar to https://github.com/aspnet/Extensions/pull/1239.

.azure/pipelines/ci.yml

@@ -300,6 +300,9 @@ jobs:
     - powershell: "& ./.azure/pipelines/tools/SetupTestEnvironment.ps1 Setup signalrclienttests.exe"
       displayName: Start AppVerifier
     afterBuild:
+    - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true"
+      displayName: Run Flaky Tests
+      continueOnError: true
     - powershell: "& ./.azure/pipelines/tools/SetupTestEnvironment.ps1 Shutdown signalrclienttests.exe"
       displayName: Stop AppVerifier
       condition: always()
@@ -307,6 +310,7 @@ jobs:
     - name: Windows_Test_Logs
       path: artifacts/logs/
       publishOnError: true
+
 - template: jobs/default-build.yml
   parameters:
     condition: ne(variables['SkipTests'], 'true')
@@ -318,6 +322,10 @@ jobs:
     beforeBuild:
     - bash: "./eng/scripts/install-nginx-mac.sh"
       displayName: Installing Nginx
+    afterBuild:
+    - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
+      displayName: Run Flaky Tests
+      continueOnError: true
     artifacts:
     - name: MacOS_Test_Logs
       path: artifacts/logs/
@@ -333,6 +341,10 @@ jobs:
     beforeBuild:
     - bash: "./eng/scripts/install-nginx-linux.sh"
       displayName: Installing Nginx
+    afterBuild:
+    - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
+      displayName: Run Flaky Tests
+      continueOnError: true
     artifacts:
     - name: Linux_Test_Logs
       path: artifacts/logs/

.azure/pipelines/jobs/default-build.yml

@@ -174,6 +174,8 @@ jobs:
       testRunner: vstest
       testResultsFiles: '**/artifacts/**/*.trx'
       mergeTestResults: true
+      buildConfiguration: $(BuildConfiguration)
+      buildPlatform: $(AgentOsName)
   - ${{ each artifact in parameters.artifacts }}:
     - task: PublishBuildArtifacts@1
       displayName: Upload artifacts from ${{ artifact.path }}

eng/FlakyTests.AfterArcade.props

@@ -0,0 +1,7 @@
+<Project>
+  <!-- Override where xUnit logs and results go if we're doing the flaky run -->
+  <PropertyGroup Condition="'$(RunFlakyTests)' == 'true'">
+    <ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Flaky\</ArtifactsLogDir>
+    <ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Flaky\</ArtifactsTestResultsDir>
+  </PropertyGroup>
+</Project>

eng/FlakyTests.BeforeArcade.props

@@ -0,0 +1,18 @@
+<Project>
+  <!-- Local Dev Flakiness -->
+  <PropertyGroup>
+    <_FlakyRunAdditionalArgs>-trait "Flaky:All=true"</_FlakyRunAdditionalArgs>
+    <_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true"</_NonFlakyRunAdditionalArgs>
+  </PropertyGroup>
+
+  <!-- Azure Pipelines Flakiness -->
+  <PropertyGroup Condition="'$(AGENT_OS)' != ''">
+    <_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
+    <_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == ''">$(_NonFlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
+    <TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == 'true'">$(_FlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
+  </PropertyGroup>
+</Project>

eng/helix/vstest/runtests.cmd

@@ -1,3 +1,6 @@
+REM Disable "!Foo!" expansions because they break the filter syntax
+setlocal disableextensions
+
 set target=%1
 set sdkVersion=%2
 set runtimeVersion=%3
@@ -18,12 +21,35 @@ set HELIX=%helixQueue%
 
 %DOTNET_ROOT%\dotnet vstest %target% -lt >discovered.txt
 find /c "Exception thrown" discovered.txt
-if %errorlevel% equ 0 (
-    echo Exception thrown during test discovery.
-    type discovered.txt
+REM "ERRORLEVEL is not %ERRORLEVEL%" https://blogs.msdn.microsoft.com/oldnewthing/20080926-00/?p=20743/
+if not errorlevel 1 (
+    echo Exception thrown during test discovery. 1>&2
+    type discovered.txt 1>&2
     exit 1
 )
 
-%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal
+set exit_code=0
 
+REM Run non-flaky tests first
+REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
+REM only puts the explicit filter traits the user provided in
+REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
+set NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
+echo Running non-flaky tests.
+%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal --TestCaseFilter:%NONFLAKY_FILTER%
+if errorlevel 1 (
+    echo Failure in non-flaky test 1>&2
+    set exit_code=1
+    REM DO NOT EXIT
+)
+
+set FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
+echo Running known-flaky tests.
+%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal --TestCaseFilter:%FLAKY_FILTER%
+if errorlevel 1 (
+    echo Failure in flaky test 1>&2
+    REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
+)
+
+exit %exit_code%
 

eng/helix/vstest/runtests.sh

@@ -67,4 +67,25 @@ if grep -q "Exception thrown" discovered.txt; then
     exit 1
 fi
 
-$DOTNET_ROOT/dotnet vstest $1 --logger:trx
+# Run non-flaky tests first
+# We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
+# only puts the explicit filter traits the user provided in the flaky attribute
+# Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
+NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$HELIX!=true"
+echo "Running non-flaky tests."
+$DOTNET_ROOT/dotnet vstest $1 --logger:trx --TestCaseFilter:"$NONFLAKY_FILTER"
+nonflaky_exitcode=$?
+if [ $nonflaky_exitcode != 0 ]; then
+    echo "Non-flaky tests failed!" 1>&2
+    # DO NOT EXIT
+fi
+
+FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$HELIX=true"
+echo "Running known-flaky tests."
+$DOTNET_ROOT/dotnet vstest $1 --logger:trx --TestCaseFilter:"$FLAKY_FILTER"
+if [ $? != 0 ]; then
+    echo "Flaky tests failed!" 1>&2
+    # DO NOT EXIT
+fi
+
+exit $nonflaky_exitcode

korebuild-lock.txt

@@ -1,2 +1,2 @@
-version:3.0.0-build-20190306.2
-commithash:18c06e0b774622c87560e6f21b97e099307fd10c
+version:3.0.0-build-20190314.2
+commithash:e3a8a2aae198f1ef26309714ccba6835be2437c3