diff --git a/.azure/pipelines/helix-test.yml b/.azure/pipelines/helix-test.yml new file mode 100644 index 0000000000..96b9d6ccaa --- /dev/null +++ b/.azure/pipelines/helix-test.yml @@ -0,0 +1,27 @@ +# Don't run CI for this config yet. We're not ready to move official builds on to Azure Pipelines +trigger: none + +# Run PR validation on all branches +pr: + branches: + include: + - '*' + +jobs: +- template: jobs/default-build.yml + parameters: + jobName: Helix_quarantine_x64 + jobDisplayName: 'Tests: Helix Quarantine x64' + agentOs: Windows + timeoutInMinutes: 240 + steps: + - script: .\restore.cmd -ci + displayName: Restore + - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl + displayName: Run build.cmd helix target + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + artifacts: + - name: Helix_logs + path: artifacts/log/ + publishOnError: true diff --git a/eng/helix/content/runtests.cmd b/eng/helix/content/runtests.cmd index b0a910502c..2e1f87878b 100644 --- a/eng/helix/content/runtests.cmd +++ b/eng/helix/content/runtests.cmd @@ -8,6 +8,7 @@ set sdkVersion=%3 set runtimeVersion=%4 set helixQueue=%5 set arch=%6 +set quarantined=%7 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk set DOTNET_ROOT=%DOTNET_HOME%\%arch% @@ -38,25 +39,25 @@ if not errorlevel 1 ( set exit_code=0 -REM Run non-flaky tests first -REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute -REM only puts the explicit filter traits the user provided in -REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md -set NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true" -echo Running non-flaky tests. -%DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%NONFLAKY_FILTER% -if errorlevel 1 ( - echo Failure in non-flaky test 1>&2 - set exit_code=1 - REM DO NOT EXIT -) - -set FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true" -echo Running known-flaky tests. -%DOTNET_ROOT%\dotnet vstest %target% --TestCaseFilter:%FLAKY_FILTER% -if errorlevel 1 ( - echo Failure in flaky test 1>&2 - REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1 +set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true" +if (%quarantined%==true) ( + echo Running all tests. + %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit + if errorlevel 1 ( + echo Failure in flaky test 1>&2 + REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1 + ) +) else ( + REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute + REM only puts the explicit filter traits the user provided in + REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md + echo Running non-quarantined tests. + %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%NONQUARANTINE_FILTER% + if errorlevel 1 ( + echo Failure in non-flaky test 1>&2 + set exit_code=1 + REM DO NOT EXIT + ) ) echo "Copying TestResults\TestResults.xml to ." diff --git a/eng/helix/content/runtests.sh b/eng/helix/content/runtests.sh index 3934d5541c..663b20afb3 100644 --- a/eng/helix/content/runtests.sh +++ b/eng/helix/content/runtests.sh @@ -4,6 +4,8 @@ test_binary_path="$1" dotnet_sdk_version="$2" dotnet_runtime_version="$3" helix_queue_name="$4" +target_arch="$5" +quarantined="$6" RESET="\033[0m" RED="\033[0;31m" @@ -87,25 +89,27 @@ if grep -q "Exception thrown" discovered.txt; then exit 1 fi -# Run non-flaky tests first -# We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute +exit_code=0 + +# We need to specify all possible quarantined filters that apply to this environment, because the quarantine attribute # only puts the explicit filter traits the user provided in the flaky attribute # Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md -NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true" -echo "Running non-flaky tests." -$DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$NONFLAKY_FILTER" -nonflaky_exitcode=$? -if [ $nonflaky_exitcode != 0 ]; then - echo "Non-flaky tests failed!" 1>&2 - # DO NOT EXIT -fi - -FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true" -echo "Running known-flaky tests." -$DOTNET_ROOT/dotnet vstest $test_binary_path --TestCaseFilter:"$FLAKY_FILTER" -if [ $? != 0 ]; then - echo "Flaky tests failed!" 1>&2 - # DO NOT EXIT +NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true" +if [ "$quarantined" == true ]; then + echo "Running all tests including quarantined." + $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit + if [ $? != 0 ]; then + echo "Quarantined tests failed!" 1>&2 + # DO NOT EXIT + fi +else + echo "Running non-quarantined tests." + $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$NONQUARANTINE_FILTER" + exit_code=$? + if [ $exit_code != 0 ]; then + echo "Non-quarantined tests failed!" 1>&2 + # DO NOT EXIT + fi fi echo "Copying TestResults/TestResults to ." @@ -114,4 +118,4 @@ echo "Copying artifacts/logs to $HELIX_WORKITEM_UPLOAD_ROOT/../" shopt -s globstar cp artifacts/log/**/*.log $HELIX_WORKITEM_UPLOAD_ROOT/../ cp artifacts/log/**/*.log $HELIX_WORKITEM_UPLOAD_ROOT/ -exit $nonflaky_exitcode +exit $exit_code diff --git a/eng/targets/Helix.props b/eng/targets/Helix.props index 8d71616ea6..5a41315124 100644 --- a/eng/targets/Helix.props +++ b/eng/targets/Helix.props @@ -12,9 +12,12 @@ true 00:30:00 + false false true - $(MSBuildProjectName)-$(TargetFramework) + <_HelixRunType Condition="'$(RunQuarantinedTests)' == 'true'">quarantined + <_HelixRunType Condition="'$(RunQuarantinedTests)' != 'true'">unquarantined + $(MSBuildProjectName)--$(TargetFramework)--$(_HelixRunType) false false 10.15.3 diff --git a/eng/targets/Helix.targets b/eng/targets/Helix.targets index 91e952efe3..c6a20b6055 100644 --- a/eng/targets/Helix.targets +++ b/eng/targets/Helix.targets @@ -83,8 +83,8 @@ Usage: dotnet msbuild /t:Helix src/MyTestProject.csproj $(TargetFileName) @(HelixPreCommand) @(HelixPostCommand) - call runtests.cmd $(TargetFileName) $(TargetFrameworkIdentifier) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) - ./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) + call runtests.cmd $(TargetFileName) $(TargetFrameworkIdentifier) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests) + ./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests) $(HelixTimeout)