EnableAzurePipelinesReporter for helix (#8094)

* EnableAzurePipelinesReporter for helix

* Update Helix SDK

* Revert a fix that broke things even more

* Try publishing test assets

* Try fixing Kestrel tests on Helix

* Try a temp file in the working directory

* Skip Http2Spec tests on Debian 8

* Missed update

* Try a different format

* Disable failing test on Fedora

* Mark flaky test

.azure/pipelines/helix-test.yml

@@ -17,6 +17,8 @@ jobs:
     steps:
     - script: .\build.cmd -all -ci /t:Helix /p:IsHelixJob=true /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\logs\SendToHelix.binlog
       displayName: Run build.cmd helix target
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
     artifacts:
     - name: Helix_logs
       path: artifacts/logs/

build/repo.targets

@@ -151,7 +151,7 @@
              Properties="$(BuildProperties);__DummyTarget=GenerateBuildAssetManifest" />
   </Target>
 
-  <Target Name="Helix" DependsOnTargets="Restore">
+  <Target Name="Helix">
     <MSBuild Projects="$(MSBuildThisFileDirectory)..\eng\helix\helix.proj"
              Targets="Test"
              Properties="$(BuildProperties);$(SolutionProperties);"

eng/Version.Details.xml

@@ -388,9 +388,9 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>ed63a7faf2d119829dd971be91bb46b6ebaa1dd2</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19252.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19253.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ed63a7faf2d119829dd971be91bb46b6ebaa1dd2</Sha>
+      <Sha>e31dd0f6fd12a136a2310a308d235c360703b221</Sha>
     </Dependency>
     <Dependency Name="Microsoft.AspNetCore.Testing" Version="3.0.0-preview6.19252.7" CoherentParentDependency="Microsoft.EntityFrameworkCore">
       <Uri>https://github.com/aspnet/Extensions</Uri>

eng/helix/helix.proj

@@ -15,7 +15,7 @@
     <HelixBuild Condition=" '$(CI)' == 'true' ">$(BUILD_BUILDNUMBER)</HelixBuild>
     <WaitForWorkItemCompletion Condition=" '$(CI)' == 'true' ">false</WaitForWorkItemCompletion>
     <FailOnMissionControlTestFailure>false</FailOnMissionControlTestFailure>
-    <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
+    <EnableAzurePipelinesReporter>true</EnableAzurePipelinesReporter>
     <IsExternal>true</IsExternal>
     <Creator>aspnetcore</Creator>
     <SkipInvalidConfigurations>true</SkipInvalidConfigurations>

eng/helix/vstest/runtests.cmd

@@ -36,7 +36,7 @@ REM only puts the explicit filter traits the user provided in
 REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
 set NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
 echo Running non-flaky tests.
-%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal --TestCaseFilter:%NONFLAKY_FILTER%
+%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --TestCaseFilter:%NONFLAKY_FILTER%
 if errorlevel 1 (
     echo Failure in non-flaky test 1>&2
     set exit_code=1
@@ -45,7 +45,7 @@ if errorlevel 1 (
 
 set FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
 echo Running known-flaky tests.
-%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal --TestCaseFilter:%FLAKY_FILTER%
+%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --TestCaseFilter:%FLAKY_FILTER%
 if errorlevel 1 (
     echo Failure in flaky test 1>&2
     REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1

eng/helix/vstest/runtests.sh

@@ -79,7 +79,6 @@ if [ $nonflaky_exitcode != 0 ]; then
     echo "Non-flaky tests failed!" 1>&2
     # DO NOT EXIT
 fi
-
 FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$HELIX=true"
 echo "Running known-flaky tests."
 $DOTNET_ROOT/dotnet vstest $1 --logger:trx --TestCaseFilter:"$FLAKY_FILTER"

eng/targets/Helix.targets

@@ -52,7 +52,7 @@
 
   <Target Name="_CreateHelixWorkItem" Condition="$(BuildHelixPayload)">
 
-    <MSBuild Projects="$(MSBuildProjectFullPath)" Targets="_PublishHelixArchive" RemoveProperties="HelixTargetQueue" />
+    <MSBuild Projects="$(MSBuildProjectFullPath)" Targets="_PublishHelixArchive" RemoveProperties="HelixTargetQueue;TestRunId" />
 
     <ConvertToAbsolutePath Paths="$(PublishDir)">
       <Output TaskParameter="AbsolutePaths" PropertyName="PublishAbsoluteDir" />

global.json

@@ -4,6 +4,6 @@
   },
   "msbuild-sdks": {
     "Yarn.MSBuild": "1.13.0",
-    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19252.2"
+    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19253.2"
   }
 }

src/Servers/Kestrel/Directory.Build.props

@@ -6,6 +6,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
+    <KestrelRoot>$(MSBuildThisFileDirectory)</KestrelRoot>
     <KestrelSharedSourceRoot>$(MSBuildThisFileDirectory)shared\</KestrelSharedSourceRoot>
   </PropertyGroup>
 

src/Servers/Kestrel/Kestrel/test/GeneratedCodeTests.cs

@@ -17,12 +17,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
         [Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2223", FlakyOn.Helix.All)]
         public void GeneratedCodeIsUpToDate()
         {
-            var repositoryRoot = typeof(GeneratedCodeTests).Assembly.GetCustomAttributes<AssemblyMetadataAttribute>().First(f => string.Equals(f.Key, "RepositoryRoot", StringComparison.OrdinalIgnoreCase)).Value;
+            var httpHeadersGeneratedPath = Path.Combine(AppContext.BaseDirectory,"shared", "GeneratedContent", "HttpHeaders.Generated.cs");
-
+            var httpProtocolGeneratedPath = Path.Combine(AppContext.BaseDirectory,"shared", "GeneratedContent", "HttpProtocol.Generated.cs");
-            var httpHeadersGeneratedPath = Path.Combine(repositoryRoot, "src/Servers/Kestrel/Core/src/Internal/Http/HttpHeaders.Generated.cs");
+            var httpUtilitiesGeneratedPath = Path.Combine(AppContext.BaseDirectory,"shared", "GeneratedContent", "HttpUtilities.Generated.cs");
-            var httpProtocolGeneratedPath = Path.Combine(repositoryRoot, "src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.Generated.cs");
+            var transportConnectionGeneratedPath = Path.Combine(AppContext.BaseDirectory,"shared", "GeneratedContent", "TransportConnection.Generated.cs");
-            var httpUtilitiesGeneratedPath = Path.Combine(repositoryRoot, "src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.Generated.cs");
-            var transportConnectionGeneratedPath = Path.Combine(repositoryRoot, "src/Servers/Kestrel/Transport.Abstractions/src/Internal/TransportConnection.Generated.cs");
 
             var testHttpHeadersGeneratedPath = Path.GetTempFileName();
             var testHttpProtocolGeneratedPath = Path.GetTempFileName();

src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
@@ -9,6 +9,10 @@
     <Compile Include="$(KestrelSharedSourceRoot)test\*.cs" LinkBase="shared" />
     <Compile Include="$(SharedSourceRoot)test\SkipOnHelixAttribute.cs" />
     <Content Include="$(KestrelSharedSourceRoot)test\TestCertificates\*.pfx" LinkBase="shared\TestCertificates" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(KestrelRoot)Core\src\Internal\Http\HttpHeaders.Generated.cs" LinkBase="shared\GeneratedContent" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(KestrelRoot)Core\src\Internal\Http\HttpProtocol.Generated.cs" LinkBase="shared\GeneratedContent" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(KestrelRoot)Core\src\Internal\Infrastructure\HttpUtilities.Generated.cs" LinkBase="shared\GeneratedContent" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="$(KestrelRoot)Transport.Abstractions\src\Internal\TransportConnection.Generated.cs" LinkBase="shared\GeneratedContent" CopyToOutputDirectory="PreserveNewest" />
 
     <AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
       <_Parameter1>RepositoryRoot</_Parameter1>

src/Servers/Kestrel/test/FunctionalTests/Http2/HandshakeTests.cs

@@ -20,6 +20,7 @@ using Xunit;
 namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests.Http2
 {
     [OSSkipCondition(OperatingSystems.MacOSX, SkipReason = "Missing SslStream ALPN support: https://github.com/dotnet/corefx/issues/30492")]
+    [SkipOnHelix(Queues = "Debian.8.Amd64.Open")] // Debian 8 uses OpenSSL 1.0.1 which does not support HTTP/2
     [MinimumOSVersion(OperatingSystems.Windows, WindowsVersions.Win10)]
     public class HandshakeTests : LoggedTest
     {

src/Servers/Kestrel/test/FunctionalTests/Http2/ShutdownTests.cs

@@ -23,6 +23,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests.Http2
 {
     [OSSkipCondition(OperatingSystems.MacOSX, SkipReason = "Missing SslStream ALPN support: https://github.com/dotnet/corefx/issues/30492")]
     [MinimumOSVersion(OperatingSystems.Windows, WindowsVersions.Win10)]
+    [SkipOnHelix(Queues = "Debian.8.Amd64.Open")] // Debian 8 uses OpenSSL 1.0.1 which does not support HTTP/2
     public class ShutdownTests : TestApplicationErrorLoggerLoggedTest
     {
         private static X509Certificate2 _x509Certificate2 = TestResources.GetTestCertificate();
@@ -42,6 +43,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests.Http2
         }
 
         [ConditionalFact]
+        [SkipOnHelix(Queues = "Fedora.28.Amd64.Open")] // https://github.com/aspnet/AspNetCore/issues/9985
+        [Flaky("https://github.com/aspnet/AspNetCore/issues/9985", FlakyOn.Helix.All)]
         public async Task GracefulShutdownWaitsForRequestsToFinish()
         {
             var requestStarted = new TaskCompletionSource<object>(TaskCreationOptions.RunContinuationsAsynchronously);

src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecCommands.cs

@@ -16,6 +16,33 @@ namespace Interop.FunctionalTests
 {
     public static class H2SpecCommands
     {
+        #region chmod
+        // user permissions
+        const int S_IRUSR = 0x100;
+        const int S_IWUSR = 0x80;
+        const int S_IXUSR = 0x40;
+
+        // group permission
+        const int S_IRGRP = 0x20;
+        const int S_IWGRP = 0x10;
+        const int S_IXGRP = 0x8;
+
+        // other permissions
+        const int S_IROTH = 0x4;
+        const int S_IWOTH = 0x2;
+        const int S_IXOTH = 0x1;
+
+        const int _0755 =
+            S_IRUSR | S_IXUSR | S_IWUSR
+            | S_IRGRP | S_IXGRP
+            | S_IROTH | S_IXOTH;
+
+        [DllImport("libc", SetLastError = true)]
+        private static extern int chmod(string pathname, int mode);
+
+        private static int chmod755(string pathname) => chmod(pathname, _0755);
+        #endregion
+
         private const int TimeoutSeconds = 15;
 
         private static string GetToolLocation()
@@ -27,11 +54,15 @@ namespace Interop.FunctionalTests
             }
             else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
             {
-                return Path.Combine(root, "linux", "h2spec");
+                var toolPath = Path.Combine(root, "linux", "h2spec");
+                chmod755(toolPath);
+                return toolPath;
             }
             else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
             {
-                return Path.Combine(root, "darwin", "h2spec");
+                var toolPath = Path.Combine(root, "darwin", "h2spec");
+                chmod755(toolPath);
+                return toolPath;
             }
             throw new NotImplementedException("Invalid OS");
         }

src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs

@@ -21,6 +21,7 @@ namespace Interop.FunctionalTests
     [OSSkipCondition(OperatingSystems.MacOSX, SkipReason = "Missing SslStream ALPN support: https://github.com/dotnet/corefx/issues/30492")]
     [MinimumOSVersion(OperatingSystems.Windows, WindowsVersions.Win81,
         SkipReason = "Missing Windows ALPN support: https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation#Support")]
+    [SkipOnHelix(Queues = "Debian.8.Amd64.Open")] // Debian 8 uses OpenSSL 1.0.1 which does not support HTTP/2
     public class H2SpecTests : LoggedTest
     {
         [ConditionalTheory]

src/Servers/Kestrel/test/Sockets.FunctionalTests/Sockets.FunctionalTests.csproj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <Compile Include="..\FunctionalTests\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)NullScope.cs" />
+    <Compile Include="$(SharedSourceRoot)test\SkipOnHelixAttribute.cs" />
     <Compile Include="$(KestrelSharedSourceRoot)test\*.cs" LinkBase="shared" />
     <Compile Include="$(KestrelSharedSourceRoot)test\TransportTestHelpers\*.cs" LinkBase="shared\TransportTestHelpers" />
     <Content Include="$(KestrelSharedSourceRoot)test\TestCertificates\*.pfx" LinkBase="shared\TestCertificates" CopyToOutputDirectory="PreserveNewest" />

src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj

@@ -32,10 +32,16 @@
     <RemoveDir Directories="$(TargetDir)TestProjects" Condition="Exists('$(TargetDir)TestProjects')" />
   </Target>
 
-  <Target Name="PublishDotNetWatch" BeforeTargets="Build">
+  <Target Name="PublishDotNetWatchOnBuild" BeforeTargets="Build">
     <MSBuild Projects="..\src\dotnet-watch.csproj"
       Targets="Publish"
       Properties="PublishDir=$(OutputPath)\tool\;Configuration=$(Configuration)" />
   </Target>
 
+  <Target Name="PublishDotNetWatchOnPublish" BeforeTargets="Publish">
+    <MSBuild Projects="..\src\dotnet-watch.csproj"
+      Targets="Publish"
+      Properties="PublishDir=$(PublishDir)\tool\;Configuration=$(Configuration)" />
+  </Target>
+
 </Project>