Force conformance tests to run on the CI (#228)

.appveyor.yml

@@ -5,12 +5,17 @@ environment:
   global:
     DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    ASPNETCORE_WSTEST_PATH: "$(APPVEYOR_BUILD_FOLDER)/vendor/virtualenv/Scripts/wstest.exe"
+cache:
+  - vendor\VCForPython27.msi
 branches:
   only:
     - master
     - /release\/.*/
     - dev
     - /^(.*\/)?ci-.*$/
+install:
+  - ps: .\build\setup-wstest.ps1
 build_script:
   - ps: .\run.ps1 default-build
 clone_depth: 1

build/setup-wstest.ps1

@@ -0,0 +1,52 @@
+function has($cmd) { !!(Get-Command $cmd -ErrorAction SilentlyContinue) }
+
+# Download VCForPython27 if necessary
+$VendorDir = Join-Path (Get-Location) "vendor"
+
+if(!(Test-Path $VendorDir)) {
+    mkdir $VendorDir
+}
+
+$VirtualEnvDir = Join-Path $VendorDir "virtualenv";
+$ScriptsDir = Join-Path $VirtualEnvDir "Scripts"
+$WsTest = Join-Path $ScriptsDir "wstest.exe"
+
+$VCPythonMsi = Join-Path $VendorDir "VCForPython27.msi"
+if(!(Test-Path $VCPythonMsi)) {
+    Write-Host "Downloading VCForPython27.msi"
+    Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile "$VCPythonMsi"
+}
+else {
+    Write-Host "Using VCForPython27.msi from Cache"
+}
+
+# Install VCForPython27
+Write-Host "Installing VCForPython27"
+
+# Launch this way to ensure we wait for msiexec to complete. It's a Windows app so it won't block the console by default.
+Start-Process msiexec "/i","$VCPythonMsi","/qn","/quiet","/norestart" -Wait
+
+Write-Host "Installed VCForPython27"
+
+# Install Python
+if(!(has python)) {
+    choco install python2
+}
+
+if(!(has python)) {
+    throw "Failed to install python2"
+}
+
+# Install virtualenv
+pip install virtualenv
+
+# Make a virtualenv in .virtualenv
+virtualenv $VirtualEnvDir
+
+& "$ScriptsDir\python" --version
+& "$ScriptsDir\pip" --version
+
+# Install autobahn into the virtualenv
+& "$ScriptsDir\pip" install autobahntestsuite
+
+Write-Host "Using wstest from: '$WsTest'"

test/Microsoft.AspNetCore.WebSockets.ConformanceTest/Autobahn/Executable.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;

test/Microsoft.AspNetCore.WebSockets.ConformanceTest/Autobahn/Wstest.cs

@@ -1,4 +1,5 @@
-using System;
+using System;
+using System.IO;
 
 namespace Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn
 {
@@ -9,18 +10,28 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn
     {
         private static Lazy<Wstest> _instance = new Lazy<Wstest>(Create);
 
+        public static readonly string DefaultLocation = LocateWstest();
+
         public static Wstest Default => _instance.Value;
 
         public Wstest(string path) : base(path) { }
 
         private static Wstest Create()
+        {
+            var location = LocateWstest();
+
+            return (location == null || !File.Exists(location)) ? null : new Wstest(location);
+        }
+
+        private static string LocateWstest()
         {
             var location = Environment.GetEnvironmentVariable("ASPNETCORE_WSTEST_PATH");
             if (string.IsNullOrEmpty(location))
             {
                 location = Locate("wstest");
             }
-            return location == null ? null : new Wstest(location);
+
+            return location;
         }
     }
 }

test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs

@@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Testing.xunit;
 using Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Testing;
+using Xunit;
 using Xunit.Abstractions;
 
 namespace Microsoft.AspNetCore.WebSockets.ConformanceTest
@@ -27,6 +28,11 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest
         [SkipIfWsTestNotPresent]
         public async Task AutobahnTestSuite()
         {
+            // If we're on CI, we want to actually fail if WsTest isn't installed, rather than just skipping the test
+            // The SkipIfWsTestNotPresent attribute ensures that this test isn't skipped on CI, so we just need to check that Wstest is present
+            // And we use Assert.True to provide an error message
+            Assert.True(Wstest.Default != null, $"The 'wstest' executable (Autobahn WebSockets Test Suite) could not be found at '{Wstest.DefaultLocation}'. Run the Build Agent setup scripts to install it or see https://github.com/crossbario/autobahn-testsuite for instructions on manual installation.");
+
             using (StartLog(out var loggerFactory))
             {
                 var logger = loggerFactory.CreateLogger<AutobahnTests>();

test/Microsoft.AspNetCore.WebSockets.ConformanceTest/SkipIfWsTestNotPresentAttribute.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using Microsoft.AspNetCore.Testing.xunit;
 using Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn;
 
@@ -7,7 +7,12 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
     public class SkipIfWsTestNotPresentAttribute : Attribute, ITestCondition
     {
-        public bool IsMet => Wstest.Default != null;
+        public bool IsMet => IsOnCi || Wstest.Default != null;
         public string SkipReason => "Autobahn Test Suite is not installed on the host machine.";
+
+        private static bool IsOnCi =>
+            !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION")) ||
+            string.Equals(Environment.GetEnvironmentVariable("TRAVIS"), "true", StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(Environment.GetEnvironmentVariable("APPVEYOR"), "true", StringComparison.OrdinalIgnoreCase);
     }
-}
+}