diff --git a/appveyor.yml b/appveyor.yml
index 9744c940ec..1041615c68 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -8,9 +8,6 @@ branches:
     - /^(.*\/)?ci-.*$/
 build_script:
   - ps: .\build.ps1
-install:
-  - set PATH=C:\Python27\scripts;%PATH%
-  - pip install autobahntestsuite
 clone_depth: 1
 test: off
 deploy: off
diff --git a/test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs b/test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs
index 98757be0c6..760eb6ecca 100644
--- a/test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs
+++ b/test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs
@@ -65,6 +65,7 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest
                             // IIS Express tests are a bit flaky, some tests fail occasionally or get non-strict passes
                             // https://github.com/aspnet/WebSockets/issues/100
                             await tester.DeployTestAndAddToSpec(ServerType.IISExpress, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token, expectationConfig: expect => expect
+                                .OkOrFail("2.6", "6.22.22") // Getting some transient failures on the CI for these. See https://github.com/aspnet/WebSockets/issues/152
                                 .OkOrFail(Enumerable.Range(1, 20).Select(i => $"5.{i}").ToArray())); // 5.* occasionally fail on IIS express
                         }