Run browser functional tests in CI builds (#1487)

.travis.yml

@@ -15,6 +15,7 @@ os:
 - osx
 osx_image: xcode8.2
 addons:
+  chrome: stable
   apt:
     packages:
     - libunwind8

build/repo.targets

@@ -10,22 +10,35 @@
     </NPMPackage>
   </ItemGroup>
 
-  <Target Name="RestoreNpm" AfterTargets="Restore" Condition="'$(PreflightRestore)' != 'True'">
+  <PropertyGroup>
+    <RestoreDependsOn>$(RestoreDependsOn);RestoreNpm</RestoreDependsOn>
+  </PropertyGroup>
+  <Target Name="RestoreNpm" Condition="'$(PreflightRestore)' != 'True'">
     <Message Text="Restoring NPM modules" Importance="high" />
     <Exec Command="npm install --no-optional" WorkingDirectory="$(RepositoryRoot)client-ts" />
     <Exec Command="npm install --no-optional" WorkingDirectory="$(RepositoryRoot)client-ts/FunctionalTests" />
     <Exec Command="npm install --no-optional" WorkingDirectory="$(RepositoryRoot)client-ts/signalr" />
     <Exec Command="npm install --no-optional" WorkingDirectory="$(RepositoryRoot)client-ts/signalr-protocol-msgpack" />
+    <Exec Command="npm install --no-optional" WorkingDirectory="$(RepositoryRoot)client-ts/webdriver-tap-runner" />
   </Target>
 
-  <Target Name="RunTSClientNodeTests" AfterTargets="Test">
+  <PropertyGroup>
+    <TestDependsOn>$(TestDependsOn);RunTSClientNodeTests;RunBrowserTests</TestDependsOn>
+  </PropertyGroup>
+
+  <Target Name="RunTSClientNodeTests">
     <Message Text="Running TypeScript client Node tests" Importance="high" />
     <Exec Command="npm test" WorkingDirectory="$(RepositoryRoot)client-ts" IgnoreStandardErrorWarningFormat="true" />
   </Target>
 
+  <Target Name="RunBrowserTests">
+    <Message Text="Running TypeScript client Browser tests" Importance="high" />
+    <Exec Command="npm run ci-test -- --configuration $(Configuration) -v" WorkingDirectory="$(RepositoryRoot)client-ts/FunctionalTests" IgnoreStandardErrorWarningFormat="true" />
+  </Target>
+
   <PropertyGroup>
-    <PackageDependsOn>$(PackageDependsOn);PackNPMPackages</PackageDependsOn>
     <GetArtifactInfoDependsOn>$(GetArtifactInfoDependsOn);GetNpmArtifactInfo</GetArtifactInfoDependsOn>
+    <PrepareDependsOn>$(PrepareDependsOn);GetNpmArtifactInfo</PrepareDependsOn>
   </PropertyGroup>
 
   <Target Name="GetNpmArtifactInfo">
@@ -49,10 +62,21 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="PackNPMPackages" AfterTargets="Package">
+  <PropertyGroup>
+    <CompileDependsOn>Restore;BuildNPMPackages;$(CompileDependsOn)</CompileDependsOn>
+  </PropertyGroup>
+  <Target Name="BuildNPMPackages" DependsOnTargets="RestoreNpm;GetNpmArtifactInfo">
+    <Message Text="Building %(NPMPackage.PackageId)..." Importance="high" />
+    <Exec Command="npm run build" WorkingDirectory="%(NPMPackage.FullPath)" />
+  </Target>
+
+  <PropertyGroup>
+    <PackageDependsOn>Compile;PackNPMPackages;$(PackageDependsOn)</PackageDependsOn>
+  </PropertyGroup>
+  <Target Name="PackNPMPackages" DependsOnTargets="BuildNPMPackages">
+    <Message Text="Packing %(NPMPackage.PackageId)..." Importance="high" />
     <Copy SourceFiles="%(NPMPackage.PackageJson)" DestinationFiles="%(NPMPackage.PackageJson).bak" />
     <Exec Command="npm --no-git-tag-version --allow-same-version version $(PackageVersion)" WorkingDirectory="%(NPMPackage.FullPath)" />
-    <Exec Command="npm run build" WorkingDirectory="%(NPMPackage.FullPath)" />
     <Exec Command="npm pack" WorkingDirectory="%(NPMPackage.FullPath)" />
     <Delete Files="%(NPMPackage.ArtifactPath)" Condition="Exists('%(NPMPackage.ArtifactPath)')" />
     <Move SourceFiles="%(NPMPackage.OutputTar)" DestinationFiles="%(NPMPackage.ArtifactPath)" />

client-ts/FunctionalTests/EchoEndPoint.cs

@@ -4,6 +4,7 @@
 using System.Buffers;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Protocols;
+using Microsoft.AspNetCore.Protocols.Features;
 using Microsoft.AspNetCore.Sockets;
 
 namespace FunctionalTests
@@ -26,6 +27,21 @@ namespace FunctionalTests
             {
                 connection.Transport.Input.AdvanceTo(result.Buffer.End);
             }
+
+            // Wait for the user to close
+            var tcs = new TaskCompletionSource<object>(TaskCreationOptions.RunContinuationsAsynchronously);
+            connection.Transport.Input.OnWriterCompleted((ex, state) => 
+            {
+                if (ex != null) 
+                {
+                    ((TaskCompletionSource<object>)state).TrySetException(ex);
+                }
+                else
+                {
+                    ((TaskCompletionSource<object>)state).TrySetResult(null);
+                }
+            }, tcs);
+            await tcs.Task;
         }
     }
 }

client-ts/FunctionalTests/Program.cs

@@ -14,7 +14,7 @@ namespace FunctionalTests
             var host = new WebHostBuilder()
                 .ConfigureLogging(factory =>
                 {
-                    factory.AddConsole();
+                    factory.AddConsole(options => options.IncludeScopes = true);
                     factory.AddFilter("Console", level => level >= LogLevel.Information);
                     factory.AddDebug();
                 })

client-ts/FunctionalTests/Startup.cs

@@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Sockets;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.IdentityModel.Tokens;
 using Newtonsoft.Json.Serialization;
@@ -80,10 +81,18 @@ namespace FunctionalTests
             }
 
             app.UseFileServer();
-            app.UseSockets(options => options.MapEndPoint<EchoEndPoint>("/echo"));
+            app.UseSockets(routes =>
-            app.UseSignalR(options => options.MapHub<TestHub>("/testhub"));
+            {
-            app.UseSignalR(options => options.MapHub<UncreatableHub>("/uncreatable"));
+                routes.MapEndPoint<EchoEndPoint>("/echo");
-            app.UseSignalR(options => options.MapHub<HubWithAuthorization>("/authorizedhub"));
+            });
+
+            app.UseSignalR(routes =>
+            {
+                routes.MapHub<TestHub>("/testhub");
+                routes.MapHub<TestHub>("/testhub-nowebsockets", options => options.Transports = TransportType.ServerSentEvents | TransportType.LongPolling);
+                routes.MapHub<UncreatableHub>("/uncreatable");
+                routes.MapHub<HubWithAuthorization>("/authorizedhub");
+            });
 
             app.Use(next => async (context) =>
             {

client-ts/FunctionalTests/TestHub.cs

@@ -5,6 +5,7 @@ using System;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.AspNetCore.Sockets;
 
 namespace FunctionalTests
 {
@@ -52,6 +53,11 @@ namespace FunctionalTests
             throw new InvalidOperationException(message);
         }
 
+        public string GetActiveTransportName()
+        {
+            return Context.Connection.Metadata[ConnectionMetadataNames.Transport].ToString();
+        }
+
         public ComplexObject EchoComplexObject(ComplexObject complexObject)
         {
             return complexObject;

client-ts/FunctionalTests/package.json

@@ -6,16 +6,14 @@
   "main": "index.js",
   "dependencies": {},
   "devDependencies": {
-    "@types/selenium-webdriver": "^3.0.8",
+    "@types/debug": "0.0.30",
-    "@types/yargs": "^11.0.0",
+    "@types/node": "^9.4.6",
+    "debug": "^3.1.0",
     "es6-promise": "^4.2.2",
-    "faucet": "0.0.1",
+    "tap-spec": "^4.1.1",
-    "selenium-webdriver": "^4.0.0-alpha.1",
+    "tap-teamcity": "^3.0.2",
-    "tap-mocha-reporter": "^3.0.6",
+    "tee": "^0.2.0",
-    "tree-kill": "^1.2.0",
+    "ts-node": "^4.1.0"
-    "ts-node": "^4.1.0",
-    "webdriver-manager": "^12.0.6",
-    "yargs": "^11.0.0"
   },
   "scripts": {
     "clean": "node ../node_modules/rimraf/bin.js ./wwwroot/dist ./obj/js",
@@ -23,8 +21,9 @@
     "build:lint": "node ../node_modules/tslint/bin/tslint -c ../tslint.json -p ./tsconfig.json",
     "build:tsc": "node ../node_modules/typescript/bin/tsc --project ./tsconfig.json",
     "build:rollup": "node ../node_modules/rollup/bin/rollup -c",
-    "rawtest": "ts-node --project ./run-tests.tsconfig.json ./run-tests.ts --browser chrome -h",
+    "pretest": "npm run build",
-    "test": "npm run rawtest | faucet"
+    "test": "dotnet build && ts-node --project ./selenium/tsconfig.json ./selenium/run-tests.ts",
+    "ci-test": "ts-node --project ./selenium/tsconfig.json ./selenium/run-ci-tests.ts"
   },
   "author": "",
   "license": "Apache-2.0"

client-ts/FunctionalTests/run-tests.ts

@@ -1,258 +0,0 @@
-// console.log messages should be prefixed with "#" to ensure stdout continues to conform to TAP (Test Anything Protocol)
-// https://testanything.org/tap-version-13-specification.html
-
-import { ChildProcess, spawn, spawnSync } from "child_process";
-import * as os from "os";
-import * as path from "path";
-import { Readable } from "stream";
-
-import { Builder, By, Capabilities, logging, WebDriver, WebElement } from "selenium-webdriver";
-import * as kill from "tree-kill";
-import { argv } from "yargs";
-
-const rootDir = __dirname;
-
-const verbose = argv.v || argv.verbose || false;
-const browser = argv.browser || "chrome";
-const headless = argv.headless || argv.h || false;
-
-let webDriver: ChildProcess;
-let dotnet: ChildProcess;
-
-console.log("TAP version 13");
-
-function logverbose(message: any) {
-    if (verbose) {
-        console.log(message);
-    }
-}
-
-function runCommand(command: string, args: string[]) {
-    args = args || [];
-    const result = spawnSync(command, args, {
-        cwd: rootDir,
-    });
-    if (result.status !== 0) {
-        console.error("Bail out!"); // Part of the TAP protocol
-        console.error(`Command ${command} ${args.join(" ")} failed:`);
-        console.error("stderr:");
-        console.error(result.stderr);
-        console.error("stdout:");
-        console.error(result.stdout);
-        shutdown(1);
-    }
-}
-
-const logExtractorRegex = /[^ ]+ [^ ]+ "(.*)"/;
-
-function getMessage(logMessage: string): string {
-    const r = logExtractorRegex.exec(logMessage);
-
-    // Unescape \"
-    if (r && r.length >= 2) {
-        return r[1].replace(/\\"/g, "\"");
-    } else {
-        return logMessage;
-    }
-}
-
-async function waitForElement(driver: WebDriver, id: string): Promise<WebElement> {
-    while (true) {
-        const elements = await driver.findElements(By.id(id));
-        if (elements && elements.length > 0) {
-            return elements[0];
-        }
-    }
-}
-
-async function isComplete(element: WebElement): Promise<boolean> {
-    return (await element.getAttribute("data-done")) === "1";
-}
-
-async function getLogEntry(index: number, element: WebElement): Promise<WebElement> {
-    const elements = await element.findElements(By.id(`__tap_item_${index}`));
-    if (elements && elements.length > 0) {
-        return elements[0];
-    }
-    return null;
-}
-
-async function getEntryContent(element: WebElement): Promise<string> {
-    return await element.getAttribute("innerHTML");
-}
-
-async function flushEntries(index: number, element: WebElement): Promise<void> {
-    let entry = await getLogEntry(index, element);
-    while (entry) {
-        index += 1;
-        console.log(await getEntryContent(entry));
-        entry = await getLogEntry(index, element);
-    }
-}
-
-function applyCapabilities(builder: Builder) {
-    if (browser === "chrome") {
-        const caps = Capabilities.chrome();
-        const args = [];
-
-        if (headless) {
-            console.log("# Using Headless Mode");
-            args.push("--headless");
-            if (process.platform === "win32") {
-                args.push("--disable-gpu");
-            }
-        }
-
-        caps.set("chromeOptions", {
-            args,
-        });
-        builder.withCapabilities(caps);
-    }
-}
-
-async function runTests(port: number, serverUrl: string): Promise<void> {
-    const webDriverUrl = `http://localhost:${port}/wd/hub`;
-    console.log(`# Using WebDriver at ${webDriverUrl}`);
-    console.log(`# Launching ${browser} browser`);
-    const logPrefs = new logging.Preferences();
-    logPrefs.setLevel(logging.Type.BROWSER, logging.Level.INFO);
-
-    const builder = new Builder()
-        .usingServer(webDriverUrl)
-        .setLoggingPrefs(logPrefs)
-        .forBrowser(browser);
-
-    applyCapabilities(builder);
-
-    const driver = await builder.build();
-    try {
-        await driver.get(serverUrl);
-
-        let index = 0;
-        console.log("# Running tests");
-        const element = await waitForElement(driver, "__tap_list");
-        const success = true;
-        while (!await isComplete(element)) {
-            const entry = await getLogEntry(index, element);
-            if (entry) {
-                index += 1;
-                console.log(await getEntryContent(entry));
-            }
-        }
-
-        // Flush remaining entries
-        await flushEntries(index, element);
-        console.log("# End of tests");
-    } catch (e) {
-        console.error("Error: " + e.toString());
-    } finally {
-        await driver.quit();
-    }
-}
-
-function waitForMatch(command: string, process: ChildProcess, regex: RegExp): Promise<RegExpMatchArray> {
-    return new Promise<RegExpMatchArray>((resolve, reject) => {
-        try {
-            let lastLine = "";
-
-            async function onData(this: Readable, chunk: string | Buffer): Promise<void> {
-                try {
-                    chunk = chunk.toString();
-
-                    // Process lines
-                    let lineEnd = chunk.indexOf(os.EOL);
-                    while (lineEnd >= 0) {
-                        const chunkLine = lastLine + chunk.substring(0, lineEnd);
-                        lastLine = "";
-
-                        chunk = chunk.substring(lineEnd + os.EOL.length);
-
-                        logverbose(`# ${command}: ${chunkLine}`);
-                        const results = regex.exec(chunkLine);
-                        if (results && results.length > 0) {
-                            this.removeAllListeners("data");
-                            resolve(results);
-                            return;
-                        }
-                        lineEnd = chunk.indexOf(os.EOL);
-                    }
-                    lastLine = chunk.toString();
-                } catch (e) {
-                    this.removeAllListeners("data");
-                    reject(e);
-                }
-            }
-
-            process.on("close", async (code, signal) => {
-                console.log(`# ${command} process exited with code: ${code}`);
-                await shutdown(1);
-            });
-
-            process.stdout.on("data", onData.bind(process.stdout));
-            process.stderr.on("data", onData.bind(process.stderr));
-        } catch (e) {
-            reject(e);
-        }
-    });
-}
-
-async function cleanUpProcess(name: string, process: ChildProcess): Promise<void> {
-    return new Promise<void>((resolve, reject) => {
-        try {
-            if (process && !process.killed) {
-                console.log(`# Killing ${name} process (PID: ${process.pid})`);
-                kill(process.pid, "SIGTERM", () => {
-                    console.log("# Killed dotnet process");
-                    resolve();
-                });
-            }
-            else {
-                resolve();
-            }
-        } catch (e) {
-            reject(e);
-        }
-    });
-}
-
-async function shutdown(code: number): Promise<void> {
-    await cleanUpProcess("dotnet", dotnet);
-    await cleanUpProcess("webDriver", webDriver);
-    process.exit(code);
-}
-
-// "async main" via IIFE
-(async function () {
-    const webDriverManagerPath = path.resolve(__dirname, "node_modules", "webdriver-manager", "bin", "webdriver-manager");
-
-    // This script launches the functional test app and then uses Selenium WebDriver to run the tests and verify the results.
-    console.log("# Updating WebDrivers...");
-    runCommand(process.execPath, [webDriverManagerPath, "update"]);
-    console.log("# Updated WebDrivers");
-
-    console.log("# Launching WebDriver...");
-    webDriver = spawn(process.execPath, [webDriverManagerPath, "start"]);
-
-    const webDriverRegex = /\d+:\d+:\d+.\d+ INFO - Selenium Server is up and running on port (\d+)/;
-
-    // The message we're waiting for is written to stderr for some reason
-    let results = await waitForMatch("webdriver-server", webDriver, webDriverRegex);
-    let webDriverPort = Number.parseInt(results[1]);
-
-    console.log("# WebDriver Launched");
-    console.log("# Launching Functional Test server...");
-    dotnet = spawn("dotnet", [path.resolve(__dirname, "bin", "Debug", "netcoreapp2.1", "FunctionalTests.dll")], {
-        cwd: rootDir,
-    });
-
-    const regex = /Now listening on: (http:\/\/localhost:([\d])+)/;
-    results = await waitForMatch("dotnet", dotnet, regex);
-    try {
-        console.log("# Functional Test server launched.");
-        await runTests(webDriverPort, results[1]);
-        await shutdown(0);
-    } catch (e) {
-        console.error(`Bail out! Error running tests: ${e}`);
-        await shutdown(1);
-    }
-})();

client-ts/FunctionalTests/run-tests.tsconfig.json

@@ -1,5 +0,0 @@
-{
-    "compilerOptions": {
-        "module": "commonjs"
-    }
-}

client-ts/FunctionalTests/selenium/run-ci-tests.ts

@@ -0,0 +1,96 @@
+import { ChildProcess, spawn, spawnSync } from "child_process";
+import { existsSync } from "fs";
+import * as path from "path";
+
+import * as tapTeamCity from "tap-teamcity";
+import * as tee from "tee";
+
+const teamcity = !!process.env.TEAMCITY_VERSION;
+
+let force = process.env.ASPNETCORE_SIGNALR_FORCE_BROWSER_TESTS === "true";
+let chromePath = process.env.ASPNETCORE_CHROME_PATH;
+
+let configuration;
+let verbose;
+
+for (let i = 2; i < process.argv.length; i += 1) {
+    switch (process.argv[i]) {
+        case "-f":
+        case "--force":
+            force = true;
+            break;
+
+        case "--chrome":
+            i += 1;
+            chromePath = process.argv[i];
+            break;
+
+        case "--configuration":
+            i += 1;
+            configuration = process.argv[i];
+            break;
+
+        case "-v":
+        case "--verbose":
+            verbose = true;
+            break;
+    }
+}
+
+function failPrereq(error: string) {
+    if (force) {
+        console.error(`Browser functional tests cannot be run: ${error}`);
+        process.exit(1);
+    } else {
+        console.log(`Skipping browser functional Tests: ${error}`);
+
+        // Zero exit code because we don't want to fail the build.
+        process.exit(0);
+    }
+}
+
+function getChromeBinaryPath(): string {
+    if (chromePath) {
+        return chromePath;
+    } else {
+        switch (process.platform) {
+            case "win32":
+                // tslint:disable-next-line:no-string-literal
+                return path.resolve(process.env.LOCALAPPDATA, "Google", "Chrome", "Application", "chrome.exe");
+            case "darwin":
+                return path.resolve("/", "Applications", "Google Chrome.app", "Contents", "MacOS", "Google Chrome");
+            case "linux":
+                return path.resolve("/", "usr", "bin", "google-chrome");
+        }
+    }
+}
+
+// Check prerequisites
+const chromeBinary = getChromeBinaryPath();
+if (!existsSync(chromeBinary)) {
+    failPrereq(`Unable to locate Google Chrome at '${chromeBinary}'. Use the '--chrome' argument or 'ASPNETCORE_CHROME_PATH' environment variable to specify an alternate path`);
+} else {
+    console.log(`Using Google Chrome Browser from '${chromeBinary}`);
+}
+
+// Launch the tests
+const args = ["test", "--", "--chrome", chromeBinary];
+if (configuration) {
+    args.push("--configuration");
+    args.push(configuration);
+}
+if (verbose) {
+    args.push("--verbose");
+}
+if (teamcity) {
+    args.push("--raw");
+}
+
+const testProcess = spawn("npm", args, { cwd: path.resolve(__dirname, "..") });
+testProcess.stderr.pipe(process.stderr);
+if (teamcity) {
+    testProcess.stdout.pipe(tapTeamCity()).pipe(process.stdout);
+}
+
+testProcess.stdout.pipe(process.stdout);
+testProcess.on("close", (code) => process.exit(code));

client-ts/FunctionalTests/selenium/run-tests.ts

@@ -0,0 +1,162 @@
+import { ChildProcess, spawn } from "child_process";
+import * as fs from "fs";
+import { EOL } from "os";
+import * as path from "path";
+import { PassThrough, Readable } from "stream";
+
+import * as tapSpec from "tap-spec";
+
+import { run } from "../../webdriver-tap-runner/lib";
+
+import * as _debug from "debug";
+const debug = _debug("signalr-functional-tests:run");
+
+process.on("unhandledRejection", (reason) => {
+    console.error(`Unhandled promise rejection: ${reason}`);
+    process.exit(1);
+});
+
+// Don't let us hang the build. If this process takes more than 10 minutes, we're outta here
+setTimeout(() => {
+    console.error("Bail out! Tests took more than 10 minutes to run. Aborting.");
+    process.exit(1);
+}, 1000 * 60 * 10);
+
+function waitForMatch(command: string, process: ChildProcess, regex: RegExp): Promise<RegExpMatchArray> {
+    return new Promise<RegExpMatchArray>((resolve, reject) => {
+        const commandDebug = _debug(`signalr-functional-tests:${command}`);
+        try {
+            let lastLine = "";
+
+            async function onData(this: Readable, chunk: string | Buffer): Promise<void> {
+                try {
+                    chunk = chunk.toString();
+
+                    // Process lines
+                    let lineEnd = chunk.indexOf(EOL);
+                    while (lineEnd >= 0) {
+                        const chunkLine = lastLine + chunk.substring(0, lineEnd);
+                        lastLine = "";
+
+                        chunk = chunk.substring(lineEnd + EOL.length);
+
+                        const results = regex.exec(chunkLine);
+                        commandDebug(chunkLine);
+                        if (results && results.length > 0) {
+                            resolve(results);
+                            return;
+                        }
+                        lineEnd = chunk.indexOf(EOL);
+                    }
+                    lastLine = chunk.toString();
+                } catch (e) {
+                    this.removeAllListeners("data");
+                    reject(e);
+                }
+            }
+
+            process.on("close", async (code, signal) => {
+                console.log(`${command} process exited with code: ${code}`);
+                global.process.exit(1);
+            });
+
+            process.stdout.on("data", onData.bind(process.stdout));
+            process.stderr.on("data", (chunk) => {
+                onData.bind(process.stderr)(chunk);
+                console.error(`${command} | ${chunk.toString()}`);
+            });
+        } catch (e) {
+            reject(e);
+        }
+    });
+}
+
+let raw = false;
+let configuration = "Debug";
+let chromePath: string;
+let spec: string;
+
+for (let i = 2; i < process.argv.length; i += 1) {
+    switch (process.argv[i]) {
+        case "--raw":
+            raw = true;
+            break;
+        case "--configuration":
+            i += 1;
+            configuration = process.argv[i];
+            break;
+        case "-v":
+        case "--verbose":
+            _debug.enable("signalr-functional-tests:*");
+            break;
+        case "--chrome":
+            i += 1;
+            chromePath = process.argv[i];
+            break;
+        case "--spec":
+            i += 1;
+            spec = process.argv[i];
+            break;
+    }
+}
+
+if (chromePath) {
+    debug(`Using Google Chrome at: '${chromePath}'`);
+}
+
+function createOutput() {
+    if (raw) {
+        return process.stdout;
+    } else {
+        const output = tapSpec();
+        output.pipe(process.stdout);
+        return output;
+    }
+}
+
+(async () => {
+    try {
+        const serverPath = path.resolve(__dirname, "..", "bin", configuration, "netcoreapp2.1", "FunctionalTests.dll");
+
+        debug(`Launching Functional Test Server: ${serverPath}`);
+        const dotnet = spawn("dotnet", [serverPath], {
+            env: {
+                ...process.env,
+                ["ASPNETCORE_URLS"]: "http://127.0.0.1:0"
+            },
+        });
+
+        function cleanup() {
+            if (dotnet && !dotnet.killed) {
+                console.log("Terminating dotnet process");
+                dotnet.kill();
+            }
+        }
+
+        process.on("SIGINT", cleanup);
+        process.on("exit", cleanup);
+
+        debug("Waiting for Functional Test Server to start");
+        const results = await waitForMatch("dotnet", dotnet, /Now listening on: (http:\/\/[^\/]+:[\d]+)/);
+        debug(`Functional Test Server has started at ${results[1]}`);
+
+        let url = results[1] + "?cacheBust=true";
+        if (spec) {
+            url += `&spec=${encodeURI(spec)}`;
+        }
+
+        debug(`Using server url: ${url}`);
+
+        const failureCount = await run("SignalR Browser Functional Tests", {
+            browser: "chrome",
+            chromeBinaryPath: chromePath,
+            output: createOutput(),
+            url,
+            webdriverPort: 9515,
+        });
+        process.exit(failureCount);
+    } catch (e) {
+        console.error("Error: " + e.toString());
+        process.exit(1);
+    }
+})();

client-ts/FunctionalTests/selenium/tsconfig.json

@@ -0,0 +1,6 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "es2015"
+    }
+}

client-ts/FunctionalTests/ts/Common.ts

@@ -37,7 +37,9 @@ export function eachTransportAndProtocol(action: (transport: TransportType, prot
     }
     getTransportTypes().forEach((t) => {
         return protocols.forEach((p) => {
+            if (t !== TransportType.ServerSentEvents || !(p instanceof MessagePackHubProtocol)) {
                 return action(t, p);
+            }
         });
     });
 }

client-ts/FunctionalTests/ts/ConnectionTests.ts

@@ -1,14 +1,15 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-import { HttpConnection, LogLevel, TransportType } from "@aspnet/signalr";
+import { HttpConnection, LogLevel, TransferFormat, TransportType } from "@aspnet/signalr";
 import { eachTransport, ECHOENDPOINT_URL } from "./Common";
 
 describe("connection", () => {
-    if (typeof WebSocket !== "undefined") {
     it("can connect to the server without specifying transport explicitly", (done) => {
         const message = "Hello World!";
-            const connection = new HttpConnection(ECHOENDPOINT_URL);
+        const connection = new HttpConnection(ECHOENDPOINT_URL, {
+            logger: LogLevel.Trace,
+        });
 
         let received = "";
         connection.onreceive = (data) => {
@@ -23,14 +24,13 @@ describe("connection", () => {
             done();
         };
 
-            connection.start().then(() => {
+        connection.start(TransferFormat.Text).then(() => {
             connection.send(message);
         }).catch((e) => {
-                fail();
+            fail(e);
             done();
         });
     });
-    }
 
     eachTransport((transportType) => {
         it("over " + TransportType[transportType] + " can send and receive messages", (done) => {
@@ -55,10 +55,10 @@ describe("connection", () => {
                 done();
             };
 
-            connection.start().then(() => {
+            connection.start(TransferFormat.Text).then(() => {
                 connection.send(message);
             }).catch((e) => {
-                fail();
+                fail(e);
                 done();
             });
         });

client-ts/FunctionalTests/ts/HubConnectionTests.ts

@@ -1,11 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-import { HubConnection, LogLevel, TransportType } from "@aspnet/signalr";
+import { HubConnection, JsonHubProtocol, LogLevel, TransportType } from "@aspnet/signalr";
+import { MessagePackHubProtocol } from "@aspnet/signalr-protocol-msgpack";
 
 import { eachTransport, eachTransportAndProtocol } from "./Common";
 
 const TESTHUBENDPOINT_URL = "/testhub";
+const TESTHUB_NOWEBSOCKETS_ENDPOINT_URL = "/testhub-nowebsockets";
 
 describe("hubConnection", () => {
     eachTransportAndProtocol((transportType, protocol) => {
@@ -394,15 +396,8 @@ describe("hubConnection", () => {
                             // and a Uint8Array even though Buffer instances are also Uint8Array instances
                             value.ByteArray = new Uint8Array(value.ByteArray);
 
-                            // GUIDs are serialized as raw type which is a string containing bytes which need to
+                            // GUIDs are serialized as Buffer as well.
-                            // be extracted. Note that with msgpack5 the original bytes will be encoded with utf8
+                            value.GUID = new Uint8Array(value.GUID);
-                            // and needs to be decoded. To not go into utf8 encoding intricacies the test uses values
-                            // less than 0x80.
-                            const guidBytes = [];
-                            for (let i = 0; i < value.GUID.length; i++) {
-                                guidBytes.push(value.GUID.charCodeAt(i));
-                            }
-                            value.GUID = new Uint8Array(guidBytes);
                         }
                         expect(value).toEqual(complexObject);
                     })
@@ -470,10 +465,9 @@ describe("hubConnection", () => {
             it("can connect to hub with authorization", async (done) => {
                 const message = "你好，世界！";
 
-                let hubConnection;
+                try {
-                getJwtToken("http://" + document.location.host + "/generateJwtToken")
+                    const jwtToken = await getJwtToken("http://" + document.location.host + "/generateJwtToken");
-                    .then((jwtToken) => {
+                    const hubConnection = new HubConnection("/authorizedhub", {
-                        hubConnection = new HubConnection("/authorizedhub", {
                         accessTokenFactory: () => jwtToken,
                         logger: LogLevel.Trace,
                         transport: transportType,
@@ -482,19 +476,18 @@ describe("hubConnection", () => {
                         expect(error).toBe(undefined);
                         done();
                     });
-                        return hubConnection.start();
+                    await hubConnection.start();
-                    })
+                    const response = await hubConnection.invoke("Echo", message);
-                    .then(() => {
+
-                        return hubConnection.invoke("Echo", message);
-                    })
-                    .then((response) => {
                     expect(response).toEqual(message);
+
+                    await hubConnection.stop();
+
                     done();
-                    })
+                } catch (err) {
-                    .catch((err) => {
                     fail(err);
                     done();
-                    });
+                }
             });
 
             if (transportType !== TransportType.LongPolling) {

client-ts/FunctionalTests/ts/WebDriverReporter.ts

@@ -13,7 +13,7 @@ function formatValue(v: any): string {
 }
 
 class WebDriverReporter implements jasmine.CustomReporter {
-    private element: HTMLUListElement;
+    private element: HTMLDivElement;
     private specCounter: number = 1; // TAP number start at 1
     private recordCounter: number = 0;
 
@@ -22,7 +22,7 @@ class WebDriverReporter implements jasmine.CustomReporter {
         // For example, Chrome supports scraping console.log from WebDriver which would be ideal, but Firefox does not :(
 
         // Create an element for the output
-        this.element = document.createElement("ul");
+        this.element = document.createElement("div");
         this.element.setAttribute("id", "__tap_list");
 
         if (!show) {
@@ -37,16 +37,18 @@ class WebDriverReporter implements jasmine.CustomReporter {
     }
 
     public specDone(result: jasmine.CustomReporterResult): void {
-        if (result.status === "failed") {
+        if (result.status === "disabled") {
+            return;
+        } else if (result.status === "failed") {
             this.taplog(`not ok ${this.specCounter} ${result.fullName}`);
 
-            // Include YAML block with failed expectations
+            // Just report the first failure
             this.taplog("  ---");
-            this.taplog(" failures:");
             for (const expectation of result.failedExpectations) {
+                // Include YAML block with failed expectations
                 this.taplog(`    - message: ${expectation.message}`);
                 if (expectation.matcherName) {
-                    this.taplog(`     matcher: ${expectation.matcherName}`);
+                    this.taplog(`      operator: ${expectation.matcherName}`);
                 }
                 if (expectation.expected) {
                     this.taplog(`      expected: ${formatValue(expectation.expected)}`);
@@ -69,13 +71,12 @@ class WebDriverReporter implements jasmine.CustomReporter {
 
     private taplog(msg: string) {
         for (const line of msg.split(/\r|\n|\r\n/)) {
-            const li = this.document.createElement("li");
+            const input = this.document.createElement("input");
-            li.setAttribute("style", "font-family: monospace; white-space: pre");
+            input.setAttribute("id", `__tap_item_${this.recordCounter}`);
-            li.setAttribute("id", `__tap_item_${this.recordCounter}`);
             this.recordCounter += 1;
 
-            li.innerHTML = line;
+            input.value = line;
-            this.element.appendChild(li);
+            this.element.appendChild(input);
         }
     }
 }

client-ts/FunctionalTests/ts/WebSocketTests.ts

@@ -14,21 +14,14 @@ if (typeof WebSocket !== "undefined") {
                 webSocket.send(message);
             };
 
-            let received = "";
             webSocket.onmessage = (event) => {
-                received += event.data;
+                expect(event.data).toEqual(message);
-                if (received === message) {
                 webSocket.close();
-                }
             };
 
             webSocket.onclose = (event) => {
-                if (!event.wasClean) {
+                expect(event.code).toEqual(1000);
-                    fail("connection closed with unexpected status code: " + event.code + " " + event.reason);
+                expect(event.wasClean).toBe(true, "WebSocket did not close cleanly");
-                }
-
-                // Jasmine doesn't like tests without expectations
-                expect(event.wasClean).toBe(true);
 
                 done();
             };

client-ts/FunctionalTests/ts/index.ts

@@ -1,16 +1,9 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+console.log("SignalR Functional Tests Loaded");
+
 import "es6-promise/dist/es6-promise.auto.js";
-
-// Load SignalR
-import { getParameterByName } from "./Utils";
-
-const minified = getParameterByName("release") === "true" ? ".min" : "";
-document.write(
-    '<script type="text/javascript" src="lib/signalr/signalr' + minified + '.js"><\/script>' +
-    '<script type="text/javascript" src="lib/signalr/signalr-protocol-msgpack' + minified + '.js"><\/script>');
-
 import "./ConnectionTests";
 import "./HubConnectionTests";
 import "./WebDriverReporter";

client-ts/FunctionalTests/wwwroot/default.html

@@ -28,11 +28,11 @@
         }
 
         var minified = getParameterByName('release') === 'true' ? '.min' : '';
-        var cacheBust = Math.random().toString().substring(2);
+        var cacheBust = getParameterByName('cacheBust') === 'true' ? ('?' + Math.random().toString().substring(2)) : '';
         document.write(
-            '<script type="text/javascript" src="lib/signalr/signalr' + minified + '.js?' + cacheBust + '"><\/script>' +
+            '<script type="text/javascript" src="lib/signalr/signalr' + minified + '.js' + cacheBust + '"><\/script>' +
-            '<script type="text/javascript" src="lib/signalr/signalr-protocol-msgpack' + minified + '.js?' + cacheBust + '"><\/script>' +
+            '<script type="text/javascript" src="lib/signalr/signalr-protocol-msgpack' + minified + '.js' + cacheBust + '"><\/script>' +
-            '<script type="text/javascript" src="dist/signalr-functional-tests.js?' + cacheBust + '"><\/script>');
+            '<script type="text/javascript" src="dist/signalr-functional-tests.js' + cacheBust + '"><\/script>');
     </script>
 </body>
 

client-ts/signalr/spec/HttpConnection.spec.ts

@@ -15,13 +15,13 @@ const commonOptions: IHttpConnectionOptions = {
 
 describe("HttpConnection", () => {
     it("cannot be created with relative url if document object is not present", () => {
-        expect(() => new HttpConnection("/test", TransferFormat.Text, commonOptions))
+        expect(() => new HttpConnection("/test", commonOptions))
             .toThrow(new Error("Cannot resolve '/test'."));
     });
 
     it("cannot be created with relative url if window object is not present", () => {
         (global as any).window = {};
-        expect(() => new HttpConnection("/test", TransferFormat.Text, commonOptions))
+        expect(() => new HttpConnection("/test", commonOptions))
             .toThrow(new Error("Cannot resolve '/test'."));
         delete (global as any).window;
     });
@@ -34,10 +34,10 @@ describe("HttpConnection", () => {
                 .on("GET", (r) => ""),
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org", options);
 
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
             fail();
             done();
         } catch (e) {
@@ -51,7 +51,7 @@ describe("HttpConnection", () => {
             ...commonOptions,
             httpClient: new TestHttpClient()
                 .on("POST", (r) => {
-                    connection.start()
+                    connection.start(TransferFormat.Text)
                         .then(() => {
                             fail();
                             done();
@@ -64,10 +64,10 @@ describe("HttpConnection", () => {
                 }),
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org", options);
 
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
         } catch (e) {
             // This exception is thrown after the actual verification is completed.
             // The connection is not setup to be running so just ignore the error.
@@ -86,16 +86,16 @@ describe("HttpConnection", () => {
                 .on("GET", (r) => ""),
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org", options);
 
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
         } catch (e) {
             expect(e).toBe("reached negotiate");
         }
 
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
         } catch (e) {
             expect(e).toBe("reached negotiate");
         }
@@ -117,10 +117,10 @@ describe("HttpConnection", () => {
                 }),
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org", options);
 
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
             done();
         } catch (e) {
             fail();
@@ -129,7 +129,7 @@ describe("HttpConnection", () => {
     });
 
     it("can stop a non-started connection", async (done) => {
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, commonOptions);
+        const connection = new HttpConnection("http://tempuri.org", commonOptions);
         await connection.stop();
         done();
     });
@@ -159,10 +159,10 @@ describe("HttpConnection", () => {
             transport: fakeTransport,
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org?q=myData", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org?q=myData", options);
 
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
             fail();
             done();
         } catch (e) {
@@ -190,10 +190,10 @@ describe("HttpConnection", () => {
                     }),
             } as IHttpConnectionOptions;
 
-            connection = new HttpConnection(givenUrl, TransferFormat.Text, options);
+            connection = new HttpConnection(givenUrl, options);
 
             try {
-                await connection.start();
+                await connection.start(TransferFormat.Text);
                 done();
             } catch (e) {
                 fail();
@@ -218,9 +218,9 @@ describe("HttpConnection", () => {
                 transport: requestedTransport,
             } as IHttpConnectionOptions;
 
-            const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+            const connection = new HttpConnection("http://tempuri.org", options);
             try {
-                await connection.start();
+                await connection.start(TransferFormat.Text);
                 fail();
                 done();
             } catch (e) {
@@ -238,9 +238,9 @@ describe("HttpConnection", () => {
                 .on("GET", (r) => ""),
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org", options);
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
             fail();
             done();
         } catch (e) {
@@ -256,9 +256,9 @@ describe("HttpConnection", () => {
             transport: TransportType.WebSockets,
         } as IHttpConnectionOptions;
 
-        const connection = new HttpConnection("http://tempuri.org", TransferFormat.Text, options);
+        const connection = new HttpConnection("http://tempuri.org", options);
         try {
-            await connection.start();
+            await connection.start(TransferFormat.Text);
             fail();
             done();
         } catch (e) {
@@ -274,14 +274,21 @@ describe("HttpConnection", () => {
             // Force TypeScript to let us call the constructor incorrectly :)
             expect(() => new (HttpConnection as any)()).toThrowError("The 'url' argument is required.");
         });
+    });
 
+    describe("startAsync", () => {
         it("throws if no TransferFormat is provided", async () => {
-            // Force TypeScript to let us call the constructor incorrectly :)
+            // Force TypeScript to let us call start incorrectly
-            expect(() => new (HttpConnection as any)("http://tempuri.org")).toThrowError("The 'transferFormat' argument is required.");
+            const connection: any = new HttpConnection("http://tempuri.org");
+
+            expect(() => connection.start()).toThrowError("The 'transferFormat' argument is required.");
         });
 
         it("throws if an unsupported TransferFormat is provided", async () => {
-            expect(() => new HttpConnection("http://tempuri.org", 42)).toThrowError("Unknown transferFormat value: 42.");
+            // Force TypeScript to let us call start incorrectly
+            const connection: any = new HttpConnection("http://tempuri.org");
+
+            expect(() => connection.start(42)).toThrowError("Unknown transferFormat value: 42.");
         });
     });
 });

client-ts/signalr/src/HttpConnection.ts

@@ -39,19 +39,15 @@ export class HttpConnection implements IConnection {
     private readonly httpClient: HttpClient;
     private readonly logger: ILogger;
     private readonly options: IHttpConnectionOptions;
-    private readonly transferFormat: TransferFormat;
     private transport: ITransport;
     private connectionId: string;
     private startPromise: Promise<void>;
 
     public readonly features: any = {};
 
-    constructor(url: string, transferFormat: TransferFormat, options: IHttpConnectionOptions = {}) {
+    constructor(url: string, options: IHttpConnectionOptions = {}) {
         Arg.isRequired(url, "url");
-        Arg.isRequired(transferFormat, "transferFormat");
-        Arg.isIn(transferFormat, TransferFormat, "transferFormat");
 
-        this.transferFormat = transferFormat;
         this.logger = LoggerFactory.createLogger(options.logger);
         this.baseUrl = this.resolveUrl(url);
 
@@ -63,18 +59,21 @@ export class HttpConnection implements IConnection {
         this.options = options;
     }
 
-    public async start(): Promise<void> {
+    public start(transferFormat: TransferFormat): Promise<void> {
+        Arg.isRequired(transferFormat, "transferFormat");
+        Arg.isIn(transferFormat, TransferFormat, "transferFormat");
+
         if (this.connectionState !== ConnectionState.Disconnected) {
             return Promise.reject(new Error("Cannot start a connection that is not in the 'Disconnected' state."));
         }
 
         this.connectionState = ConnectionState.Connecting;
 
-        this.startPromise = this.startInternal();
+        this.startPromise = this.startInternal(transferFormat);
         return this.startPromise;
     }
 
-    private async startInternal(): Promise<void> {
+    private async startInternal(transferFormat: TransferFormat): Promise<void> {
         try {
             if (this.options.transport === TransportType.WebSockets) {
                 // No need to add a connection ID in this case
@@ -103,14 +102,14 @@ export class HttpConnection implements IConnection {
 
                 if (this.connectionId) {
                     this.url = this.baseUrl + (this.baseUrl.indexOf("?") === -1 ? "?" : "&") + `id=${this.connectionId}`;
-                    this.transport = this.createTransport(this.options.transport, negotiateResponse.availableTransports, this.transferFormat);
+                    this.transport = this.createTransport(this.options.transport, negotiateResponse.availableTransports, transferFormat);
                 }
             }
 
             this.transport.onreceive = this.onreceive;
             this.transport.onclose = (e) => this.stopConnection(true, e);
 
-            await this.transport.connect(this.url, this.transferFormat, this);
+            await this.transport.connect(this.url, transferFormat, this);
 
             // only change the state if we were connecting to not overwrite
             // the state if the connection is already marked as Disconnected
@@ -131,7 +130,7 @@ export class HttpConnection implements IConnection {
 
         for (const endpoint of availableTransports) {
             const transport = this.resolveTransport(endpoint, requestedTransport, requestedTransferFormat);
-            if (transport) {
+            if (typeof transport === "number") {
                 return this.constructTransport(transport);
             }
         }
@@ -154,19 +153,19 @@ export class HttpConnection implements IConnection {
 
     private resolveTransport(endpoint: IAvailableTransport, requestedTransport: TransportType, requestedTransferFormat: TransferFormat): TransportType | null {
         const transport = TransportType[endpoint.transport];
-        if (!transport) {
+        if (transport === null || transport === undefined) {
             this.logger.log(LogLevel.Trace, `Skipping transport '${endpoint.transport}' because it is not supported by this client.`);
         } else {
             const transferFormats = endpoint.transferFormats.map((s) => TransferFormat[s]);
             if (!requestedTransport || transport === requestedTransport) {
                 if (transferFormats.indexOf(requestedTransferFormat) >= 0) {
-                    this.logger.log(LogLevel.Trace, `Selecting transport '${transport}'`);
+                    this.logger.log(LogLevel.Trace, `Selecting transport '${TransportType[transport]}'`);
                     return transport;
                 } else {
-                    this.logger.log(LogLevel.Trace, `Skipping transport '${transport}' because it does not support the requested transfer format '${requestedTransferFormat}'.`);
+                    this.logger.log(LogLevel.Trace, `Skipping transport '${TransportType[transport]}' because it does not support the requested transfer format '${TransferFormat[requestedTransferFormat]}'.`);
                 }
             } else {
-                this.logger.log(LogLevel.Trace, `Skipping transport '${transport}' because it was disabled by the client.`);
+                this.logger.log(LogLevel.Trace, `Skipping transport '${TransportType[transport]}' because it was disabled by the client.`);
             }
         }
         return null;

client-ts/signalr/src/HubConnection.ts

@@ -42,7 +42,7 @@ export class HubConnection {
         this.protocol = options.protocol || new JsonHubProtocol();
 
         if (typeof urlOrConnection === "string") {
-            this.connection = new HttpConnection(urlOrConnection, this.protocol.transferFormat, options);
+            this.connection = new HttpConnection(urlOrConnection, options);
         } else {
             this.connection = urlOrConnection;
         }
@@ -133,7 +133,7 @@ export class HubConnection {
     }
 
     public async start(): Promise<void> {
-        await this.connection.start();
+        await this.connection.start(this.protocol.transferFormat);
 
         await this.connection.send(
             TextMessageFormat.write(

client-ts/signalr/src/IConnection.ts

@@ -2,12 +2,12 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 import { ConnectionClosed, DataReceived } from "./Common";
-import { ITransport, TransportType } from "./Transports";
+import { ITransport, TransferFormat, TransportType } from "./Transports";
 
 export interface IConnection {
     readonly features: any;
 
-    start(): Promise<void>;
+    start(transferFormat: TransferFormat): Promise<void>;
     send(data: any): Promise<void>;
     stop(error?: Error): Promise<void>;
 

client-ts/webdriver-tap-runner/bin.ts

@@ -0,0 +1,24 @@
+import * as path from "path";
+
+import * as yargs from "yargs";
+
+import * as _debug from "debug";
+import { run } from "./lib";
+const debug = _debug("webdriver-tap-runner:bin");
+
+const argv = yargs
+    .option("url", { demand: true, description: "The URL of the server to test against" })
+    .option("name", { demand: true, description: "The name of the test run" })
+    .option("browser", { alias: "b", default: "chrome", description: "The browser to use (only 'chrome' is supported right now)" })
+    .option("webdriver-port", { default: 9515, description: "The port on which to launch the WebDriver server", number: true })
+    .option("chrome-driver-log", { })
+    .option("chrome-driver-log-verbose", { })
+    .argv;
+
+run(argv.name, {
+    browser: argv.browser,
+    chromeDriverLogFile: argv["chrome-driver-log"],
+    chromeVerboseLogging: !!argv["chrome-driver-log-verbose"],
+    url: argv.url,
+    webdriverPort: argv["webdriver-port"],
+}).then((failures) => process.exit(failures));

client-ts/webdriver-tap-runner/lib.ts

@@ -0,0 +1,202 @@
+import * as http from "http";
+import * as path from "path";
+import { promisify } from "util";
+
+import { ChildProcess, spawn, SpawnOptions } from "child_process";
+import * as chromedriver from "chromedriver";
+import { EOL } from "os";
+import { Builder, logging, WebDriver, WebElement } from "selenium-webdriver";
+import { Options as ChromeOptions } from "selenium-webdriver/chrome";
+import { Readable, Writable } from "stream";
+
+import { delay, flushEntries, getEntryContent, getLogEntry, isComplete, waitForElement } from "./utils";
+
+import * as _debug from "debug";
+const debug = _debug("webdriver-tap-runner:bin");
+
+
+export interface RunnerOptions {
+    browser: string;
+    url: string;
+    chromeBinaryPath?: string,
+    chromeDriverLogFile?: string;
+    chromeVerboseLogging?: boolean;
+    output?: Writable;
+    webdriverPort: number;
+}
+
+function applyBrowserSettings(options: RunnerOptions, builder: Builder) {
+    if (options.browser === "chrome") {
+        const chromeOptions = new ChromeOptions();
+        chromeOptions.headless();
+
+        // If we're root, we need to disable the sandbox.
+        if (process.getuid && process.getuid() === 0) {
+            chromeOptions.addArguments("--no-sandbox");
+        }
+
+        if (options.chromeBinaryPath) {
+            debug(`Using Chrome Binary Path: ${options.chromeBinaryPath}`);
+            chromeOptions.setChromeBinaryPath(options.chromeBinaryPath);
+        }
+
+        builder.setChromeOptions(chromeOptions);
+    }
+}
+
+function writeToDebug(name: string) {
+    const writer = _debug(name);
+    let lastLine: string;
+    return (chunk: Buffer | string) => {
+        const str = chunk.toString();
+        const lines = str.split(/\r?\n/g);
+        const lastLineComplete = str.endsWith("\n");
+
+        if (lines.length > 0 && lastLine) {
+            lines[0] = lastLine + lines[0];
+        }
+
+        const end = lastLineComplete ? lines.length : (lines.length - 1)
+        for (let i = 0; i < end; i += 1) {
+            writer(lines[i]);
+        }
+        if (lastLineComplete && lines.length > 0) {
+            lastLine = lines[lines.length - 1];
+        }
+    }
+}
+
+let driverInstance: ChildProcess;
+function startDriver(browser: string, port: number) {
+    let processName: string;
+    if (browser === "chrome") {
+        processName = path.basename(chromedriver.path);
+        driverInstance = spawn(chromedriver.path, [`--port=${port}`]);
+    } else {
+        throw new Error(`Unsupported browser: ${browser}`);
+    }
+
+    // Capture output
+    driverInstance.stdout.on("data", writeToDebug(`webdriver-tap-runner:${processName}:stdout`));
+    driverInstance.stderr.on("data", writeToDebug(`webdriver-tap-runner:${processName}:stderr`));
+}
+
+function stopDriver(browser: string) {
+    if (driverInstance && !driverInstance.killed) {
+        debug("Killing WebDriver...");
+        driverInstance.kill();
+        debug("Killed WebDriver");
+    }
+}
+
+function pingUrl(url: string): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+        const request = http.request(url);
+        request.on("response", (resp: http.IncomingMessage) => {
+            if (resp.statusCode >= 400) {
+                reject(new Error(`Received ${resp.statusCode} ${resp.statusMessage} from server`));
+            } else {
+                resolve();
+            }
+        });
+        request.on("error", (error) => reject(error));
+        request.end();
+    });
+}
+
+async function pingWithRetry(url: string): Promise<boolean> {
+    for (let i = 0; i < 5; i += 1) {
+        try {
+            debug(`Pinging URL: ${url}`);
+            await pingUrl(url);
+            return true;
+        } catch (e) {
+            debug(`Error reaching server: '${e}', retrying...`);
+            await delay(100);
+        }
+    }
+    debug("Retry limit exhausted");
+    return false;
+}
+
+export async function run(runName: string, options: RunnerOptions): Promise<number> {
+    const output = options.output || (process.stdout as Writable);
+
+    debug(`Using WebDriver port: ${options.webdriverPort}`);
+
+    startDriver(options.browser, options.webdriverPort);
+
+    // Wait for the server to start
+    const serverUrl = `http://localhost:${options.webdriverPort}`;
+    if (!await pingWithRetry(`${serverUrl}/status`)) {
+        console.log("WebDriver did not start in time.");
+        process.exit(1);
+    }
+
+    try {
+        // Shut selenium down when we shut down.
+        process.on("exit", () => {
+            stopDriver(options.browser);
+        });
+
+        // Build WebDriver
+        const builder = new Builder()
+            .usingServer(serverUrl);
+
+        // Set the browser
+        debug(`Using '${options.browser}' browser`);
+        builder.forBrowser(options.browser);
+
+        applyBrowserSettings(options, builder);
+
+        // Build driver
+        const driver = builder.build();
+
+        let failureCount = 0;
+        try {
+            // Navigate to the URL
+            debug(`Navigating to ${options.url}`);
+            await driver.get(options.url);
+
+            // Wait for the TAP results list
+            const listElement = await waitForElement(driver, "__tap_list");
+
+            output.write(`TAP version 13${EOL}`);
+            output.write(`# ${runName}${EOL}`);
+
+            // Process messages until the test run is complete
+            let index = 0;
+            while (!await isComplete(listElement)) {
+                const entry = await getLogEntry(index, listElement);
+                if (entry) {
+                    index += 1;
+                    const content = await getEntryContent(entry);
+                    if (content.startsWith("not ok")) {
+                        failureCount += 1;
+                    }
+                    output.write(content + EOL);
+                }
+            }
+
+            // Flush any remaining messages
+            await flushEntries(index, listElement, (entry) => {
+                if (entry.startsWith("not ok")) {
+                    failureCount += 1;
+                }
+                output.write(entry + EOL);
+            });
+
+        } finally {
+            // Shut down
+            debug("Shutting WebDriver down...");
+            await driver.quit();
+        }
+
+        // We're done!
+        debug("Test run complete");
+        return failureCount;
+    } finally {
+        debug("Shutting Selenium server down...");
+        stopDriver(options.browser);
+    }
+}

client-ts/webdriver-tap-runner/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "selenium-tap-runner",
+  "version": "1.0.0",
+  "description": "Run Browser tests in a Selenium browser and proxy TAP results to the console",
+  "main": "dist/lib.js",
+  "scripts": {
+    "update-selenium": "selenium-standalone install",
+    "exec": "ts-node ./bin.ts",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "chromedriver": "^2.35.0",
+    "debug": "^3.1.0",
+    "selenium-webdriver": "^4.0.0-alpha.1",
+    "yargs": "^11.0.0"
+  },
+  "devDependencies": {
+    "@types/debug": "0.0.30",
+    "@types/selenium-webdriver": "^3.0.8",
+    "@types/yargs": "^11.0.0",
+    "ts-node": "^5.0.0"
+  }
+}

client-ts/webdriver-tap-runner/tsconfig.json

@@ -0,0 +1,7 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "lib": [ "es2016" ]
+    }
+}

client-ts/webdriver-tap-runner/utils.ts

@@ -0,0 +1,61 @@
+import * as os from "os";
+
+import { By, logging, WebDriver, WebElement } from "selenium-webdriver";
+
+import * as _debug from "debug";
+const debug = _debug("webdriver-tap-runner:utils");
+
+export function delay(ms: number): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+        setTimeout(resolve, ms);
+    });
+}
+
+export async function waitForElement(driver: WebDriver, id: string): Promise<WebElement> {
+    debug(`Waiting for '${id}' element`);
+    for (let attempts = 0; attempts < 2; attempts += 1) {
+        const elements = await driver.findElements(By.id(id));
+        if (elements && elements.length > 0) {
+            debug(`Found '${id}' element`);
+            return elements[0];
+        }
+
+        debug(`Waiting 5 sec for '${id}' element to appear...`);
+        await delay(5 * 1000);
+    }
+
+    // We failed to find the item
+    // Collect page source
+    const source = await driver.getPageSource();
+    const logs = await driver.manage().logs().get(logging.Type.BROWSER);
+    const messages = logs.map((l) => `[${l.level}] ${l.message}`)
+        .join(os.EOL);
+    throw new Error(
+        `Failed to find element '${id}'. Page Source:${os.EOL}${source}${os.EOL}` +
+        `Browser Logs (${logs.length} messages):${os.EOL}${messages}${os.EOL}`);
+}
+
+export async function isComplete(element: WebElement): Promise<boolean> {
+    return (await element.getAttribute("data-done")) === "1";
+}
+
+export async function getLogEntry(index: number, element: WebElement): Promise<WebElement> {
+    const elements = await element.findElements(By.id(`__tap_item_${index}`));
+    if (elements && elements.length > 0) {
+        return elements[0];
+    }
+    return null;
+}
+
+export async function getEntryContent(element: WebElement): Promise<string> {
+    return await element.getAttribute("value");
+}
+
+export async function flushEntries(index: number, element: WebElement, cb: (entry: string) => void): Promise<void> {
+    let entry = await getLogEntry(index, element);
+    while (entry) {
+        index += 1;
+        cb(await getEntryContent(entry));
+        entry = await getLogEntry(index, element);
+    }
+}