Make it possible to run the Browser Functional Tests from the command line (#1448)

client-ts/FunctionalTests/FunctionalTests.csproj

@@ -26,11 +26,6 @@
   </ItemGroup>
 
   <Target Name="ClientBuild" BeforeTargets="AfterBuild">
-      <!-- This will result in a double build of the JavaScript modules. Not sure the best way to deal with that right now. -->
-      <Exec Command="npm run build" WorkingDirectory="$(MSBuildThisFileDirectory)/.." />
-
-      <Exec Command="npm run build" />
-          
       <ItemGroup>
         <MsgPack5Files Include="$(MSBuildThisFileDirectory)../signalr-protocol-msgpack/node_modules/msgpack5/dist/*.js" />
       </ItemGroup>

client-ts/FunctionalTests/package.json

@@ -6,14 +6,25 @@
   "main": "index.js",
   "dependencies": {},
   "devDependencies": {
-    "es6-promise": "^4.2.2"
+    "@types/selenium-webdriver": "^3.0.8",
+    "@types/yargs": "^11.0.0",
+    "es6-promise": "^4.2.2",
+    "faucet": "0.0.1",
+    "selenium-webdriver": "^4.0.0-alpha.1",
+    "tap-mocha-reporter": "^3.0.6",
+    "tree-kill": "^1.2.0",
+    "ts-node": "^4.1.0",
+    "webdriver-manager": "^12.0.6",
+    "yargs": "^11.0.0"
   },
   "scripts": {
-    "clean": "node ../node_modules/rimraf/bin.js ./wwwroot/dist",
+    "clean": "node ../node_modules/rimraf/bin.js ./wwwroot/dist ./obj/js",
-    "build": "npm run build:lint && npm run build:tsc && npm run build:rollup",
+    "build": "npm run clean && npm run build:lint && npm run build:tsc && npm run build:rollup",
     "build:lint": "node ../node_modules/tslint/bin/tslint -c ../tslint.json -p ./tsconfig.json",
     "build:tsc": "node ../node_modules/typescript/bin/tsc --project ./tsconfig.json",
-    "build:rollup": "node ../node_modules/rollup/bin/rollup -c"
+    "build:rollup": "node ../node_modules/rollup/bin/rollup -c",
+    "rawtest": "ts-node --project ./run-tests.tsconfig.json ./run-tests.ts --browser chrome -h",
+    "test": "npm run rawtest | faucet"
   },
   "author": "",
   "license": "Apache-2.0"

client-ts/FunctionalTests/run-tests.ts

@@ -0,0 +1,258 @@
+// console.log messages should be prefixed with "#" to ensure stdout continues to conform to TAP (Test Anything Protocol)
+// https://testanything.org/tap-version-13-specification.html
+
+import { ChildProcess, spawn, spawnSync } from "child_process";
+import * as os from "os";
+import * as path from "path";
+import { Readable } from "stream";
+
+import { Builder, By, Capabilities, logging, WebDriver, WebElement } from "selenium-webdriver";
+import * as kill from "tree-kill";
+import { argv } from "yargs";
+
+const rootDir = __dirname;
+
+const verbose = argv.v || argv.verbose || false;
+const browser = argv.browser || "chrome";
+const headless = argv.headless || argv.h || false;
+
+let webDriver: ChildProcess;
+let dotnet: ChildProcess;
+
+console.log("TAP version 13");
+
+function logverbose(message: any) {
+    if (verbose) {
+        console.log(message);
+    }
+}
+
+function runCommand(command: string, args: string[]) {
+    args = args || [];
+    const result = spawnSync(command, args, {
+        cwd: rootDir,
+    });
+    if (result.status !== 0) {
+        console.error("Bail out!"); // Part of the TAP protocol
+        console.error(`Command ${command} ${args.join(" ")} failed:`);
+        console.error("stderr:");
+        console.error(result.stderr);
+        console.error("stdout:");
+        console.error(result.stdout);
+        shutdown(1);
+    }
+}
+
+const logExtractorRegex = /[^ ]+ [^ ]+ "(.*)"/;
+
+function getMessage(logMessage: string): string {
+    const r = logExtractorRegex.exec(logMessage);
+
+    // Unescape \"
+    if (r && r.length >= 2) {
+        return r[1].replace(/\\"/g, "\"");
+    } else {
+        return logMessage;
+    }
+}
+
+async function waitForElement(driver: WebDriver, id: string): Promise<WebElement> {
+    while (true) {
+        const elements = await driver.findElements(By.id(id));
+        if (elements && elements.length > 0) {
+            return elements[0];
+        }
+    }
+}
+
+async function isComplete(element: WebElement): Promise<boolean> {
+    return (await element.getAttribute("data-done")) === "1";
+}
+
+async function getLogEntry(index: number, element: WebElement): Promise<WebElement> {
+    const elements = await element.findElements(By.id(`__tap_item_${index}`));
+    if (elements && elements.length > 0) {
+        return elements[0];
+    }
+    return null;
+}
+
+async function getEntryContent(element: WebElement): Promise<string> {
+    return await element.getAttribute("innerHTML");
+}
+
+async function flushEntries(index: number, element: WebElement): Promise<void> {
+    let entry = await getLogEntry(index, element);
+    while (entry) {
+        index += 1;
+        console.log(await getEntryContent(entry));
+        entry = await getLogEntry(index, element);
+    }
+}
+
+function applyCapabilities(builder: Builder) {
+    if (browser === "chrome") {
+        const caps = Capabilities.chrome();
+        const args = [];
+
+        if (headless) {
+            console.log("# Using Headless Mode");
+            args.push("--headless");
+            if (process.platform === "win32") {
+                args.push("--disable-gpu");
+            }
+        }
+
+        caps.set("chromeOptions", {
+            args,
+        });
+        builder.withCapabilities(caps);
+    }
+}
+
+async function runTests(port: number, serverUrl: string): Promise<void> {
+    const webDriverUrl = `http://localhost:${port}/wd/hub`;
+    console.log(`# Using WebDriver at ${webDriverUrl}`);
+    console.log(`# Launching ${browser} browser`);
+    const logPrefs = new logging.Preferences();
+    logPrefs.setLevel(logging.Type.BROWSER, logging.Level.INFO);
+
+    const builder = new Builder()
+        .usingServer(webDriverUrl)
+        .setLoggingPrefs(logPrefs)
+        .forBrowser(browser);
+
+    applyCapabilities(builder);
+
+    const driver = await builder.build();
+    try {
+        await driver.get(serverUrl);
+
+        let index = 0;
+        console.log("# Running tests");
+        const element = await waitForElement(driver, "__tap_list");
+        const success = true;
+        while (!await isComplete(element)) {
+            const entry = await getLogEntry(index, element);
+            if (entry) {
+                index += 1;
+                console.log(await getEntryContent(entry));
+            }
+        }
+
+        // Flush remaining entries
+        await flushEntries(index, element);
+        console.log("# End of tests");
+    } catch (e) {
+        console.error("Error: " + e.toString());
+    } finally {
+        await driver.quit();
+    }
+}
+
+function waitForMatch(command: string, process: ChildProcess, regex: RegExp): Promise<RegExpMatchArray> {
+    return new Promise<RegExpMatchArray>((resolve, reject) => {
+        try {
+            let lastLine = "";
+
+            async function onData(this: Readable, chunk: string | Buffer): Promise<void> {
+                try {
+                    chunk = chunk.toString();
+
+                    // Process lines
+                    let lineEnd = chunk.indexOf(os.EOL);
+                    while (lineEnd >= 0) {
+                        const chunkLine = lastLine + chunk.substring(0, lineEnd);
+                        lastLine = "";
+
+                        chunk = chunk.substring(lineEnd + os.EOL.length);
+
+                        logverbose(`# ${command}: ${chunkLine}`);
+                        const results = regex.exec(chunkLine);
+                        if (results && results.length > 0) {
+                            this.removeAllListeners("data");
+                            resolve(results);
+                            return;
+                        }
+                        lineEnd = chunk.indexOf(os.EOL);
+                    }
+                    lastLine = chunk.toString();
+                } catch (e) {
+                    this.removeAllListeners("data");
+                    reject(e);
+                }
+            }
+
+            process.on("close", async (code, signal) => {
+                console.log(`# ${command} process exited with code: ${code}`);
+                await shutdown(1);
+            });
+
+            process.stdout.on("data", onData.bind(process.stdout));
+            process.stderr.on("data", onData.bind(process.stderr));
+        } catch (e) {
+            reject(e);
+        }
+    });
+}
+
+async function cleanUpProcess(name: string, process: ChildProcess): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+        try {
+            if (process && !process.killed) {
+                console.log(`# Killing ${name} process (PID: ${process.pid})`);
+                kill(process.pid, "SIGTERM", () => {
+                    console.log("# Killed dotnet process");
+                    resolve();
+                });
+            }
+            else {
+                resolve();
+            }
+        } catch (e) {
+            reject(e);
+        }
+    });
+}
+
+async function shutdown(code: number): Promise<void> {
+    await cleanUpProcess("dotnet", dotnet);
+    await cleanUpProcess("webDriver", webDriver);
+    process.exit(code);
+}
+
+// "async main" via IIFE
+(async function () {
+    const webDriverManagerPath = path.resolve(__dirname, "node_modules", "webdriver-manager", "bin", "webdriver-manager");
+
+    // This script launches the functional test app and then uses Selenium WebDriver to run the tests and verify the results.
+    console.log("# Updating WebDrivers...");
+    runCommand(process.execPath, [webDriverManagerPath, "update"]);
+    console.log("# Updated WebDrivers");
+
+    console.log("# Launching WebDriver...");
+    webDriver = spawn(process.execPath, [webDriverManagerPath, "start"]);
+
+    const webDriverRegex = /\d+:\d+:\d+.\d+ INFO - Selenium Server is up and running on port (\d+)/;
+
+    // The message we're waiting for is written to stderr for some reason
+    let results = await waitForMatch("webdriver-server", webDriver, webDriverRegex);
+    let webDriverPort = Number.parseInt(results[1]);
+
+    console.log("# WebDriver Launched");
+    console.log("# Launching Functional Test server...");
+    dotnet = spawn("dotnet", [path.resolve(__dirname, "bin", "Debug", "netcoreapp2.1", "FunctionalTests.dll")], {
+        cwd: rootDir,
+    });
+
+    const regex = /Now listening on: (http:\/\/localhost:([\d])+)/;
+    results = await waitForMatch("dotnet", dotnet, regex);
+    try {
+        console.log("# Functional Test server launched.");
+        await runTests(webDriverPort, results[1]);
+        await shutdown(0);
+    } catch (e) {
+        console.error(`Bail out! Error running tests: ${e}`);
+        await shutdown(1);
+    }
+})();

client-ts/FunctionalTests/run-tests.tsconfig.json

@@ -0,0 +1,5 @@
+{
+    "compilerOptions": {
+        "module": "commonjs"
+    }
+}

client-ts/FunctionalTests/ts/HubConnectionTests.ts

@@ -9,7 +9,7 @@ const TESTHUBENDPOINT_URL = "/testhub";
 
 describe("hubConnection", () => {
     eachTransportAndProtocol((transportType, protocol) => {
-        describe(protocol.name + " over " + TransportType[transportType] + " transport", () => {
+        describe("using " + protocol.name + " over " + TransportType[transportType] + " transport", () => {
             it("can invoke server method and receive result", (done) => {
                 const message = "你好，世界！";
 
@@ -465,7 +465,7 @@ describe("hubConnection", () => {
     });
 
     eachTransport((transportType) => {
-        describe(" over " + TransportType[transportType] + " transport", () => {
+        describe("over " + TransportType[transportType] + " transport", () => {
 
             it("can connect to hub with authorization", async (done) => {
                 const message = "你好，世界！";

client-ts/FunctionalTests/ts/Utils.ts

@@ -0,0 +1,14 @@
+
+export function getParameterByName(name: string) {
+    const url = window.location.href;
+    name = name.replace(/[\[\]]/g, "\\$&");
+    const regex = new RegExp("[?&]" + name + "(=([^&#]*)|&|#|$)");
+    const results = regex.exec(url);
+    if (!results) {
+        return null;
+    }
+    if (!results[2]) {
+        return "";
+    }
+    return decodeURIComponent(results[2].replace(/\+/g, " "));
+}

client-ts/FunctionalTests/ts/WebDriverReporter.ts

@@ -0,0 +1,83 @@
+import { getParameterByName } from "./Utils";
+
+function formatValue(v: any): string {
+    if (v === undefined) {
+        return "<undefined>";
+    } else if (v === null) {
+        return "<null>";
+    } else if (v.toString) {
+        return v.toString();
+    } else {
+        return v;
+    }
+}
+
+class WebDriverReporter implements jasmine.CustomReporter {
+    private element: HTMLUListElement;
+    private specCounter: number = 1; // TAP number start at 1
+    private recordCounter: number = 0;
+
+    constructor(private document: Document, show: boolean = false) {
+        // We write to the DOM because it's the most compatible way for WebDriver to read.
+        // For example, Chrome supports scraping console.log from WebDriver which would be ideal, but Firefox does not :(
+
+        // Create an element for the output
+        this.element = document.createElement("ul");
+        this.element.setAttribute("id", "__tap_list");
+
+        if (!show) {
+            this.element.setAttribute("style", "display: none");
+        }
+
+        document.body.appendChild(this.element);
+    }
+
+    public jasmineStarted(suiteInfo: jasmine.SuiteInfo): void {
+        this.taplog(`1..${suiteInfo.totalSpecsDefined}`);
+    }
+
+    public specDone(result: jasmine.CustomReporterResult): void {
+        if (result.status === "failed") {
+            this.taplog(`not ok ${this.specCounter} ${result.fullName}`);
+
+            // Include YAML block with failed expectations
+            this.taplog(" ---");
+            this.taplog(" failures:");
+            for (const expectation of result.failedExpectations) {
+                this.taplog(`   - message: ${expectation.message}`);
+                if (expectation.matcherName) {
+                    this.taplog(`     matcher: ${expectation.matcherName}`);
+                }
+                if (expectation.expected) {
+                    this.taplog(`     expected: ${formatValue(expectation.expected)}`);
+                }
+                if (expectation.actual) {
+                    this.taplog(`     actual: ${formatValue(expectation.actual)}`);
+                }
+            }
+            this.taplog(" ...");
+        } else {
+            this.taplog(`ok ${this.specCounter} ${result.fullName}`);
+        }
+
+        this.specCounter += 1;
+    }
+
+    public jasmineDone(runDetails: jasmine.RunDetails): void {
+        this.element.setAttribute("data-done", "1");
+    }
+
+    private taplog(msg: string) {
+        for (const line of msg.split(/\r|\n|\r\n/)) {
+            const li = this.document.createElement("li");
+            li.setAttribute("style", "font-family: monospace; white-space: pre");
+            li.setAttribute("id", `__tap_item_${this.recordCounter}`);
+            this.recordCounter += 1;
+
+            li.innerHTML = line;
+            this.element.appendChild(li);
+        }
+    }
+}
+
+jasmine.getEnv().addReporter(new WebDriverReporter(window.document, getParameterByName("displayTap") === "true"));

client-ts/FunctionalTests/ts/index.ts

@@ -3,6 +3,15 @@
 
 import "es6-promise/dist/es6-promise.auto.js";
 
+// Load SignalR
+import { getParameterByName } from "./Utils";
+
+const minified = getParameterByName("release") === "true" ? ".min" : "";
+document.write(
+    '<script type="text/javascript" src="lib/signalr/signalr' + minified + '.js"><\/script>' +
+    '<script type="text/javascript" src="lib/signalr/signalr-protocol-msgpack' + minified + '.js"><\/script>');
+
 import "./ConnectionTests";
 import "./HubConnectionTests";
+import "./WebDriverReporter";
 import "./WebSocketTests";

client-ts/FunctionalTests/tsconfig.json

@@ -15,7 +15,7 @@
     "lib": [ "es2015.promise", "es5", "dom", "es2015.collection" ]
   },
   "include": [
-    "./**/*",
+    "./ts/**/*",
     "../signalr/dist/esm/**/*.d.ts"
   ],
   "exclude": [

client-ts/FunctionalTests/wwwroot/default.html

@@ -6,6 +6,10 @@
     <meta name="viewport" content="width=device-width">
     <title>SignalR Client End-to-End tests</title>
     <link rel="stylesheet" href="lib/jasmine/jasmine.css" />
+</head>
+
+<body>
+
     <script type="text/javascript" src="lib/jasmine/jasmine.js"></script>
     <script type="text/javascript" src="lib/jasmine/jasmine-html.js"></script>
     <script type="text/javascript" src="lib/jasmine/boot.js"></script>
@@ -24,15 +28,12 @@
         }
 
         var minified = getParameterByName('release') === 'true' ? '.min' : '';
+        var cacheBust = Math.random().toString().substring(2);
         document.write(
-            '<script type="text/javascript" src="lib/signalr/signalr' + minified + '.js"><\/script>' +
+            '<script type="text/javascript" src="lib/signalr/signalr' + minified + '.js?' + cacheBust + '"><\/script>' +
-            '<script type="text/javascript" src="lib/signalr/signalr-protocol-msgpack' + minified + '.js"><\/script>');
+            '<script type="text/javascript" src="lib/signalr/signalr-protocol-msgpack' + minified + '.js?' + cacheBust + '"><\/script>' +
+            '<script type="text/javascript" src="dist/signalr-functional-tests.js?' + cacheBust + '"><\/script>');
     </script>
-
-    <script src="dist/signalr-functional-tests.js"></script>
-</head>
-
-<body>
 </body>
 
 </html>

client-ts/signalr-protocol-msgpack/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@aspnet/signalr-protocol-msgpack",
-  "version": "1.0.0-preview2-t000",
+  "version": "1.0.0-preview1-t000",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

client-ts/signalr/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@aspnet/signalr",
-  "version": "1.0.0-preview2-t000",
+  "version": "1.0.0-preview1-t000",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {