From 1b9e65553636d4a9ecaf3534abea7eac7b60b1bf Mon Sep 17 00:00:00 2001 From: Andrew Stanton-Nurse Date: Wed, 5 Dec 2018 10:04:23 -0800 Subject: [PATCH] Add SignalR Daily Tests scripts (#4336) --- .azure/pipelines/signalr-daily-tests.yml | 17 +++ src/SignalR/build/repo.targets | 24 +++- src/SignalR/build/splat-browser-logs.ps1 | 44 +++++++ .../scripts/karma.base.conf.js | 1 + .../scripts/karma.sauce.conf.js | 29 ++++- .../ts/FunctionalTests/scripts/run-tests.ts | 110 ++++++++++++++++-- .../clients/ts/FunctionalTests/ts/Common.ts | 27 ++++- .../FunctionalTests/ts/HubConnectionTests.ts | 17 ++- .../FunctionalTests/ts/LogBannerReporter.ts | 11 +- .../clients/ts/common/package-lock.json | 13 +-- .../package-lock.json | 2 +- .../clients/ts/signalr/package-lock.json | 2 +- 12 files changed, 260 insertions(+), 37 deletions(-) create mode 100644 .azure/pipelines/signalr-daily-tests.yml create mode 100644 src/SignalR/build/splat-browser-logs.ps1 diff --git a/.azure/pipelines/signalr-daily-tests.yml b/.azure/pipelines/signalr-daily-tests.yml new file mode 100644 index 0000000000..1b13105f63 --- /dev/null +++ b/.azure/pipelines/signalr-daily-tests.yml @@ -0,0 +1,17 @@ +# Uses Scheduled Triggers, which aren't supported in YAML yet. +# https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=vsts&tabs=yaml#scheduled + +# Daily Tests for ASP.NET Core SignalR +# These use Sauce Labs resources, hence they run daily rather than per-commit. + +# The only Daily Tests we have run in Sauce Labs and only need to run on one machine (because they just trigger SauceLabs) +# Hence we use the 'default-build.yml' template because it represents a single phase +jobs: +- template: jobs/default-build.yml + parameters: + buildDirectory: src/SignalR + buildArgs: "/p:DailyTests=true /p:SauceUser='$(asplab-sauce-labs-username)' /p:SauceKey='$(asplab-sauce-labs-access-key)'" + agentOs: Windows + jobName: SignalRDailyTests + jobDisplayName: "SignalR Daily Tests" + diff --git a/src/SignalR/build/repo.targets b/src/SignalR/build/repo.targets index ce7a1193e2..a307f480d4 100644 --- a/src/SignalR/build/repo.targets +++ b/src/SignalR/build/repo.targets @@ -44,8 +44,28 @@ - - + + + + + + + + + + + sauce.local + <_TestSauceArgs>--verbose --no-color --configuration $(Configuration) --sauce-user "$(SauceUser)" --sauce-key "$(SauceKey)" + <_TestSauceArgs Condition="'$(BrowserTestHostName)' != ''">$(_TestSauceArgs) --use-hostname "$(BrowserTestHostName)" + + + diff --git a/src/SignalR/build/splat-browser-logs.ps1 b/src/SignalR/build/splat-browser-logs.ps1 new file mode 100644 index 0000000000..85266f57b4 --- /dev/null +++ b/src/SignalR/build/splat-browser-logs.ps1 @@ -0,0 +1,44 @@ +# Takes an input browser log file and splits it into separate files for each browser +param( + [Parameter(Mandatory = $true, Position = 0)][string]$InputFile, + [Parameter(Mandatory = $false)][string]$OutputDirectory +) + +if (!$OutputDirectory) { + $OutputDirectory = Split-Path -Parent $InputFile +} + +$browserParser = [regex]"(?[a-zA-Z]*) (?[^ ]*) \((?[^\)]*)\)"; +Write-Host "Processing log file..."; +$browsers = @{} +Get-Content $InputFile | ForEach-Object { + $openSquare = $_.IndexOf("["); + $closeSquare = $_.IndexOf("]"); + if (($openSquare -ge 0) -and ($closeSquare -ge 0)) { + $browser = $_.Substring($openSquare + 1, $closeSquare - 1); + $message = $_.Substring($closeSquare + 1).Trim(); + + # Parse the browser + $m = $browserParser.Match($browser) + if ($m.Success) { + $name = $m.Groups["name"].Value; + $version = $m.Groups["version"].Value; + $os = $m.Groups["os"].Value; + + # Generate a new file name + $fileName = "$($name)_$($version.Replace(".", "_")).log" + $lines = $browsers[$fileName] + if (!$lines) { + $lines = @(); + } + + $browsers[$fileName] = $lines + $message + } + } +} + +$browsers.Keys | ForEach-Object { + Write-Host "Writing to $_ ..." + $destination = Join-Path $OutputDirectory $_ + [IO.File]::WriteAllText($destination, [string]::Join([Environment]::NewLine, $browsers[$_])) +} \ No newline at end of file diff --git a/src/SignalR/clients/ts/FunctionalTests/scripts/karma.base.conf.js b/src/SignalR/clients/ts/FunctionalTests/scripts/karma.base.conf.js index e1a8953f8c..d71f326f1f 100644 --- a/src/SignalR/clients/ts/FunctionalTests/scripts/karma.base.conf.js +++ b/src/SignalR/clients/ts/FunctionalTests/scripts/karma.base.conf.js @@ -31,6 +31,7 @@ try { // Log browser messages to a file, not the terminal. browserConsoleLogOptions: { level: "debug", + format: "[%b] %T: %m", terminal: false }, diff --git a/src/SignalR/clients/ts/FunctionalTests/scripts/karma.sauce.conf.js b/src/SignalR/clients/ts/FunctionalTests/scripts/karma.sauce.conf.js index 46d78a64d6..5ebebba045 100644 --- a/src/SignalR/clients/ts/FunctionalTests/scripts/karma.sauce.conf.js +++ b/src/SignalR/clients/ts/FunctionalTests/scripts/karma.sauce.conf.js @@ -16,7 +16,7 @@ try { base: "SauceLabs", browserName: "safari", version: "latest", - platform: "OS X 10.13", + platform: "macOS 10.13", }, // Google Chrome Latest, any OS. @@ -46,8 +46,25 @@ try { }; // Mobile Browsers - // TODO: Fill this in. - var mobileBrowsers = {}; + // These are a bit too slow and cause Karma to time out trying to "capture" the browser. + var mobileBrowsers = { + // // Latest iOS + // sl_ios_safari: { + // base: "SauceLabs", + // browserName: "Safari", + // deviceName: "iPhone XS Simulator", + // platformName: "iOS", + // platformVersion: "12.0", + // }, + // // Latest Android Chrome + // sl_android_chrome: { + // base: "SauceLabs", + // browserName: "Chrome", + // platformName: "Android", + // platformVersion: "6.0", + // deviceName: "Android Emulator" + // } + }; var customLaunchers = { ...evergreenBrowsers, @@ -64,7 +81,11 @@ try { connectOptions: { // Required to enable WebSockets through the Sauce Connect proxy. noSslBumpDomains: ["all"] - } + }, + build: process.env.BUILD_BUILDNUMBER, + tags: ["aspnet-SignalR", "daily-tests"], + username: process.env.SAUCE_USERNAME, + accessKey: process.env.SAUCE_ACCESS_KEY }, }); } catch (e) { diff --git a/src/SignalR/clients/ts/FunctionalTests/scripts/run-tests.ts b/src/SignalR/clients/ts/FunctionalTests/scripts/run-tests.ts index 5ae55bad6c..5e11511d9a 100644 --- a/src/SignalR/clients/ts/FunctionalTests/scripts/run-tests.ts +++ b/src/SignalR/clients/ts/FunctionalTests/scripts/run-tests.ts @@ -15,13 +15,21 @@ const debug = _debug("signalr-functional-tests:run"); const ARTIFACTS_DIR = path.resolve(__dirname, "..", "..", "..", "..", "artifacts"); const LOGS_DIR = path.resolve(ARTIFACTS_DIR, "logs"); +const HOSTSFILE_PATH = process.platform === "win32" ? `${process.env.SystemRoot}\\System32\\drivers\\etc\\hosts` : null; + // Promisify things from fs we want to use. const fs = { createWriteStream: _fs.createWriteStream, exists: promisify(_fs.exists), mkdir: promisify(_fs.mkdir), + appendFile: promisify(_fs.appendFile), + readFile: promisify(_fs.readFile), }; +if (!_fs.existsSync(LOGS_DIR)) { + _fs.mkdirSync(LOGS_DIR); +} + process.on("unhandledRejection", (reason) => { console.error(`Unhandled promise rejection: ${reason}`); process.exit(1); @@ -96,6 +104,11 @@ let spec: string; let sauce = false; let allBrowsers = false; let noColor = false; +let skipNode = false; +let sauceUser = null; +let sauceKey = null; +let publicIp = false; +let hostname = null; for (let i = 2; i < process.argv.length; i += 1) { switch (process.argv[i]) { @@ -119,6 +132,10 @@ for (let i = 2; i < process.argv.length; i += 1) { sauce = true; console.log("Running on SauceLabs."); break; + case "--skip-node": + skipNode = true; + console.log("Running on SauceLabs."); + break; case "-a": case "--all-browsers": allBrowsers = true; @@ -126,9 +143,29 @@ for (let i = 2; i < process.argv.length; i += 1) { case "--no-color": noColor = true; break; + case "--sauce-user": + i += 1; + sauceUser = process.argv[i]; + break; + case "--sauce-key": + i += 1; + sauceKey = process.argv[i]; + break; + case "--use-hostname": + i += 1; + hostname = process.argv[i]; + break; } } +if (sauceUser && !process.env.SAUCE_USERNAME) { + process.env.SAUCE_USERNAME = sauceUser; +} + +if (sauceKey && !process.env.SAUCE_ACCESS_KEY) { + process.env.SAUCE_ACCESS_KEY = sauceKey; +} + const configFile = sauce ? path.resolve(__dirname, "karma.sauce.conf.js") : path.resolve(__dirname, "karma.local.conf.js"); @@ -168,12 +205,17 @@ function runKarma(karmaConfig) { } function runJest(httpsUrl: string, httpUrl: string) { + if (skipNode) { + console.log("Skipping NodeJS tests because '--skip-node' was specified."); + return 0; + } + const jestPath = path.resolve(__dirname, "..", "..", "common", "node_modules", "jest", "bin", "jest.js"); const configPath = path.resolve(__dirname, "..", "func.jest.config.js"); console.log("Starting Node tests using Jest."); return new Promise((resolve, reject) => { - const logStream = fs.createWriteStream(path.resolve(__dirname, "..", "..", "..", "..", "artifacts", "logs", "node.functionaltests.log")); + const logStream = fs.createWriteStream(path.resolve(LOGS_DIR, "node.functionaltests.log")); // Use NODE_TLS_REJECT_UNAUTHORIZED to allow our test cert to be used by the Node tests (NEVER use this environment variable outside of testing) const p = exec(`"${process.execPath}" "${jestPath}" --config "${configPath}"`, { env: { SERVER_URL: `${httpsUrl};${httpUrl}`, NODE_TLS_REJECT_UNAUTHORIZED: 0 }, timeout: 200000, maxBuffer: 10 * 1024 * 1024 }, (error: any, stdout, stderr) => { @@ -199,7 +241,7 @@ function runJest(httpsUrl: string, httpUrl: string) { if (sauce) { // SauceLabs can only proxy certain ports for Edge and Safari. // https://wiki.saucelabs.com/display/DOCS/Sauce+Connect+Proxy+FAQS - desiredServerUrl = "http://127.0.0.1:9000"; + desiredServerUrl = "http://127.0.0.1:9000;https://127.0.0.1:9001"; } const dotnet = spawn("dotnet", [serverPath], { @@ -216,7 +258,7 @@ function runJest(httpsUrl: string, httpUrl: string) { } } - const logStream = fs.createWriteStream(path.resolve(__dirname, "..", "..", "..", "..", "artifacts", "logs", "ts.functionaltests.dotnet.log")); + const logStream = fs.createWriteStream(path.resolve(LOGS_DIR, "ts.functionaltests.dotnet.log")); dotnet.stdout.pipe(logStream); process.on("SIGINT", cleanup); @@ -224,11 +266,27 @@ function runJest(httpsUrl: string, httpUrl: string) { debug("Waiting for Functional Test Server to start"); const matches = await waitForMatches("dotnet", dotnet, /Now listening on: (https?:\/\/[^\/]+:[\d]+)/, 2); - const httpsUrl = matches[1]; - const httpUrl = matches[3]; - debug(`Functional Test Server has started at ${httpsUrl} and ${httpUrl}`); - debug(`Using SignalR Server: ${httpsUrl} and ${httpUrl}`); + // The order of HTTP and HTTPS isn't guaranteed + let httpsUrl; + let httpUrl; + if (matches[1].indexOf("https://") == 0) { + httpsUrl = matches[1]; + } else if (matches[3].indexOf("https://") == 0) { + httpsUrl = matches[3]; + } + if (matches[1].indexOf("http://") == 0) { + httpUrl = matches[1]; + } else if (matches[3].indexOf("http://") == 0) { + httpUrl = matches[3]; + } + + if (!httpUrl || !httpsUrl) { + console.error("Unable to identify URLs"); + process.exit(1); + } + + debug(`Functional Test Server has started at ${httpsUrl} and ${httpUrl}`); // Start karma server const conf = { @@ -243,14 +301,48 @@ function runJest(httpsUrl: string, httpUrl: string) { if (!await fs.exists(LOGS_DIR)) { await fs.mkdir(LOGS_DIR); } - conf.browserConsoleLogOptions.path = path.resolve(LOGS_DIR, `browserlogs.console.${new Date().toISOString().replace(/:|\./g, "-")}`); + conf.browserConsoleLogOptions.path = path.resolve(LOGS_DIR, `browserlogs.console.log`); if (noColor) { conf.colors = false; } + if (hostname) { + if (process.platform !== "win32") { + throw new Error("Can't use '--use-hostname' on non-Windows platform."); + } + + // Register a custom hostname in the hosts file (requires Admin, but AzDO agents run as Admin) + // Used to work around issues in Sauce Labs + debug(`Updating Hosts file (${HOSTSFILE_PATH}) to register host name '${hostname}'`); + await fs.appendFile(HOSTSFILE_PATH, `${EOL}127.0.0.1 ${hostname}${EOL}`); + + conf.hostname = hostname; + + // Rewrite the URL. Try with the host name and the IP address just to make sure + httpUrl = httpUrl.replace(/localhost/g, hostname); + httpsUrl = httpsUrl.replace(/localhost/g, hostname); + httpUrl = httpUrl.replace(/\d+\.\d+\.\d+\.\d+/g, hostname); + httpsUrl = httpsUrl.replace(/\d+\.\d+\.\d+\.\d+/g, hostname); + } + + conf.client.args = []; + + if (sauce) { + // Configure Sauce Connect logging + conf.sauceLabs.connectOptions.logfile = path.resolve(LOGS_DIR, "sauceConnect.log"); + + // Don't use https, Safari and Edge don't trust the cert. + httpsUrl = ""; + + conf.client.args = [...conf.client.args, '--sauce']; + } + + debug(`Using SignalR Servers: ${httpsUrl} (https) and ${httpUrl} (http)`); + // Pass server URL to tests - conf.client.args = ["--server", `${httpsUrl};${httpUrl}`]; + conf.client.args = [...conf.client.args, "--server", `${httpsUrl};${httpUrl}`]; + debug(`Passing client args: ${conf.client.args.join(" ")}`); const jestExit = await runJest(httpsUrl, httpUrl); diff --git a/src/SignalR/clients/ts/FunctionalTests/ts/Common.ts b/src/SignalR/clients/ts/FunctionalTests/ts/Common.ts index 37b9b532fb..86bfe113fd 100644 --- a/src/SignalR/clients/ts/FunctionalTests/ts/Common.ts +++ b/src/SignalR/clients/ts/FunctionalTests/ts/Common.ts @@ -4,40 +4,55 @@ import { HttpTransportType, IHubProtocol, JsonHubProtocol } from "@aspnet/signalr"; import { MessagePackHubProtocol } from "@aspnet/signalr-protocol-msgpack"; -export let ENDPOINT_BASE_URL: string; -export let ENDPOINT_BASE_HTTPS_URL: string; +// On slower CI machines, these tests sometimes take longer than 5s +jasmine.DEFAULT_TIMEOUT_INTERVAL = 10 * 1000; + +export let ENDPOINT_BASE_URL: string = ""; +export let ENDPOINT_BASE_HTTPS_URL: string = ""; if (typeof window !== "undefined" && (window as any).__karma__) { const args = (window as any).__karma__.config.args as string[]; let httpsServer = ""; let httpServer = ""; + let sauce = false; for (let i = 0; i < args.length; i += 1) { switch (args[i]) { case "--server": i += 1; const urls = args[i].split(";"); - httpsServer = urls[0]; httpServer = urls[1]; - console.log(httpServer); + httpsServer = urls[0]; + break; + case "--sauce": + sauce = true; break; } } + // Increase test timeout in sauce because of the proxy + if (sauce) { + // Double the timeout. + jasmine.DEFAULT_TIMEOUT_INTERVAL *= 2; + } + // Running in Karma? Need to use an absolute URL ENDPOINT_BASE_URL = httpServer; ENDPOINT_BASE_HTTPS_URL = httpsServer; - console.log(`Using SignalR Server: ${ENDPOINT_BASE_URL}`); } else if (typeof document !== "undefined") { ENDPOINT_BASE_URL = `${document.location.protocol}//${document.location.host}`; } else if (process && process.env && process.env.SERVER_URL) { const urls = process.env.SERVER_URL.split(";"); - ENDPOINT_BASE_HTTPS_URL = urls[0]; ENDPOINT_BASE_URL = urls[1]; + ENDPOINT_BASE_HTTPS_URL = urls[0]; } else { throw new Error("The server could not be found."); } +console.log(`Using SignalR HTTP Server: '${ENDPOINT_BASE_URL}'`); +console.log(`Using SignalR HTTPS Server: '${ENDPOINT_BASE_HTTPS_URL}'`); +console.log(`Jasmine DEFAULT_TIMEOUT_INTERVAL: ${jasmine.DEFAULT_TIMEOUT_INTERVAL}`); + export const ECHOENDPOINT_URL = ENDPOINT_BASE_URL + "/echo"; export function getHttpTransportTypes(): HttpTransportType[] { diff --git a/src/SignalR/clients/ts/FunctionalTests/ts/HubConnectionTests.ts b/src/SignalR/clients/ts/FunctionalTests/ts/HubConnectionTests.ts index b7665cb306..1fda5080cd 100644 --- a/src/SignalR/clients/ts/FunctionalTests/ts/HubConnectionTests.ts +++ b/src/SignalR/clients/ts/FunctionalTests/ts/HubConnectionTests.ts @@ -12,7 +12,8 @@ import "./LogBannerReporter"; import { TestLogger } from "./TestLogger"; const TESTHUBENDPOINT_URL = ENDPOINT_BASE_URL + "/testhub"; -const TESTHUBENDPOINT_HTTPS_URL = ENDPOINT_BASE_HTTPS_URL + "/testhub"; +const TESTHUBENDPOINT_HTTPS_URL = ENDPOINT_BASE_HTTPS_URL ? (ENDPOINT_BASE_HTTPS_URL + "/testhub") : undefined; + const TESTHUB_NOWEBSOCKETS_ENDPOINT_URL = ENDPOINT_BASE_URL + "/testhub-nowebsockets"; // On slower CI machines, these tests sometimes take longer than 5s @@ -22,6 +23,17 @@ const commonOptions: IHttpConnectionOptions = { logMessageContent: true, }; +// Run test in Node or Chrome, but not on macOS +const shouldRunHttpsTests = + // Need to have an HTTPS URL + !!TESTHUBENDPOINT_HTTPS_URL && + + // Run on Node, unless macOS + (process && process.platform !== "darwin") && + + // Only run under Chrome browser + (typeof navigator === "undefined" || navigator.userAgent.search("Chrome") !== -1); + function getConnectionBuilder(transportType?: HttpTransportType, url?: string, options?: IHttpConnectionOptions): HubConnectionBuilder { let actualOptions: IHttpConnectionOptions = options || {}; if (transportType) { @@ -63,8 +75,7 @@ describe("hubConnection", () => { }); }); - // Run test in Node or Chrome, but not on macOS - if ((process && process.platform !== "darwin") && (typeof navigator === "undefined" || navigator.userAgent.search("Chrome") !== -1)) { + if (shouldRunHttpsTests) { it("using https, can invoke server method and receive result", (done) => { const message = "你好，世界！"; diff --git a/src/SignalR/clients/ts/FunctionalTests/ts/LogBannerReporter.ts b/src/SignalR/clients/ts/FunctionalTests/ts/LogBannerReporter.ts index 98d28da93a..387d52151e 100644 --- a/src/SignalR/clients/ts/FunctionalTests/ts/LogBannerReporter.ts +++ b/src/SignalR/clients/ts/FunctionalTests/ts/LogBannerReporter.ts @@ -1,4 +1,6 @@ export class LogBannerReporter implements jasmine.CustomReporter { + private lastTestStarted?: Date; + public jasmineStarted(): void { console.log("*** JASMINE SUITE STARTED ***"); } @@ -8,11 +10,16 @@ export class LogBannerReporter implements jasmine.CustomReporter { } public specStarted(result: jasmine.CustomReporterResult): void { - console.log(`*** SPEC STARTED: ${result.fullName} ***`); + const timestamp = new Date(); + this.lastTestStarted = timestamp; + console.log(`*** SPEC STARTED: ${result.fullName} [${timestamp.toISOString()}] ***`); } public specDone(result: jasmine.CustomReporterResult): void { - console.log(`*** SPEC DONE: ${result.fullName} ***`); + const timestamp = new Date(); + + const duration = this.lastTestStarted ? `${timestamp.getTime() - this.lastTestStarted.getTime()}ms` : "<>"; + console.log(`*** SPEC DONE: ${result.fullName} [${timestamp.toISOString()}; Duration: ${duration}] ***`); } } diff --git a/src/SignalR/clients/ts/common/package-lock.json b/src/SignalR/clients/ts/common/package-lock.json index 7ad4e6a391..8b3564efc7 100644 --- a/src/SignalR/clients/ts/common/package-lock.json +++ b/src/SignalR/clients/ts/common/package-lock.json @@ -2601,8 +2601,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -3017,8 +3016,7 @@ "safe-buffer": { "version": "5.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -3074,7 +3072,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -3118,14 +3115,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, diff --git a/src/SignalR/clients/ts/signalr-protocol-msgpack/package-lock.json b/src/SignalR/clients/ts/signalr-protocol-msgpack/package-lock.json index 68493e002a..565f657642 100644 --- a/src/SignalR/clients/ts/signalr-protocol-msgpack/package-lock.json +++ b/src/SignalR/clients/ts/signalr-protocol-msgpack/package-lock.json @@ -1,6 +1,6 @@ { "name": "@aspnet/signalr-protocol-msgpack", - "version": "3.0.0-alpha1-t000", + "version": "3.0.0-dev", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/SignalR/clients/ts/signalr/package-lock.json b/src/SignalR/clients/ts/signalr/package-lock.json index 9bedb9db53..a7d4e4a695 100644 --- a/src/SignalR/clients/ts/signalr/package-lock.json +++ b/src/SignalR/clients/ts/signalr/package-lock.json @@ -1,6 +1,6 @@ { "name": "@aspnet/signalr", - "version": "3.0.0-alpha1-t000", + "version": "3.0.0-dev", "lockfileVersion": 1, "requires": true, "dependencies": {