Add SignalR Daily Tests scripts (#4336)

This commit is contained in:
Andrew Stanton-Nurse 2018-12-05 10:04:23 -08:00 committed by BrennanConroy
parent 2e56f056d0
commit 1b9e655536
12 changed files with 260 additions and 37 deletions

View File

@ -0,0 +1,17 @@
# Uses Scheduled Triggers, which aren't supported in YAML yet.
# https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=vsts&tabs=yaml#scheduled
# Daily Tests for ASP.NET Core SignalR
# These use Sauce Labs resources, hence they run daily rather than per-commit.
# The only Daily Tests we have run in Sauce Labs and only need to run on one machine (because they just trigger SauceLabs)
# Hence we use the 'default-build.yml' template because it represents a single phase
jobs:
- template: jobs/default-build.yml
parameters:
buildDirectory: src/SignalR
buildArgs: "/p:DailyTests=true /p:SauceUser='$(asplab-sauce-labs-username)' /p:SauceKey='$(asplab-sauce-labs-access-key)'"
agentOs: Windows
jobName: SignalRDailyTests
jobDisplayName: "SignalR Daily Tests"

View File

@ -44,8 +44,28 @@
</Target>
<Target Name="RunBrowserTests">
<Message Text="Running JavaScript client Browser tests" Importance="high" />
<Exec Command="npm run test:inner -- --no-color --configuration $(Configuration)" WorkingDirectory="$(RepositoryRoot)clients/ts/FunctionalTests" IgnoreStandardErrorWarningFormat="true" />
<Message Text="Running JavaScript tests" Importance="high" />
<!-- Skip the "inner" test run when we're running DailyTests -->
<Exec Command="npm run test:inner -- --no-color --configuration $(Configuration)"
Condition="'$(DailyTests)' != 'true'"
WorkingDirectory="$(RepositoryRoot)clients/ts/FunctionalTests"
IgnoreStandardErrorWarningFormat="true" />
<!-- Optionally run "daily test" run in Sauce Labs -->
<Error Text="Required property 'SauceUser' is missing!" Condition="'$(DailyTests)' == 'true' And '$(SauceUser)' == ''" />
<Error Text="Required property 'SauceKey' is missing!" Condition="'$(DailyTests)' == 'true' And '$(SauceKey)' == ''" />
<PropertyGroup>
<BrowserTestHostName Condition="'$(CI)' == 'true'">sauce.local</BrowserTestHostName>
<_TestSauceArgs>--verbose --no-color --configuration $(Configuration) --sauce-user "$(SauceUser)" --sauce-key "$(SauceKey)"</_TestSauceArgs>
<_TestSauceArgs Condition="'$(BrowserTestHostName)' != ''">$(_TestSauceArgs) --use-hostname "$(BrowserTestHostName)"</_TestSauceArgs>
</PropertyGroup>
<Message Text="test:sauce Args = $(_TestSauceArgs)" Importance="high" />
<Exec Command="npm run test:sauce -- $(_TestSauceArgs)"
Condition="'$(DailyTests)' == 'true'"
WorkingDirectory="$(RepositoryRoot)clients/ts/FunctionalTests"
IgnoreStandardErrorWarningFormat="true" />
</Target>
<Target Name="RunJavaTests" Condition="'$(HasJava)' == 'true' AND '$(SkipJavaClient)' != 'true' ">

View File

@ -0,0 +1,44 @@
# Takes an input browser log file and splits it into separate files for each browser
param(
[Parameter(Mandatory = $true, Position = 0)][string]$InputFile,
[Parameter(Mandatory = $false)][string]$OutputDirectory
)
if (!$OutputDirectory) {
$OutputDirectory = Split-Path -Parent $InputFile
}
$browserParser = [regex]"(?<name>[a-zA-Z]*) (?<version>[^ ]*) \((?<os>[^\)]*)\)";
Write-Host "Processing log file...";
$browsers = @{}
Get-Content $InputFile | ForEach-Object {
$openSquare = $_.IndexOf("[");
$closeSquare = $_.IndexOf("]");
if (($openSquare -ge 0) -and ($closeSquare -ge 0)) {
$browser = $_.Substring($openSquare + 1, $closeSquare - 1);
$message = $_.Substring($closeSquare + 1).Trim();
# Parse the browser
$m = $browserParser.Match($browser)
if ($m.Success) {
$name = $m.Groups["name"].Value;
$version = $m.Groups["version"].Value;
$os = $m.Groups["os"].Value;
# Generate a new file name
$fileName = "$($name)_$($version.Replace(".", "_")).log"
$lines = $browsers[$fileName]
if (!$lines) {
$lines = @();
}
$browsers[$fileName] = $lines + $message
}
}
}
$browsers.Keys | ForEach-Object {
Write-Host "Writing to $_ ..."
$destination = Join-Path $OutputDirectory $_
[IO.File]::WriteAllText($destination, [string]::Join([Environment]::NewLine, $browsers[$_]))
}

View File

@ -31,6 +31,7 @@ try {
// Log browser messages to a file, not the terminal.
browserConsoleLogOptions: {
level: "debug",
format: "[%b] %T: %m",
terminal: false
},

View File

@ -16,7 +16,7 @@ try {
base: "SauceLabs",
browserName: "safari",
version: "latest",
platform: "OS X 10.13",
platform: "macOS 10.13",
},
// Google Chrome Latest, any OS.
@ -46,8 +46,25 @@ try {
};
// Mobile Browsers
// TODO: Fill this in.
var mobileBrowsers = {};
// These are a bit too slow and cause Karma to time out trying to "capture" the browser.
var mobileBrowsers = {
// // Latest iOS
// sl_ios_safari: {
// base: "SauceLabs",
// browserName: "Safari",
// deviceName: "iPhone XS Simulator",
// platformName: "iOS",
// platformVersion: "12.0",
// },
// // Latest Android Chrome
// sl_android_chrome: {
// base: "SauceLabs",
// browserName: "Chrome",
// platformName: "Android",
// platformVersion: "6.0",
// deviceName: "Android Emulator"
// }
};
var customLaunchers = {
...evergreenBrowsers,
@ -64,7 +81,11 @@ try {
connectOptions: {
// Required to enable WebSockets through the Sauce Connect proxy.
noSslBumpDomains: ["all"]
}
},
build: process.env.BUILD_BUILDNUMBER,
tags: ["aspnet-SignalR", "daily-tests"],
username: process.env.SAUCE_USERNAME,
accessKey: process.env.SAUCE_ACCESS_KEY
},
});
} catch (e) {

View File

@ -15,13 +15,21 @@ const debug = _debug("signalr-functional-tests:run");
const ARTIFACTS_DIR = path.resolve(__dirname, "..", "..", "..", "..", "artifacts");
const LOGS_DIR = path.resolve(ARTIFACTS_DIR, "logs");
const HOSTSFILE_PATH = process.platform === "win32" ? `${process.env.SystemRoot}\\System32\\drivers\\etc\\hosts` : null;
// Promisify things from fs we want to use.
const fs = {
createWriteStream: _fs.createWriteStream,
exists: promisify(_fs.exists),
mkdir: promisify(_fs.mkdir),
appendFile: promisify(_fs.appendFile),
readFile: promisify(_fs.readFile),
};
if (!_fs.existsSync(LOGS_DIR)) {
_fs.mkdirSync(LOGS_DIR);
}
process.on("unhandledRejection", (reason) => {
console.error(`Unhandled promise rejection: ${reason}`);
process.exit(1);
@ -96,6 +104,11 @@ let spec: string;
let sauce = false;
let allBrowsers = false;
let noColor = false;
let skipNode = false;
let sauceUser = null;
let sauceKey = null;
let publicIp = false;
let hostname = null;
for (let i = 2; i < process.argv.length; i += 1) {
switch (process.argv[i]) {
@ -119,6 +132,10 @@ for (let i = 2; i < process.argv.length; i += 1) {
sauce = true;
console.log("Running on SauceLabs.");
break;
case "--skip-node":
skipNode = true;
console.log("Running on SauceLabs.");
break;
case "-a":
case "--all-browsers":
allBrowsers = true;
@ -126,9 +143,29 @@ for (let i = 2; i < process.argv.length; i += 1) {
case "--no-color":
noColor = true;
break;
case "--sauce-user":
i += 1;
sauceUser = process.argv[i];
break;
case "--sauce-key":
i += 1;
sauceKey = process.argv[i];
break;
case "--use-hostname":
i += 1;
hostname = process.argv[i];
break;
}
}
if (sauceUser && !process.env.SAUCE_USERNAME) {
process.env.SAUCE_USERNAME = sauceUser;
}
if (sauceKey && !process.env.SAUCE_ACCESS_KEY) {
process.env.SAUCE_ACCESS_KEY = sauceKey;
}
const configFile = sauce ?
path.resolve(__dirname, "karma.sauce.conf.js") :
path.resolve(__dirname, "karma.local.conf.js");
@ -168,12 +205,17 @@ function runKarma(karmaConfig) {
}
function runJest(httpsUrl: string, httpUrl: string) {
if (skipNode) {
console.log("Skipping NodeJS tests because '--skip-node' was specified.");
return 0;
}
const jestPath = path.resolve(__dirname, "..", "..", "common", "node_modules", "jest", "bin", "jest.js");
const configPath = path.resolve(__dirname, "..", "func.jest.config.js");
console.log("Starting Node tests using Jest.");
return new Promise<number>((resolve, reject) => {
const logStream = fs.createWriteStream(path.resolve(__dirname, "..", "..", "..", "..", "artifacts", "logs", "node.functionaltests.log"));
const logStream = fs.createWriteStream(path.resolve(LOGS_DIR, "node.functionaltests.log"));
// Use NODE_TLS_REJECT_UNAUTHORIZED to allow our test cert to be used by the Node tests (NEVER use this environment variable outside of testing)
const p = exec(`"${process.execPath}" "${jestPath}" --config "${configPath}"`, { env: { SERVER_URL: `${httpsUrl};${httpUrl}`, NODE_TLS_REJECT_UNAUTHORIZED: 0 }, timeout: 200000, maxBuffer: 10 * 1024 * 1024 },
(error: any, stdout, stderr) => {
@ -199,7 +241,7 @@ function runJest(httpsUrl: string, httpUrl: string) {
if (sauce) {
// SauceLabs can only proxy certain ports for Edge and Safari.
// https://wiki.saucelabs.com/display/DOCS/Sauce+Connect+Proxy+FAQS
desiredServerUrl = "http://127.0.0.1:9000";
desiredServerUrl = "http://127.0.0.1:9000;https://127.0.0.1:9001";
}
const dotnet = spawn("dotnet", [serverPath], {
@ -216,7 +258,7 @@ function runJest(httpsUrl: string, httpUrl: string) {
}
}
const logStream = fs.createWriteStream(path.resolve(__dirname, "..", "..", "..", "..", "artifacts", "logs", "ts.functionaltests.dotnet.log"));
const logStream = fs.createWriteStream(path.resolve(LOGS_DIR, "ts.functionaltests.dotnet.log"));
dotnet.stdout.pipe(logStream);
process.on("SIGINT", cleanup);
@ -224,11 +266,27 @@ function runJest(httpsUrl: string, httpUrl: string) {
debug("Waiting for Functional Test Server to start");
const matches = await waitForMatches("dotnet", dotnet, /Now listening on: (https?:\/\/[^\/]+:[\d]+)/, 2);
const httpsUrl = matches[1];
const httpUrl = matches[3];
debug(`Functional Test Server has started at ${httpsUrl} and ${httpUrl}`);
debug(`Using SignalR Server: ${httpsUrl} and ${httpUrl}`);
// The order of HTTP and HTTPS isn't guaranteed
let httpsUrl;
let httpUrl;
if (matches[1].indexOf("https://") == 0) {
httpsUrl = matches[1];
} else if (matches[3].indexOf("https://") == 0) {
httpsUrl = matches[3];
}
if (matches[1].indexOf("http://") == 0) {
httpUrl = matches[1];
} else if (matches[3].indexOf("http://") == 0) {
httpUrl = matches[3];
}
if (!httpUrl || !httpsUrl) {
console.error("Unable to identify URLs");
process.exit(1);
}
debug(`Functional Test Server has started at ${httpsUrl} and ${httpUrl}`);
// Start karma server
const conf = {
@ -243,14 +301,48 @@ function runJest(httpsUrl: string, httpUrl: string) {
if (!await fs.exists(LOGS_DIR)) {
await fs.mkdir(LOGS_DIR);
}
conf.browserConsoleLogOptions.path = path.resolve(LOGS_DIR, `browserlogs.console.${new Date().toISOString().replace(/:|\./g, "-")}`);
conf.browserConsoleLogOptions.path = path.resolve(LOGS_DIR, `browserlogs.console.log`);
if (noColor) {
conf.colors = false;
}
if (hostname) {
if (process.platform !== "win32") {
throw new Error("Can't use '--use-hostname' on non-Windows platform.");
}
// Register a custom hostname in the hosts file (requires Admin, but AzDO agents run as Admin)
// Used to work around issues in Sauce Labs
debug(`Updating Hosts file (${HOSTSFILE_PATH}) to register host name '${hostname}'`);
await fs.appendFile(HOSTSFILE_PATH, `${EOL}127.0.0.1 ${hostname}${EOL}`);
conf.hostname = hostname;
// Rewrite the URL. Try with the host name and the IP address just to make sure
httpUrl = httpUrl.replace(/localhost/g, hostname);
httpsUrl = httpsUrl.replace(/localhost/g, hostname);
httpUrl = httpUrl.replace(/\d+\.\d+\.\d+\.\d+/g, hostname);
httpsUrl = httpsUrl.replace(/\d+\.\d+\.\d+\.\d+/g, hostname);
}
conf.client.args = [];
if (sauce) {
// Configure Sauce Connect logging
conf.sauceLabs.connectOptions.logfile = path.resolve(LOGS_DIR, "sauceConnect.log");
// Don't use https, Safari and Edge don't trust the cert.
httpsUrl = "";
conf.client.args = [...conf.client.args, '--sauce'];
}
debug(`Using SignalR Servers: ${httpsUrl} (https) and ${httpUrl} (http)`);
// Pass server URL to tests
conf.client.args = ["--server", `${httpsUrl};${httpUrl}`];
conf.client.args = [...conf.client.args, "--server", `${httpsUrl};${httpUrl}`];
debug(`Passing client args: ${conf.client.args.join(" ")}`);
const jestExit = await runJest(httpsUrl, httpUrl);

View File

@ -4,40 +4,55 @@
import { HttpTransportType, IHubProtocol, JsonHubProtocol } from "@aspnet/signalr";
import { MessagePackHubProtocol } from "@aspnet/signalr-protocol-msgpack";
export let ENDPOINT_BASE_URL: string;
export let ENDPOINT_BASE_HTTPS_URL: string;
// On slower CI machines, these tests sometimes take longer than 5s
jasmine.DEFAULT_TIMEOUT_INTERVAL = 10 * 1000;
export let ENDPOINT_BASE_URL: string = "";
export let ENDPOINT_BASE_HTTPS_URL: string = "";
if (typeof window !== "undefined" && (window as any).__karma__) {
const args = (window as any).__karma__.config.args as string[];
let httpsServer = "";
let httpServer = "";
let sauce = false;
for (let i = 0; i < args.length; i += 1) {
switch (args[i]) {
case "--server":
i += 1;
const urls = args[i].split(";");
httpsServer = urls[0];
httpServer = urls[1];
console.log(httpServer);
httpsServer = urls[0];
break;
case "--sauce":
sauce = true;
break;
}
}
// Increase test timeout in sauce because of the proxy
if (sauce) {
// Double the timeout.
jasmine.DEFAULT_TIMEOUT_INTERVAL *= 2;
}
// Running in Karma? Need to use an absolute URL
ENDPOINT_BASE_URL = httpServer;
ENDPOINT_BASE_HTTPS_URL = httpsServer;
console.log(`Using SignalR Server: ${ENDPOINT_BASE_URL}`);
} else if (typeof document !== "undefined") {
ENDPOINT_BASE_URL = `${document.location.protocol}//${document.location.host}`;
} else if (process && process.env && process.env.SERVER_URL) {
const urls = process.env.SERVER_URL.split(";");
ENDPOINT_BASE_HTTPS_URL = urls[0];
ENDPOINT_BASE_URL = urls[1];
ENDPOINT_BASE_HTTPS_URL = urls[0];
} else {
throw new Error("The server could not be found.");
}
console.log(`Using SignalR HTTP Server: '${ENDPOINT_BASE_URL}'`);
console.log(`Using SignalR HTTPS Server: '${ENDPOINT_BASE_HTTPS_URL}'`);
console.log(`Jasmine DEFAULT_TIMEOUT_INTERVAL: ${jasmine.DEFAULT_TIMEOUT_INTERVAL}`);
export const ECHOENDPOINT_URL = ENDPOINT_BASE_URL + "/echo";
export function getHttpTransportTypes(): HttpTransportType[] {

View File

@ -12,7 +12,8 @@ import "./LogBannerReporter";
import { TestLogger } from "./TestLogger";
const TESTHUBENDPOINT_URL = ENDPOINT_BASE_URL + "/testhub";
const TESTHUBENDPOINT_HTTPS_URL = ENDPOINT_BASE_HTTPS_URL + "/testhub";
const TESTHUBENDPOINT_HTTPS_URL = ENDPOINT_BASE_HTTPS_URL ? (ENDPOINT_BASE_HTTPS_URL + "/testhub") : undefined;
const TESTHUB_NOWEBSOCKETS_ENDPOINT_URL = ENDPOINT_BASE_URL + "/testhub-nowebsockets";
// On slower CI machines, these tests sometimes take longer than 5s
@ -22,6 +23,17 @@ const commonOptions: IHttpConnectionOptions = {
logMessageContent: true,
};
// Run test in Node or Chrome, but not on macOS
const shouldRunHttpsTests =
// Need to have an HTTPS URL
!!TESTHUBENDPOINT_HTTPS_URL &&
// Run on Node, unless macOS
(process && process.platform !== "darwin") &&
// Only run under Chrome browser
(typeof navigator === "undefined" || navigator.userAgent.search("Chrome") !== -1);
function getConnectionBuilder(transportType?: HttpTransportType, url?: string, options?: IHttpConnectionOptions): HubConnectionBuilder {
let actualOptions: IHttpConnectionOptions = options || {};
if (transportType) {
@ -63,8 +75,7 @@ describe("hubConnection", () => {
});
});
// Run test in Node or Chrome, but not on macOS
if ((process && process.platform !== "darwin") && (typeof navigator === "undefined" || navigator.userAgent.search("Chrome") !== -1)) {
if (shouldRunHttpsTests) {
it("using https, can invoke server method and receive result", (done) => {
const message = "你好,世界!";

View File

@ -1,4 +1,6 @@
export class LogBannerReporter implements jasmine.CustomReporter {
private lastTestStarted?: Date;
public jasmineStarted(): void {
console.log("*** JASMINE SUITE STARTED ***");
}
@ -8,11 +10,16 @@ export class LogBannerReporter implements jasmine.CustomReporter {
}
public specStarted(result: jasmine.CustomReporterResult): void {
console.log(`*** SPEC STARTED: ${result.fullName} ***`);
const timestamp = new Date();
this.lastTestStarted = timestamp;
console.log(`*** SPEC STARTED: ${result.fullName} [${timestamp.toISOString()}] ***`);
}
public specDone(result: jasmine.CustomReporterResult): void {
console.log(`*** SPEC DONE: ${result.fullName} ***`);
const timestamp = new Date();
const duration = this.lastTestStarted ? `${timestamp.getTime() - this.lastTestStarted.getTime()}ms` : "<<unknown>>";
console.log(`*** SPEC DONE: ${result.fullName} [${timestamp.toISOString()}; Duration: ${duration}] ***`);
}
}

View File

@ -2601,8 +2601,7 @@
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"aproba": {
"version": "1.2.0",
@ -3017,8 +3016,7 @@
"safe-buffer": {
"version": "5.1.1",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"safer-buffer": {
"version": "2.1.2",
@ -3074,7 +3072,6 @@
"version": "3.0.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -3118,14 +3115,12 @@
"wrappy": {
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"yallist": {
"version": "3.0.2",
"bundled": true,
"dev": true,
"optional": true
"dev": true
}
}
},

View File

@ -1,6 +1,6 @@
{
"name": "@aspnet/signalr-protocol-msgpack",
"version": "3.0.0-alpha1-t000",
"version": "3.0.0-dev",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

View File

@ -1,6 +1,6 @@
{
"name": "@aspnet/signalr",
"version": "3.0.0-alpha1-t000",
"version": "3.0.0-dev",
"lockfileVersion": 1,
"requires": true,
"dependencies": {