microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-09-20 00:41:16 +03:00
Code Issues Projects Releases Wiki Activity

test: remove a copy of folio, use upstream (#6080)

Browse Source
This commit is contained in:
Dmitry Gozman 2021-04-05 13:23:49 -07:00 committed by GitHub
parent af48a8a1f1
commit e3cf675624
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
53 changed files with 33 additions and 6648 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,7 +1,7 @@
/node_modules/
/test-results/
/test/coverage-report
/test/test-user-data-dir*
/tests/coverage-report
.local-browsers/
/.dev_profile*
.DS_Store

6
package-lock.json generated
View File

@ -8131,9 +8131,9 @@
}
},
"folio": {
"version": "0.3.17",
"resolved": "https://registry.npmjs.org/folio/-/folio-0.3.17.tgz",
"integrity": "sha512-aYbhifQ/A0E6ZwEUdBGU900+aW2R243pxkWF0GhYceQxFTbkoCLIEHegTOpo4VtNsNHyM3sI/Xz3DkmITiwRcg==",
"version": "0.3.20-alpha",
"resolved": "https://registry.npmjs.org/folio/-/folio-0.3.20-alpha.tgz",
"integrity": "sha512-uJWYgfLa1l91NSbBV2pxKR7g91/Ti7cLuvfVI6M9uI5H7JaDhEEgPsLM9RG7rwoJ7AtIOFRJSbdMIwyRwvzWxw==",
"dev": true,
"requires": {
"@babel/code-frame": "^7.10.4",

19
package.json
View File

@ -9,27 +9,26 @@
"node": ">=10.17.0"
},
"scripts": {
"ctest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts chromium",
"ftest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts firefox",
"wtest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts webkit",
"atest": "npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts",
"test": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts",
"ctest": "folio --config=tests/config/default.config.ts chromium",
"ftest": "folio --config=tests/config/default.config.ts firefox",
"wtest": "folio --config=tests/config/default.config.ts webkit",
"atest": "folio --config=tests/config/android.config.ts",
"test": "folio --config=tests/config/default.config.ts",
"eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
"tsc": "tsc -p .",
"tsc-installer": "tsc -p ./src/install/tsconfig.json",
"doc": "node utils/doclint/cli.js",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && node utils/generate_channels.js && node utils/generate_types/ --check-clean && npm run test-types && folio utils/doclint/test/",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && node utils/generate_channels.js && node utils/generate_types/ --check-clean && npm run test-types",
"clean": "rimraf lib",
"prepare": "node install-from-github.js",
"build": "node utils/build/build.js",
"watch": "node utils/build/build.js --watch",
"test-types": "node utils/generate_types/ && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && tsc -p ./test/",
"test-types": "node utils/generate_types/ && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && tsc -p ./tests/",
"roll-browser": "node utils/roll_browser.js",
"check-deps": "node utils/check_deps.js",
"build-android-driver": "./utils/build_android_driver.sh",
"storybook": "start-storybook -p 6006 -s public",
"build-storybook": "build-storybook -s public",
"build-folio": "tsc -p ./tests/folio"
"build-storybook": "build-storybook -s public"
},
"author": {
"name": "Microsoft Corporation"
@ -80,7 +79,7 @@
"eslint-plugin-notice": "^0.9.10",
"eslint-plugin-react-hooks": "^4.2.0",
"file-loader": "^6.1.0",
"folio": "=0.3.17",
"folio": "=0.3.20-alpha",
"formidable": "^1.2.2",
"html-webpack-plugin": "^4.4.1",
"ncp": "^2.0.0",

197
test/fixtures.ts
View File

@ -1,197 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import assert from 'assert';
import childProcess from 'child_process';
import fs from 'fs';
import path from 'path';
import util from 'util';
import os from 'os';
import type { AndroidDevice, Browser, BrowserContext, BrowserType, Page } from '../index';
import { installCoverageHooks } from './coverage';
import { folio as httpFolio } from './http.fixtures';
import { folio as playwrightFolio } from './playwright.fixtures';
import { PlaywrightClient } from '../lib/remote/playwrightClient';
import { start } from '../lib/outofprocess';
import { removeFolders } from '../lib/utils/utils';
export { expect, config } from 'folio';
const mkdtempAsync = util.promisify(fs.mkdtemp);
const getExecutablePath = browserName => {
if (browserName === 'chromium' && process.env.CRPATH)
return process.env.CRPATH;
if (browserName === 'firefox' && process.env.FFPATH)
return process.env.FFPATH;
if (browserName === 'webkit' && process.env.WKPATH)
return process.env.WKPATH;
};
type ModeParameters = {
mode: 'default' | 'driver' | 'service';
};
type WorkerFixtures = {
toImpl: (rpcObject: any) => any;
androidDevice: AndroidDevice;
androidDeviceBrowser: BrowserContext;
};
type TestFixtures = {
createUserDataDir: () => Promise<string>;
launchPersistent: (options?: Parameters<BrowserType<Browser>['launchPersistentContext']>[1]) => Promise<{ context: BrowserContext, page: Page }>;
};
const fixtures = playwrightFolio.union(httpFolio).extend<TestFixtures, WorkerFixtures, ModeParameters>();
fixtures.mode.initParameter('Testing mode', process.env.PWMODE as any || 'default');
fixtures.createUserDataDir.init(async ({ }, run) => {
const dirs: string[] = [];
async function createUserDataDir() {
// We do not put user data dir in testOutputPath,
// because we do not want to upload them as test result artifacts.
//
// Additionally, it is impossible to upload user data dir after test run:
// - Firefox removes lock file later, presumably from another watchdog process?
// - WebKit has circular symlinks that makes CI go crazy.
const dir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
dirs.push(dir);
return dir;
}
await run(createUserDataDir);
await removeFolders(dirs);
});
fixtures.launchPersistent.init(async ({ createUserDataDir, browserOptions, browserType }, run) => {
let context;
async function launchPersistent(options) {
if (context)
throw new Error('can only launch one persitent context');
const userDataDir = await createUserDataDir();
context = await browserType.launchPersistentContext(userDataDir, { ...browserOptions, ...options });
const page = context.pages()[0];
return { context, page };
}
await run(launchPersistent);
if (context)
await context.close();
});
fixtures.browserOptions.override(async ({ browserName, headful, slowMo, browserChannel }, run) => {
const executablePath = getExecutablePath(browserName);
if (executablePath)
console.error(`Using executable at ${executablePath}`);
await run({
channel: browserChannel as any,
executablePath,
handleSIGINT: false,
slowMo,
headless: !headful,
});
});
fixtures.playwright.override(async ({ browserName, testWorkerIndex, platform, mode }, run) => {
assert(platform); // Depend on platform to generate all tests.
const { coverage, uninstall } = installCoverageHooks(browserName);
require('../lib/utils/utils').setUnderTest();
if (mode === 'driver') {
const playwrightObject = await start();
await run(playwrightObject);
await playwrightObject.stop();
await teardownCoverage();
} else if (mode === 'service') {
const port = 9407 + testWorkerIndex * 2;
const spawnedProcess = childProcess.fork(path.join(__dirname, '..', 'lib', 'service.js'), [String(port)], {
stdio: 'pipe'
});
spawnedProcess.stderr.pipe(process.stderr);
await new Promise<void>(f => {
spawnedProcess.stdout.on('data', data => {
if (data.toString().includes('Listening on'))
f();
});
});
spawnedProcess.unref();
const onExit = (exitCode, signal) => {
throw new Error(`Server closed with exitCode=${exitCode} signal=${signal}`);
};
spawnedProcess.on('exit', onExit);
const client = await PlaywrightClient.connect(`ws://localhost:${port}/ws`);
await run(client.playwright());
await client.close();
spawnedProcess.removeListener('exit', onExit);
const processExited = new Promise(f => spawnedProcess.on('exit', f));
spawnedProcess.kill();
await processExited;
await teardownCoverage();
} else {
const playwright = require('../index');
await run(playwright);
await teardownCoverage();
}
async function teardownCoverage() {
uninstall();
const coveragePath = path.join(__dirname, 'coverage-report', testWorkerIndex + '.json');
const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key));
await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
}
});
fixtures.toImpl.init(async ({ playwright }, run) => {
await run((playwright as any)._toImpl);
}, { scope: 'worker' });
fixtures.testParametersPathSegment.override(async ({ browserName }, run) => {
await run(browserName);
});
fixtures.androidDevice.init(async ({ playwright }, runTest) => {
const [device] = await playwright._android.devices();
await device.shell('am force-stop org.chromium.webview_shell');
await device.shell('am force-stop com.android.chrome');
device.setDefaultTimeout(120000);
await runTest(device);
await device.close();
}, { scope: 'worker' });
fixtures.androidDeviceBrowser.init(async ({ androidDevice }, runTest) => {
await runTest(await androidDevice.launchBrowser());
}, { scope: 'worker' });
if (process.env.PW_ANDROID_TESTS) {
fixtures.page.override(async ({ androidDeviceBrowser }, run) => {
for (const page of androidDeviceBrowser.pages())
await page.close();
run(await androidDeviceBrowser.newPage());
});
}
export const folio = fixtures.build();
folio.generateParametrizedTests(
'platform',
process.env.PWTESTREPORT ? ['win32', 'darwin', 'linux'] : [process.platform as ('win32' | 'linux' | 'darwin')]);
export const it = folio.it;
export const fit = folio.fit;
export const test = folio.test;
export const xit = folio.xit;
export const describe = folio.describe;
export const beforeEach = folio.beforeEach;
export const afterEach = folio.afterEach;
export const beforeAll = folio.beforeAll;
export const afterAll = folio.afterAll;

102
test/http.fixtures.ts
View File

@ -1,102 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { folio as base } from 'folio';
import path from 'path';
import socks from 'socksv5';
import { Server as WebSocketServer } from 'ws';
import { TestServer } from '../utils/testserver';
type HttpWorkerFixtures = {
asset: (path: string) => string;
httpService: { server: TestServer, httpsServer: TestServer };
socksPort: number,
};
type HttpTestFixtures = {
server: TestServer;
httpsServer: TestServer;
webSocketServer: WebSocketServer;
};
const fixtures = base.extend<HttpTestFixtures, HttpWorkerFixtures>();
fixtures.httpService.init(async ({ testWorkerIndex }, test) => {
const assetsPath = path.join(__dirname, 'assets');
const cachedPath = path.join(__dirname, 'assets', 'cached');
const port = 8907 + testWorkerIndex * 3;
const server = await TestServer.create(assetsPath, port);
server.enableHTTPCache(cachedPath);
const httpsPort = port + 1;
const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort);
httpsServer.enableHTTPCache(cachedPath);
await test({ server, httpsServer });
await Promise.all([
server.stop(),
httpsServer.stop(),
]);
}, { scope: 'worker' });
fixtures.asset.init(async ({ }, test) => {
await test(p => path.join(__dirname, `assets`, p));
}, { scope: 'worker' });
fixtures.server.init(async ({ httpService }, test) => {
httpService.server.reset();
await test(httpService.server);
});
fixtures.httpsServer.init(async ({ httpService }, test) => {
httpService.httpsServer.reset();
await test(httpService.httpsServer);
});
fixtures.webSocketServer.init(async ({ testWorkerIndex }, run) => {
const webSocketServer = new WebSocketServer({
port: 8907 + testWorkerIndex * 3 + 2,
});
await run(webSocketServer);
await new Promise(x => webSocketServer.close(x));
});
fixtures.socksPort.init(async ({ testWorkerIndex }, run) => {
const server = socks.createServer((info, accept, deny) => {
let socket;
if ((socket = accept(true))) {
// Catch and ignore ECONNRESET errors.
socket.on('error', () => {});
const body = '<html><title>Served by the SOCKS proxy</title></html>';
socket.end([
'HTTP/1.1 200 OK',
'Connection: close',
'Content-Type: text/html',
'Content-Length: ' + Buffer.byteLength(body),
'',
body
].join('\r\n'));
}
});
const socksPort = 9107 + testWorkerIndex * 2;
server.listen(socksPort, 'localhost');
server.useAuth(socks.auth.None());
await run(socksPort);
server.close();
}, { scope: 'worker' });
export const folio = fixtures.build();

209
test/playwright.fixtures.ts
View File

@ -1,209 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* =============================================================================
* DO NOT EDIT THIS FILE
* EDIT THE ONE IN @playwright/test and copy it over here.
* =============================================================================
*/
import { config, folio as baseFolio } from 'folio';
import type { Browser, BrowserContext, BrowserContextOptions, BrowserType, LaunchOptions, Page } from '../index';
export { expect, config } from 'folio';
// Test timeout for e2e tests is 30 seconds.
config.timeout = 30000;
// Parameters ------------------------------------------------------------------
// ... these can be used to run tests in different modes.
type PlaywrightParameters = {
// Browser type name.
browserName: 'chromium' | 'firefox' | 'webkit';
// Browser release channel, if applicable.
browserChannel: string | undefined;
// Whether to run tests headless or headful.
headful: boolean;
// Operating system.
platform: 'win32' | 'linux' | 'darwin';
// Generate screenshot on failure.
screenshotOnFailure: boolean;
// Slows down Playwright operations by the specified amount of milliseconds.
slowMo: number;
// Whether to record videos for all tests.
video: boolean;
};
// Worker fixture declarations -------------------------------------------------
// ... these live as long as the worker process.
type PlaywrightWorkerFixtures = {
// Playwright library.
playwright: typeof import('../index');
// Browser type (Chromium / WebKit / Firefox)
browserType: BrowserType<Browser>;
// Default browserType.launch() options.
browserOptions: LaunchOptions;
// Browser instance, shared for the worker.
browser: Browser;
// True iff browserName is Chromium
isChromium: boolean;
// True iff browserName is Firefox
isFirefox: boolean;
// True iff browserName is WebKit
isWebKit: boolean;
// True iff running on Windows.
isWindows: boolean;
// True iff running on Mac.
isMac: boolean;
// True iff running on Linux.
isLinux: boolean;
};
// Test fixture definitions, those are created for each test ------------------
type PlaywrightTestFixtures = {
// Default browser.newContext() options.
contextOptions: BrowserContextOptions;
// Factory for creating a context with given additional options.
contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;
// Context instance for test.
context: BrowserContext;
// Page instance for test.
page: Page;
};
const fixtures = baseFolio.extend<PlaywrightTestFixtures, PlaywrightWorkerFixtures, PlaywrightParameters>();
fixtures.browserName.initParameter('Browser type name', (process.env.BROWSER || 'chromium') as 'chromium' | 'firefox' | 'webkit');
fixtures.headful.initParameter('Whether to run tests headless or headful', process.env.HEADFUL ? true : false);
fixtures.platform.initParameter('Operating system', process.platform as ('win32' | 'linux' | 'darwin'));
fixtures.screenshotOnFailure.initParameter('Generate screenshot on failure', false);
fixtures.slowMo.initParameter('Slows down Playwright operations by the specified amount of milliseconds', 0);
fixtures.video.initParameter('Record videos while running tests', false);
fixtures.browserChannel.initParameter('Browser release channel', process.env.PW_CHROMIUM_CHANNEL);
fixtures.browserOptions.init(async ({ headful, slowMo, browserChannel }, run) => {
await run({
handleSIGINT: false,
slowMo,
headless: !headful,
channel: browserChannel as any,
});
}, { scope: 'worker' });
fixtures.playwright.init(async ({ }, run) => {
const playwright = require('playwright');
await run(playwright);
}, { scope: 'worker' });
fixtures.browserType.init(async ({ playwright, browserName }, run) => {
const browserType = (playwright as any)[browserName];
await run(browserType);
}, { scope: 'worker' });
fixtures.browser.init(async ({ browserType, browserOptions }, run) => {
const browser = await browserType.launch(browserOptions);
await run(browser);
await browser.close();
}, { scope: 'worker' });
fixtures.isChromium.init(async ({ browserName }, run) => {
await run(browserName === 'chromium');
}, { scope: 'worker' });
fixtures.isFirefox.init(async ({ browserName }, run) => {
await run(browserName === 'firefox');
}, { scope: 'worker' });
fixtures.isWebKit.init(async ({ browserName }, run) => {
await run(browserName === 'webkit');
}, { scope: 'worker' });
fixtures.isWindows.init(async ({ platform }, run) => {
await run(platform === 'win32');
}, { scope: 'worker' });
fixtures.isMac.init(async ({ platform }, run) => {
await run(platform === 'darwin');
}, { scope: 'worker' });
fixtures.isLinux.init(async ({ platform }, run) => {
await run(platform === 'linux');
}, { scope: 'worker' });
fixtures.contextOptions.init(async ({ video, testInfo }, run) => {
await run({
recordVideo: video ? { dir: testInfo.outputPath('') } : undefined,
_traceDir: process.env.PWTRACE ? testInfo.outputPath('') : undefined,
} as any);
});
fixtures.contextFactory.init(async ({ browser, contextOptions, testInfo, screenshotOnFailure }, run) => {
const contexts: BrowserContext[] = [];
async function contextFactory(options: BrowserContextOptions = {}) {
const context = await browser.newContext({ ...contextOptions, ...options });
contexts.push(context);
return context;
}
await run(contextFactory);
if (screenshotOnFailure && (testInfo.status !== testInfo.expectedStatus)) {
let ordinal = 0;
for (const context of contexts) {
for (const page of context.pages())
await page.screenshot({ timeout: 5000, path: testInfo.outputPath(`test-failed-${++ordinal}.png`) });
}
}
for (const context of contexts)
await context.close();
});
fixtures.context.init(async ({ contextFactory }, run) => {
const context = await contextFactory();
await run(context);
// Context factory is taking care of closing the context,
// so that it could capture a screenshot on failure.
});
fixtures.page.init(async ({ context }, run) => {
// Always create page off context so that they matched.
await run(await context.newPage());
// Context fixture is taking care of closing the page.
});
fixtures.testParametersPathSegment.override(async ({ browserName, platform }, run) => {
await run(browserName + '-' + platform);
});
export const folio = fixtures.build();
export const it = folio.it;
export const fit = folio.fit;
export const xit = folio.xit;
export const test = folio.test;
export const describe = folio.describe;
export const beforeEach = folio.beforeEach;
export const afterEach = folio.afterEach;
export const beforeAll = folio.beforeAll;
export const afterAll = folio.afterAll;
// If browser is not specified, we are running tests against all three browsers.
folio.generateParametrizedTests(
'browserName',
process.env.BROWSER ? [process.env.BROWSER] as any : ['chromium', 'webkit', 'firefox']);

2
test/utils.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import { expect } from './fixtures';
import { expect } from 'folio';
import type { Frame, Page } from '../index';
export async function attachFrame(page: Page, frameId: string, url: string): Promise<Frame> {

2
tests/config/android.config.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import { setConfig, Config } from '../folio/out';
import { setConfig, Config } from 'folio';
import * as path from 'path';
import { test as pageTest } from './pageTest';
import { test as androidTest } from './androidTest';

2
tests/config/androidEnv.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Env, WorkerInfo, TestInfo } from '../folio/out';
import type { Env, WorkerInfo, TestInfo } from 'folio';
import type { AndroidDevice, BrowserContext } from '../../index';
import * as os from 'os';
import { AndroidTestArgs } from './androidTest';

4
tests/config/androidTest.ts
View File

@ -14,11 +14,11 @@
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import { newTestType } from 'folio';
import type { AndroidDevice } from '../../index';
import type { CommonTestArgs } from './pageTest';
import type { ServerTestArgs } from './serverTest';
export { expect } from '../folio/out';
export { expect } from 'folio';
export type AndroidTestArgs = CommonTestArgs & {
androidDevice: AndroidDevice;

2
tests/config/browserEnv.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Env, WorkerInfo, TestInfo } from '../folio/out';
import type { Env, WorkerInfo, TestInfo } from 'folio';
import type { Browser, BrowserContext, BrowserContextOptions, BrowserType, LaunchOptions } from '../../index';
import { installCoverageHooks } from '../../test/coverage';
import { start } from '../../lib/outofprocess';

4
tests/config/browserTest.ts
View File

@ -14,11 +14,11 @@
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import { newTestType } from 'folio';
import type { Browser, BrowserContextOptions, BrowserContext, Page } from '../../index';
import type { PlaywrightTestArgs } from './playwrightTest';
import type { ServerTestArgs } from './serverTest';
export { expect } from '../folio/out';
export { expect } from 'folio';
export type BrowserTestArgs = PlaywrightTestArgs & {
browser: Browser;

2
tests/config/cliEnv.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Env, TestInfo, WorkerInfo } from '../folio/out';
import type { Env, TestInfo, WorkerInfo } from 'folio';
import { PageEnv } from './browserEnv';
import { CLIMock, CLITestArgs, Recorder } from './cliTest';
import * as http from 'http';

4
tests/config/cliTest.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import { newTestType } from 'folio';
import type { Page, BrowserContext } from '../../index';
import type { ServerTestArgs } from './serverTest';
import type { BrowserTestArgs } from './browserTest';
@ -22,7 +22,7 @@ import * as http from 'http';
import * as path from 'path';
import type { Source } from '../../src/server/supplements/recorder/recorderTypes';
import { ChildProcess, spawn } from 'child_process';
export { expect } from '../folio/out';
export { expect } from 'folio';
interface CLIHTTPServer {
setHandler: (handler: http.RequestListener) => void

2
tests/config/default.config.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import { setConfig, Config } from '../folio/out';
import { setConfig, Config } from 'folio';
import * as path from 'path';
import { test as playwrightTest, slowTest as playwrightSlowTest } from './playwrightTest';
import { test as browserTest, contextTest, proxyTest, slowTest as browserSlowTest } from './browserTest';

2
tests/config/electronEnv.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Env, TestInfo } from '../folio/out';
import type { Env, TestInfo } from 'folio';
import { PlaywrightEnv } from './browserEnv';
import * as path from 'path';
import { ElectronTestArgs } from './electronTest';

4
tests/config/electronTest.ts
View File

@ -14,11 +14,11 @@
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import { newTestType } from 'folio';
import { ElectronApplication, Page } from '../../index';
import type { CommonTestArgs } from './pageTest';
import type { ServerTestArgs } from './serverTest';
export { expect } from '../folio/out';
export { expect } from 'folio';
export type ElectronTestArgs = CommonTestArgs & {
electronApp: ElectronApplication;

4
tests/config/pageTest.ts
View File

@ -14,10 +14,10 @@
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import { newTestType } from 'folio';
import type { Page } from '../../index';
import type { ServerTestArgs } from './serverTest';
export { expect } from '../folio/out';
export { expect } from 'folio';
export type CommonTestArgs = {
mode: 'default' | 'driver' | 'service';

4
tests/config/playwrightTest.ts
View File

@ -14,12 +14,12 @@
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import { newTestType } from 'folio';
import type { Browser, BrowserType, LaunchOptions, BrowserContext, Page } from '../../index';
import { CommonTestArgs } from './pageTest';
import type { ServerTestArgs } from './serverTest';
import { RemoteServer, RemoteServerOptions } from './remoteServer';
export { expect } from '../folio/out';
export { expect } from 'folio';
export type PlaywrightTestArgs = CommonTestArgs & {
browserType: BrowserType<Browser>;

2
tests/config/serverEnv.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { WorkerInfo, TestInfo, Env } from '../folio/out';
import type { WorkerInfo, TestInfo, Env } from 'folio';
import { TestServer } from '../../utils/testserver';
import * as path from 'path';
import socks from 'socksv5';

1
tests/folio/.gitignore vendored
View File

@ -1 +0,0 @@
out/

19
tests/folio/cli.js
View File

@ -1,19 +0,0 @@
#!/usr/bin/env node
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('./out/cli');

278
tests/folio/src/cli.ts
View File

@ -1,278 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { default as ignore } from 'fstream-ignore';
import * as commander from 'commander';
import * as fs from 'fs';
import * as path from 'path';
import EmptyReporter from './reporters/empty';
import DotReporter from './reporters/dot';
import JSONReporter from './reporters/json';
import JUnitReporter from './reporters/junit';
import LineReporter from './reporters/line';
import ListReporter from './reporters/list';
import { Multiplexer } from './reporters/multiplexer';
import { Runner } from './runner';
import { Config, FullConfig, Reporter } from './types';
import { Loader } from './loader';
import { createMatcher } from './util';
export const reporters: { [name: string]: new () => Reporter } = {
'dot': DotReporter,
'json': JSONReporter,
'junit': JUnitReporter,
'line': LineReporter,
'list': ListReporter,
'null': EmptyReporter,
};
const availableReporters = Object.keys(reporters).map(r => `"${r}"`).join();
const defaultConfig: FullConfig = {
forbidOnly: false,
globalTimeout: 0,
grep: /.*/,
maxFailures: 0,
outputDir: path.resolve(process.cwd(), 'test-results'),
quiet: false,
repeatEach: 1,
retries: 0,
shard: null,
snapshotDir: '__snapshots__',
testDir: path.resolve(process.cwd()),
testIgnore: 'node_modules/**',
testMatch: '**/?(*.)+(spec|test).[jt]s',
timeout: 10000,
updateSnapshots: false,
workers: Math.ceil(require('os').cpus().length / 2),
};
const loadProgram = new commander.Command();
loadProgram.helpOption(false);
addRunnerOptions(loadProgram);
loadProgram.action(async command => {
try {
await runTests(command);
} catch (e) {
console.log(e);
process.exit(1);
}
});
loadProgram.parse(process.argv);
async function runTests(command: any) {
if (command.help === undefined) {
console.log(loadProgram.helpInformation());
process.exit(0);
}
const reporterList: string[] = command.reporter.split(',');
const reporterObjects: Reporter[] = reporterList.map(c => {
if (reporters[c])
return new reporters[c]();
try {
const p = path.resolve(process.cwd(), c);
return new (require(p).default)();
} catch (e) {
console.error('Invalid reporter ' + c, e);
process.exit(1);
}
});
const loader = new Loader();
loader.addConfig(defaultConfig);
function loadConfig(configName: string) {
const configFile = path.resolve(process.cwd(), configName);
if (fs.existsSync(configFile)) {
loader.loadConfigFile(configFile);
return true;
}
return false;
}
if (command.config) {
if (!loadConfig(command.config))
throw new Error(`${command.config} does not exist`);
} else if (!loadConfig('folio.config.ts') && !loadConfig('folio.config.js')) {
throw new Error(`Configuration file not found. Either pass --config, or create folio.config.(js|ts) file`);
}
loader.addConfig(configFromCommand(command));
loader.addConfig({ testMatch: normalizeFilePatterns(loader.config().testMatch) });
loader.addConfig({ testIgnore: normalizeFilePatterns(loader.config().testIgnore) });
const testDir = loader.config().testDir;
if (!fs.existsSync(testDir))
throw new Error(`${testDir} does not exist`);
if (!fs.statSync(testDir).isDirectory())
throw new Error(`${testDir} is not a directory`);
const allAliases = new Set(loader.runLists().map(s => s.alias));
const runListFilter: string[] = [];
const testFileFilter: string[] = [];
for (const arg of command.args) {
if (allAliases.has(arg))
runListFilter.push(arg);
else
testFileFilter.push(arg);
}
const allFiles = await collectFiles(testDir);
const testFiles = filterFiles(testDir, allFiles, testFileFilter, createMatcher(loader.config().testMatch), createMatcher(loader.config().testIgnore));
for (const file of testFiles)
loader.loadTestFile(file);
const reporter = new Multiplexer(reporterObjects);
const runner = new Runner(loader, reporter, runListFilter.length ? runListFilter : undefined);
if (command.list) {
runner.list();
return;
}
const result = await runner.run();
if (result === 'sigint')
process.exit(130);
if (result === 'forbid-only') {
console.error('=====================================');
console.error(' --forbid-only found a focused test.');
console.error('=====================================');
process.exit(1);
}
if (result === 'no-tests') {
console.error('=================');
console.error(' no tests found.');
console.error('=================');
process.exit(1);
}
process.exit(result === 'failed' ? 1 : 0);
}
async function collectFiles(testDir: string): Promise<string[]> {
const entries: any[] = [];
let callback = () => {};
const promise = new Promise<void>(f => callback = f);
ignore({ path: testDir, ignoreFiles: ['.gitignore'] })
.on('child', (entry: any) => entries.push(entry))
.on('end', callback);
await promise;
return entries.filter(e => e.type === 'File').sort((a, b) => {
if (a.depth !== b.depth && (a.dirname.startsWith(b.dirname) || b.dirname.startsWith(a.dirname)))
return a.depth - b.depth;
return a.path > b.path ? 1 : (a.path < b.path ? -1 : 0);
}).map(e => e.path);
}
function filterFiles(base: string, files: string[], filters: string[], filesMatch: (value: string) => boolean, filesIgnore: (value: string) => boolean): string[] {
return files.filter(file => {
file = path.relative(base, file);
if (filesIgnore(file))
return false;
if (!filesMatch(file))
return false;
if (filters.length && !filters.find(filter => file.includes(filter)))
return false;
return true;
});
}
function addRunnerOptions(program: commander.Command) {
program = program
.version('Version alpha')
.option('-c, --config <file>', `Configuration file (default: "folio.config.ts" or "folio.config.js")`)
.option('--forbid-only', `Fail if exclusive test(s) encountered (default: ${defaultConfig.forbidOnly})`)
.option('-g, --grep <grep>', `Only run tests matching this string or regexp (default: "${defaultConfig.grep}")`)
.option('--global-timeout <timeout>', `Specify maximum time this test suite can run in milliseconds (default: 0 for unlimited)`)
.option('-h, --help', `Display help`)
.option('-j, --workers <workers>', `Number of concurrent workers, use 1 to run in single worker (default: number of CPU cores / 2)`)
.option('--list', `Only collect all the test and report them`)
.option('--max-failures <N>', `Stop after the first N failures (default: ${defaultConfig.maxFailures})`)
.option('--output <dir>', `Folder for output artifacts (default: "test-results")`)
.option('--quiet', `Suppress stdio`)
.option('--repeat-each <repeat-each>', `Specify how many times to run the tests (default: ${defaultConfig.repeatEach})`)
.option('--reporter <reporter>', `Specify reporter to use, comma-separated, can be ${availableReporters}`, process.env.CI ? 'dot' : 'line')
.option('--retries <retries>', `Specify retry count (default: ${defaultConfig.retries})`)
.option('--shard <shard>', `Shard tests and execute only selected shard, specify in the form "current/all", 1-based, for example "3/5"`)
.option('--snapshot-dir <dir>', `Snapshot directory, relative to tests directory (default: "${defaultConfig.snapshotDir}"`)
.option('--test-dir <dir>', `Directory containing test files (default: current directory)`)
.option('--test-ignore <pattern>', `Pattern used to ignore test files (default: "${defaultConfig.testIgnore}")`)
.option('--test-match <pattern>', `Pattern used to find test files (default: "${defaultConfig.testMatch}")`)
.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds (default: ${defaultConfig.timeout})`)
.option('-u, --update-snapshots', `Whether to update snapshots with actual results (default: ${defaultConfig.updateSnapshots})`)
.option('-x', `Stop after the first failure`);
}
function configFromCommand(command: any): Config {
const config: Config = {};
if (command.forbidOnly)
config.forbidOnly = true;
if (command.globalTimeout)
config.globalTimeout = parseInt(command.globalTimeout, 10);
if (command.grep)
config.grep = maybeRegExp(command.grep);
if (command.maxFailures || command.x)
config.maxFailures = command.x ? 1 : parseInt(command.maxFailures, 10);
if (command.output)
config.outputDir = path.resolve(process.cwd(), command.output);
if (command.quiet)
config.quiet = command.quiet;
if (command.repeatEach)
config.repeatEach = parseInt(command.repeatEach, 10);
if (command.retries)
config.retries = parseInt(command.retries, 10);
if (command.shard) {
const pair = command.shard.split('/').map((t: string) => parseInt(t, 10));
config.shard = { current: pair[0] - 1, total: pair[1] };
}
if (command.snapshotDir)
config.snapshotDir = command.snapshotDir;
if (command.testDir)
config.testDir = path.resolve(process.cwd(), command.testDir);
if (command.testMatch)
config.testMatch = maybeRegExp(command.testMatch);
if (command.testIgnore)
config.testIgnore = maybeRegExp(command.testIgnore);
if (command.timeout)
config.timeout = parseInt(command.timeout, 10);
if (command.updateSnapshots)
config.updateSnapshots = !!command.updateSnapshots;
if (command.workers)
config.workers = parseInt(command.workers, 10);
return config;
}
function normalizeFilePattern(pattern: string): string {
if (!pattern.includes('/') && !pattern.includes('\\'))
pattern = '**/' + pattern;
return pattern;
}
function normalizeFilePatterns(patterns: string | RegExp | (string | RegExp)[]) {
if (typeof patterns === 'string')
patterns = normalizeFilePattern(patterns);
else if (Array.isArray(patterns))
patterns = patterns.map(item => typeof item === 'string' ? normalizeFilePattern(item) : item);
return patterns;
}
function maybeRegExp(pattern: string): string | RegExp {
const match = pattern.match(/^\/(.*)\/([gi]*)$/);
if (match)
return new RegExp(match[1], match[2]);
return pattern;
}

392
tests/folio/src/dispatcher.ts
View File

@ -1,392 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import child_process from 'child_process';
import path from 'path';
import { EventEmitter } from 'events';
import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, TestStatus, WorkerInitParams } from './ipc';
import { TestResult, Reporter } from './types';
import { Suite, Test } from './test';
import { Loader } from './loader';
type DispatcherEntry = {
runPayload: RunPayload;
hash: string;
repeatEachIndex: number;
runListIndex: number;
};
export class Dispatcher {
private _workers = new Set<Worker>();
private _freeWorkers: Worker[] = [];
private _workerClaimers: (() => void)[] = [];
private _testById = new Map<string, { test: Test, result: TestResult }>();
private _queue: DispatcherEntry[] = [];
private _stopCallback: () => void;
readonly _loader: Loader;
private _suite: Suite;
private _reporter: Reporter;
private _hasWorkerErrors = false;
private _isStopped = false;
private _failureCount = 0;
private _didRunGlobalSetup = false;
_globalSetupResult: any = undefined;
constructor(loader: Loader, suite: Suite, reporter: Reporter) {
this._loader = loader;
this._reporter = reporter;
this._suite = suite;
for (const suite of this._suite.suites) {
for (const spec of suite._allSpecs()) {
for (const test of spec.tests)
this._testById.set(test._id, { test, result: test._appendTestResult() });
}
}
this._queue = this._filesSortedByWorkerHash();
// Shard tests.
let total = this._suite.totalTestCount();
let shardDetails = '';
const shard = this._loader.config().shard;
if (shard) {
const shardSize = Math.ceil(total / shard.total);
const from = shardSize * shard.current;
const to = shardSize * (shard.current + 1);
shardDetails = `, shard ${shard.current + 1} of ${shard.total}`;
let current = 0;
total = 0;
const filteredQueue: DispatcherEntry[] = [];
for (const entry of this._queue) {
if (current >= from && current < to) {
filteredQueue.push(entry);
total += entry.runPayload.entries.length;
}
current += entry.runPayload.entries.length;
}
this._queue = filteredQueue;
}
if (process.stdout.isTTY) {
const workers = new Set<string>();
suite.findSpec(test => {
for (const variant of test.tests)
workers.add(test.file + variant._workerHash);
});
console.log();
const jobs = Math.min(this._loader.config().workers, workers.size);
console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`);
}
}
_filesSortedByWorkerHash(): DispatcherEntry[] {
const entriesByWorkerHashAndFile = new Map<string, Map<string, DispatcherEntry>>();
for (const suite of this._suite.suites) {
for (const spec of suite._allSpecs()) {
for (const test of spec.tests) {
let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
if (!entriesByFile) {
entriesByFile = new Map();
entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
}
let entry = entriesByFile.get(spec.file);
if (!entry) {
entry = {
runPayload: {
entries: [],
file: spec.file,
},
repeatEachIndex: test._repeatEachIndex,
runListIndex: test._runListIndex,
hash: test._workerHash,
};
entriesByFile.set(spec.file, entry);
}
entry.runPayload.entries.push({
retry: this._testById.get(test._id).result.retry,
testId: test._id,
});
}
}
}
const result: DispatcherEntry[] = [];
for (const entriesByFile of entriesByWorkerHashAndFile.values()) {
for (const entry of entriesByFile.values())
result.push(entry);
}
result.sort((a, b) => a.hash < b.hash ? -1 : (a.hash === b.hash ? 0 : 1));
return result;
}
async run() {
if (this._loader.globalSetup) {
this._didRunGlobalSetup = true;
this._globalSetupResult = await this._loader.globalSetup();
}
// Loop in case job schedules more jobs
while (this._queue.length && !this._isStopped)
await this._dispatchQueue();
}
async _dispatchQueue() {
const jobs = [];
while (this._queue.length) {
if (this._isStopped)
break;
const entry = this._queue.shift();
const requiredHash = entry.hash;
let worker = await this._obtainWorker(entry);
while (!this._isStopped && worker.hash && worker.hash !== requiredHash) {
worker.stop();
worker = await this._obtainWorker(entry);
}
if (this._isStopped)
break;
jobs.push(this._runJob(worker, entry));
}
await Promise.all(jobs);
}
async _runJob(worker: Worker, entry: DispatcherEntry) {
worker.run(entry.runPayload);
let doneCallback;
const result = new Promise(f => doneCallback = f);
worker.once('done', (params: DonePayload) => {
// We won't file remaining if:
// - there are no remaining
// - we are here not because something failed
// - no unrecoverable worker error
if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
this._freeWorkers.push(worker);
this._notifyWorkerClaimer();
doneCallback();
return;
}
// When worker encounters error, we will stop it and create a new one.
worker.stop();
let remaining = params.remaining;
const failedTestIds = new Set<string>();
// In case of fatal error, report all remaining tests as failing with this error.
if (params.fatalError) {
for (const { testId } of remaining) {
const { test, result } = this._testById.get(testId);
this._reporter.onTestBegin(test);
result.error = params.fatalError;
this._reportTestEnd(test, result, 'failed');
failedTestIds.add(testId);
}
// Since we pretent that all remaining tests failed, there is nothing else to run,
// except for possible retries.
remaining = [];
}
if (params.failedTestId)
failedTestIds.add(params.failedTestId);
// Only retry expected failures, not passes and only if the test failed.
for (const testId of failedTestIds) {
const pair = this._testById.get(testId);
if (pair.test.expectedStatus === 'passed' && pair.test.results.length < this._loader.config().retries + 1) {
pair.result = pair.test._appendTestResult();
remaining.unshift({
retry: pair.result.retry,
testId: pair.test._id,
});
}
}
if (remaining.length)
this._queue.unshift({ ...entry, runPayload: { ...entry.runPayload, entries: remaining } });
// This job is over, we just scheduled another one.
doneCallback();
});
return result;
}
async _obtainWorker(entry: DispatcherEntry) {
const claimWorker = (): Promise<Worker> | null => {
// Use available worker.
if (this._freeWorkers.length)
return Promise.resolve(this._freeWorkers.pop());
// Create a new worker.
if (this._workers.size < this._loader.config().workers)
return this._createWorker(entry);
return null;
};
// Note: it is important to claim the worker synchronously,
// so that we won't miss a _notifyWorkerClaimer call while awaiting.
let worker = claimWorker();
if (!worker) {
// Wait for available or stopped worker.
await new Promise<void>(f => this._workerClaimers.push(f));
worker = claimWorker();
}
return worker;
}
async _notifyWorkerClaimer() {
if (this._isStopped || !this._workerClaimers.length)
return;
const callback = this._workerClaimers.shift();
callback();
}
_createWorker(entry: DispatcherEntry) {
const worker = new Worker(this);
worker.on('testBegin', (params: TestBeginPayload) => {
const { test, result: testRun } = this._testById.get(params.testId);
testRun.workerIndex = params.workerIndex;
this._reporter.onTestBegin(test);
});
worker.on('testEnd', (params: TestEndPayload) => {
const { test, result } = this._testById.get(params.testId);
result.data = params.data;
result.duration = params.duration;
result.error = params.error;
test.expectedStatus = params.expectedStatus;
test.annotations = params.annotations;
test.timeout = params.timeout;
if (params.expectedStatus === 'skipped')
test.skipped = true;
this._reportTestEnd(test, result, params.status);
});
worker.on('stdOut', (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
const pair = this._testById.get(params.testId);
if (pair)
pair.result.stdout.push(chunk);
this._reporter.onStdOut(chunk, pair ? pair.test : undefined);
});
worker.on('stdErr', (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
const pair = this._testById.get(params.testId);
if (pair)
pair.result.stderr.push(chunk);
this._reporter.onStdErr(chunk, pair ? pair.test : undefined);
});
worker.on('teardownError', ({error}) => {
this._hasWorkerErrors = true;
this._reporter.onError(error);
});
worker.on('exit', () => {
this._workers.delete(worker);
this._notifyWorkerClaimer();
if (this._stopCallback && !this._workers.size)
this._stopCallback();
});
this._workers.add(worker);
return worker.init(entry).then(() => worker);
}
async stop() {
this._isStopped = true;
if (this._workers.size) {
const result = new Promise<void>(f => this._stopCallback = f);
for (const worker of this._workers)
worker.stop();
await result;
}
if (this._didRunGlobalSetup && this._loader.globalTeardown)
await this._loader.globalTeardown(this._globalSetupResult);
}
private _reportTestEnd(test: Test, result: TestResult, status: TestStatus) {
if (this._isStopped)
return;
result.status = status;
if (result.status !== 'skipped' && result.status !== test.expectedStatus)
++this._failureCount;
const maxFailures = this._loader.config().maxFailures;
if (!maxFailures || this._failureCount <= maxFailures)
this._reporter.onTestEnd(test, result);
if (maxFailures && this._failureCount === maxFailures)
this._isStopped = true;
}
hasWorkerErrors(): boolean {
return this._hasWorkerErrors;
}
}
let lastWorkerIndex = 0;
class Worker extends EventEmitter {
process: child_process.ChildProcess;
runner: Dispatcher;
hash: string;
index: number;
stdout: any[];
stderr: any[];
constructor(runner: Dispatcher) {
super();
this.runner = runner;
this.index = lastWorkerIndex++;
this.process = child_process.fork(path.join(__dirname, 'worker.js'), {
detached: false,
env: {
FORCE_COLOR: process.stdout.isTTY ? '1' : '0',
DEBUG_COLORS: process.stdout.isTTY ? '1' : '0',
FOLIO_WORKER_INDEX: String(this.index),
...process.env
},
// Can't pipe since piping slows down termination for some reason.
stdio: ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc']
});
this.process.on('exit', () => this.emit('exit'));
this.process.on('error', e => {}); // do not yell at a send to dead process.
this.process.on('message', (message: any) => {
const { method, params } = message;
this.emit(method, params);
});
}
async init(entry: DispatcherEntry) {
this.hash = entry.hash;
const params: WorkerInitParams = {
workerIndex: this.index,
repeatEachIndex: entry.repeatEachIndex,
runListIndex: entry.runListIndex,
globalSetupResult: this.runner._globalSetupResult,
loader: this.runner._loader.serialize(),
};
this.process.send({ method: 'init', params });
await new Promise(f => this.process.once('message', f)); // Ready ack
}
run(runPayload: RunPayload) {
this.process.send({ method: 'run', params: runPayload });
}
stop() {
this.process.send({ method: 'stop' });
}
}
function chunkFromParams(params: TestOutputPayload): string | Buffer {
if (typeof params.text === 'string')
return params.text;
return Buffer.from(params.buffer, 'base64');
}

58
tests/folio/src/expect.ts
View File

@ -1,58 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Expect } from './expectType';
import expectLibrary from 'expect';
import { currentTestInfo } from './globals';
import { compare } from './golden';
export const expect: Expect = expectLibrary;
const snapshotOrdinalSymbol = Symbol('snapshotOrdinalSymbol');
function toMatchSnapshot(received: Buffer | string, nameOrOptions?: string | { name?: string, threshold?: number }, optOptions: { threshold?: number } = {}) {
let options: { name?: string, threshold?: number };
const testInfo = currentTestInfo();
if (typeof nameOrOptions === 'string')
options = { name: nameOrOptions, ...optOptions };
else
options = { ...nameOrOptions };
let name = options.name;
if (!name) {
const ordinal = (testInfo as any)[snapshotOrdinalSymbol] || 0;
(testInfo as any)[snapshotOrdinalSymbol] = ordinal + 1;
let extension: string;
if (typeof received === 'string')
extension = '.txt';
else if (received[0] === 0x89 && received[1] === 0x50 && received[2] === 0x4E && received[3] === 0x47)
extension = '.png';
else if (received[0] === 0xFF && received[1] === 0xD8 && received[2] === 0xFF)
extension = '.jpeg';
else
extension = '.dat';
name = 'snapshot' + (ordinal ? '_' + ordinal : '') + extension;
}
const { pass, message } = compare(received, name, testInfo.snapshotPath, testInfo.outputPath, testInfo.config.updateSnapshots, options);
return { pass, message: () => message };
}
expectLibrary.extend({ toMatchSnapshot });
// TEMPORARY HACK: two folios currently fight for their own toMatchSnapshot.
export function extendAgain() {
expectLibrary.extend({ toMatchSnapshot });
}

150
tests/folio/src/expectType.ts
View File

@ -1,150 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
*/
export declare type AsymmetricMatcher = Record<string, any>;
export declare type Expect = {
<T = unknown>(actual: T): Matchers<T>;
[id: string]: AsymmetricMatcher;
not: {
[id: string]: AsymmetricMatcher;
};
};
export interface Matchers<R> {
/**
* If you know how to test something, `.not` lets you test its opposite.
*/
not: Matchers<R>;
/**
* Use resolves to unwrap the value of a fulfilled promise so any other
* matcher can be chained. If the promise is rejected the assertion fails.
*/
resolves: Matchers<Promise<R>>;
/**
* Unwraps the reason of a rejected promise so any other matcher can be chained.
* If the promise is fulfilled the assertion fails.
*/
rejects: Matchers<Promise<R>>;
/**
* Checks that a value is what you expect. It uses `===` to check strict equality.
* Don't use `toBe` with floating-point numbers.
*/
toBe(expected: unknown): R;
/**
* Using exact equality with floating point numbers is a bad idea.
* Rounding means that intuitive things fail.
* The default for numDigits is 2.
*/
toBeCloseTo(expected: number, numDigits?: number): R;
/**
* Ensure that a variable is not undefined.
*/
toBeDefined(): R;
/**
* When you don't care what a value is, you just want to
* ensure a value is false in a boolean context.
*/
toBeFalsy(): R;
/**
* For comparing floating point numbers.
*/
toBeGreaterThan(expected: number | bigint): R;
/**
* For comparing floating point numbers.
*/
toBeGreaterThanOrEqual(expected: number | bigint): R;
/**
* Ensure that an object is an instance of a class.
* This matcher uses `instanceof` underneath.
*/
toBeInstanceOf(expected: Function): R;
/**
* For comparing floating point numbers.
*/
toBeLessThan(expected: number | bigint): R;
/**
* For comparing floating point numbers.
*/
toBeLessThanOrEqual(expected: number | bigint): R;
/**
* This is the same as `.toBe(null)` but the error messages are a bit nicer.
* So use `.toBeNull()` when you want to check that something is null.
*/
toBeNull(): R;
/**
* Use when you don't care what a value is, you just want to ensure a value
* is true in a boolean context. In JavaScript, there are six falsy values:
* `false`, `0`, `''`, `null`, `undefined`, and `NaN`. Everything else is truthy.
*/
toBeTruthy(): R;
/**
* Used to check that a variable is undefined.
*/
toBeUndefined(): R;
/**
* Used to check that a variable is NaN.
*/
toBeNaN(): R;
/**
* Used when you want to check that an item is in a list.
* For testing the items in the list, this uses `===`, a strict equality check.
*/
toContain(expected: unknown): R;
/**
* Used when you want to check that an item is in a list.
* For testing the items in the list, this matcher recursively checks the
* equality of all fields, rather than checking for object identity.
*/
toContainEqual(expected: unknown): R;
/**
* Used when you want to check that two objects have the same value.
* This matcher recursively checks the equality of all fields, rather than checking for object identity.
*/
toEqual(expected: unknown): R;
/**
* Use to check if property at provided reference keyPath exists for an object.
* For checking deeply nested properties in an object you may use dot notation or an array containing
* the keyPath for deep references.
*
* Optionally, you can provide a value to check if it's equal to the value present at keyPath
* on the target object. This matcher uses 'deep equality' (like `toEqual()`) and recursively checks
* the equality of all fields.
*
* @example
*
* expect(houseForSale).toHaveProperty('kitchen.area', 20);
*/
toHaveProperty(keyPath: string | Array<string>, value?: unknown): R;
/**
* Check that a string matches a regular expression.
*/
toMatch(expected: string | RegExp): R;
/**
* Used to check that a JavaScript object matches a subset of the properties of an object
*/
toMatchObject(expected: Record<string, unknown> | Array<unknown>): R;
/**
* Use to test that objects have the same types as well as structure.
*/
toStrictEqual(expected: unknown): R;
/**
* Match snapshot
*/
toMatchSnapshot(options?: {
name?: string,
threshold?: number
}): R;
/**
* Match snapshot
*/
toMatchSnapshot(name: string, options?: {
threshold?: number
}): R;
}
export {};

25
tests/folio/src/globals.ts
View File

@ -1,25 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TestInfo } from './types';
let currentTestInfoValue: TestInfo | null = null;
export function setCurrentTestInfo(testInfo: TestInfo | null) {
currentTestInfoValue = testInfo;
}
export function currentTestInfo(): TestInfo | null {
return currentTestInfoValue;
}

171
tests/folio/src/golden.ts
View File

@ -1,171 +0,0 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import fs from 'fs';
import path from 'path';
import jpeg from 'jpeg-js';
import pixelmatch from 'pixelmatch';
import { PNG } from 'pngjs';
import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
const extensionToMimeType: { [key: string]: string } = {
'dat': 'application/octet-string',
'jpeg': 'image/jpeg',
'jpg': 'image/jpeg',
'png': 'image/png',
'txt': 'text/plain',
};
const GoldenComparators: { [key: string]: any } = {
'application/octet-string': compareBuffers,
'image/png': compareImages,
'image/jpeg': compareImages,
'text/plain': compareText,
};
function compareBuffers(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string): { diff?: object; errorMessage?: string; } | null {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be Buffer.' };
if (Buffer.compare(actualBuffer, expectedBuffer))
return { errorMessage: 'Buffers differ' };
return null;
}
function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: object; errorMessage?: string; } | null {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be Buffer.' };
const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpeg.decode(actualBuffer);
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
if (expected.width !== actual.width || expected.height !== actual.height) {
return {
errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
};
}
const diff = new PNG({width: expected.width, height: expected.height});
const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options });
return count > 0 ? { diff: PNG.sync.write(diff) } : null;
}
function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: object; errorMessage?: string; diffExtension?: string; } | null {
if (typeof actual !== 'string')
return { errorMessage: 'Actual result should be string' };
const expected = expectedBuffer.toString('utf-8');
if (expected === actual)
return null;
const dmp = new diff_match_patch();
const d = dmp.diff_main(expected, actual);
dmp.diff_cleanupSemantic(d);
return {
errorMessage: diff_prettyTerminal(d)
};
}
export function compare(actual: Buffer | string, name: string, snapshotPath: (name: string) => string, outputPath: (name: string) => string, updateSnapshots: boolean, options?: { threshold?: number }): { pass: boolean; message?: string; } {
const snapshotFile = snapshotPath(name);
if (!fs.existsSync(snapshotFile)) {
fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
fs.writeFileSync(snapshotFile, actual);
return {
pass: false,
message: snapshotFile + ' is missing in golden results, writing actual.'
};
}
const expected = fs.readFileSync(snapshotFile);
const extension = path.extname(snapshotFile).substring(1);
const mimeType = extensionToMimeType[extension] || 'application/octet-string';
const comparator = GoldenComparators[mimeType];
if (!comparator) {
return {
pass: false,
message: 'Failed to find comparator with type ' + mimeType + ': ' + snapshotFile,
};
}
const result = comparator(actual, expected, mimeType, options);
if (!result)
return { pass: true };
if (updateSnapshots) {
fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
fs.writeFileSync(snapshotFile, actual);
console.log('Updating snapshot at ' + snapshotFile);
return {
pass: true,
message: snapshotFile + ' running with --p-update-snapshots, writing actual.'
};
}
const outputFile = outputPath(name);
const expectedPath = addSuffix(outputFile, '-expected');
const actualPath = addSuffix(outputFile, '-actual');
const diffPath = addSuffix(outputFile, '-diff');
fs.writeFileSync(expectedPath, expected);
fs.writeFileSync(actualPath, actual);
if (result.diff)
fs.writeFileSync(diffPath, result.diff);
const output = [
colors.red(`Snapshot comparison failed:`),
];
if (result.errorMessage) {
output.push('');
output.push(indent(result.errorMessage, ' '));
}
output.push('');
output.push(`Expected: ${colors.yellow(expectedPath)}`);
output.push(`Received: ${colors.yellow(actualPath)}`);
if (result.diff)
output.push(` Diff: ${colors.yellow(diffPath)}`);
return {
pass: false,
message: output.join('\n'),
};
}
function indent(lines: string, tab: string) {
return lines.replace(/^(?=.+$)/gm, tab);
}
function addSuffix(filePath: string, suffix: string, customExtension?: string): string {
const dirname = path.dirname(filePath);
const ext = path.extname(filePath);
const name = path.basename(filePath, ext);
return path.join(dirname, name + suffix + (customExtension || ext));
}
function diff_prettyTerminal(diffs) {
const html = [];
for (let x = 0; x < diffs.length; x++) {
const op = diffs[x][0]; // Operation (insert, delete, equal)
const data = diffs[x][1]; // Text of change.
const text = data;
switch (op) {
case DIFF_INSERT:
html[x] = colors.green(text);
break;
case DIFF_DELETE:
html[x] = colors.strikethrough(colors.red(text));
break;
case DIFF_EQUAL:
html[x] = text;
break;
}
}
return html.join('');
}

27
tests/folio/src/index.ts
View File

@ -1,27 +0,0 @@
/**
* Copyright 2019 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { TestType } from './types';
import { newTestTypeImpl } from './spec';
export * from './types';
export { expect } from './expect';
export { setConfig, globalSetup, globalTeardown } from './spec';
export function newTestType<TestArgs = {}, TestOptions = {}>(): TestType<TestArgs, TestOptions> {
return newTestTypeImpl();
}

66
tests/folio/src/ipc.ts
View File

@ -1,66 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Config, TestStatus, TestError } from './types';
export type { TestStatus } from './types';
export type WorkerInitParams = {
workerIndex: number;
repeatEachIndex: number;
runListIndex: number;
globalSetupResult: any;
loader: {
configs: (string | Config)[];
};
};
export type TestBeginPayload = {
testId: string;
workerIndex: number,
};
export type TestEndPayload = {
testId: string;
duration: number;
status: TestStatus;
error?: TestError;
data: any;
expectedStatus: TestStatus;
annotations: any[];
timeout: number;
};
export type TestEntry = {
testId: string;
retry: number;
};
export type RunPayload = {
file: string;
entries: TestEntry[];
};
export type DonePayload = {
failedTestId?: string;
fatalError?: any;
remaining: TestEntry[];
};
export type TestOutputPayload = {
testId?: string;
text?: string;
buffer?: string;
};

94
tests/folio/src/loader.ts
View File

@ -1,94 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { installTransform } from './transform';
import { Config, FullConfig } from './types';
import { prependErrorMessage } from './util';
import { configFile, setCurrentFile, RunListDescription } from './spec';
type SerializedLoaderData = {
configs: (string | Config)[];
};
export class Loader {
globalSetup?: () => any;
globalTeardown?: (globalSetupResult: any) => any;
private _mergedConfig: FullConfig;
private _layeredConfigs: { config: Config, source?: string }[] = [];
constructor() {
this._mergedConfig = {} as any;
}
deserialize(data: SerializedLoaderData) {
for (const config of data.configs) {
if (typeof config === 'string')
this.loadConfigFile(config);
else
this.addConfig(config);
}
}
loadConfigFile(file: string) {
const revertBabelRequire = installTransform();
try {
require(file);
this.addConfig(configFile.config || {});
this._layeredConfigs[this._layeredConfigs.length - 1].source = file;
this.globalSetup = configFile.globalSetup;
this.globalTeardown = configFile.globalTeardown;
} catch (e) {
// Drop the stack.
throw new Error(e.message);
} finally {
revertBabelRequire();
}
}
addConfig(config: Config) {
this._layeredConfigs.push({ config });
this._mergedConfig = { ...this._mergedConfig, ...config };
}
loadTestFile(file: string) {
const revertBabelRequire = installTransform();
setCurrentFile(file);
try {
require(file);
} catch (e) {
prependErrorMessage(e, `Error while reading ${file}:\n`);
throw e;
} finally {
setCurrentFile();
revertBabelRequire();
}
}
config(): FullConfig {
return this._mergedConfig;
}
runLists(): RunListDescription[] {
return configFile.runLists;
}
serialize(): SerializedLoaderData {
return {
configs: this._layeredConfigs.map(c => c.source || c.config),
};
}
}

246
tests/folio/src/reporters/base.ts
View File

@ -1,246 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { codeFrameColumns } from '@babel/code-frame';
import colors from 'colors/safe';
import fs from 'fs';
import milliseconds from 'ms';
import path from 'path';
import StackUtils from 'stack-utils';
import { TestStatus, Test, Suite, TestResult, TestError, Reporter } from '../types';
import { FullConfig } from '../types';
const stackUtils = new StackUtils();
export class BaseReporter implements Reporter {
duration = 0;
config: FullConfig;
suite: Suite;
timeout: number;
fileDurations = new Map<string, number>();
monotonicStartTime: number;
constructor() {
}
onBegin(config: FullConfig, suite: Suite) {
this.monotonicStartTime = monotonicTime();
this.config = config;
this.suite = suite;
}
onTestBegin(test: Test) {
}
onStdOut(chunk: string | Buffer) {
if (!this.config.quiet)
process.stdout.write(chunk);
}
onStdErr(chunk: string | Buffer) {
if (!this.config.quiet)
process.stderr.write(chunk);
}
onTestEnd(test: Test, result: TestResult) {
const spec = test.spec;
let duration = this.fileDurations.get(spec.file) || 0;
duration += result.duration;
this.fileDurations.set(spec.file, duration);
}
onError(error: TestError) {
console.log(formatError(error));
}
onTimeout(timeout: number) {
this.timeout = timeout;
}
onEnd() {
this.duration = monotonicTime() - this.monotonicStartTime;
}
private _printSlowTests() {
const fileDurations = [...this.fileDurations.entries()];
fileDurations.sort((a, b) => b[1] - a[1]);
let insertedGap = false;
for (let i = 0; i < 10 && i < fileDurations.length; ++i) {
const baseName = path.basename(fileDurations[i][0]);
const duration = fileDurations[i][1];
if (duration < 15000)
break;
if (!insertedGap) {
insertedGap = true;
console.log();
}
console.log(colors.yellow(' Slow test: ') + baseName + colors.yellow(` (${milliseconds(duration)})`));
}
console.log();
}
epilogue(full: boolean) {
let skipped = 0;
let expected = 0;
const unexpected: Test[] = [];
const flaky: Test[] = [];
this.suite.findTest(test => {
switch (test.status()) {
case 'skipped': ++skipped; break;
case 'expected': ++expected; break;
case 'unexpected': unexpected.push(test); break;
case 'flaky': flaky.push(test); break;
}
});
if (expected)
console.log(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
if (skipped)
console.log(colors.yellow(` ${skipped} skipped`));
if (unexpected.length) {
console.log(colors.red(` ${unexpected.length} failed`));
this._printTestHeaders(unexpected);
}
if (flaky.length) {
console.log(colors.red(` ${flaky.length} flaky`));
this._printTestHeaders(flaky);
}
if (this.timeout)
console.log(colors.red(` Timed out waiting ${this.timeout / 1000}s for the entire test run`));
if (full && unexpected.length) {
console.log('');
this._printFailures(unexpected);
}
this._printSlowTests();
}
private _printTestHeaders(tests: Test[]) {
tests.forEach(test => {
console.log(formatTestHeader(this.config, test, ' '));
});
}
private _printFailures(failures: Test[]) {
failures.forEach((test, index) => {
console.log(formatFailure(this.config, test, index + 1));
});
}
hasResultWithStatus(test: Test, status: TestStatus): boolean {
return !!test.results.find(r => r.status === status);
}
willRetry(test: Test, result: TestResult): boolean {
return result.status !== 'passed' && result.status !== test.expectedStatus && test.results.length <= this.config.retries;
}
}
export function formatFailure(config: FullConfig, test: Test, index?: number): string {
const tokens: string[] = [];
tokens.push(formatTestHeader(config, test, ' ', index));
for (const result of test.results) {
if (result.status === 'passed')
continue;
tokens.push(formatResult(test, result));
}
tokens.push('');
return tokens.join('\n');
}
function formatTestHeader(config: FullConfig, test: Test, indent: string, index?: number): string {
const tokens: string[] = [];
const spec = test.spec;
let relativePath = path.relative(config.testDir, spec.file) || path.basename(spec.file);
relativePath += ':' + spec.line + ':' + spec.column;
const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
const runListName = test.alias ? `[${test.alias}] ` : '';
const header = `${indent}${index ? index + ') ' : ''}${relativePath} › ${runListName}${spec.fullTitle()}${passedUnexpectedlySuffix}`;
tokens.push(colors.red(pad(header, '=')));
return tokens.join('\n');
}
function formatResult(test: Test, result: TestResult): string {
const tokens: string[] = [];
if (result.retry)
tokens.push(colors.gray(pad(`\n Retry #${result.retry}`, '-')));
if (result.status === 'timedOut') {
tokens.push('');
tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), ' '));
} else {
tokens.push(indent(formatError(result.error, test.spec.file), ' '));
}
return tokens.join('\n');
}
function formatError(error: TestError, file?: string) {
const stack = error.stack;
const tokens = [];
if (stack) {
tokens.push('');
const messageLocation = error.stack.indexOf(error.message);
const preamble = error.stack.substring(0, messageLocation + error.message.length);
tokens.push(preamble);
const position = file ? positionInFile(stack, file) : null;
if (position) {
const source = fs.readFileSync(file, 'utf8');
tokens.push('');
tokens.push(codeFrameColumns(source, {
start: position,
},
{ highlightCode: true}
));
}
tokens.push('');
tokens.push(colors.dim(stack.substring(preamble.length + 1)));
} else {
tokens.push('');
tokens.push(error.value);
}
return tokens.join('\n');
}
function pad(line: string, char: string): string {
return line + ' ' + colors.gray(char.repeat(Math.max(0, 100 - line.length - 1)));
}
function indent(lines: string, tab: string) {
return lines.replace(/^(?=.+$)/gm, tab);
}
function positionInFile(stack: string, file: string): { column: number; line: number; } {
// Stack will have /private/var/folders instead of /var/folders on Mac.
file = fs.realpathSync(file);
for (const line of stack.split('\n')) {
const parsed = stackUtils.parseLine(line);
if (!parsed)
continue;
if (path.resolve(process.cwd(), parsed.file) === file)
return {column: parsed.column, line: parsed.line};
}
return null;
}
function monotonicTime(): number {
const [seconds, nanoseconds] = process.hrtime();
return seconds * 1000 + (nanoseconds / 1000000 | 0);
}
const asciiRegex = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', 'g');
export function stripAscii(str: string): string {
return str.replace(asciiRegex, '');
}

57
tests/folio/src/reporters/dot.ts
View File

@ -1,57 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import { BaseReporter } from './base';
import { Test, TestResult } from '../types';
class DotReporter extends BaseReporter {
private _counter = 0;
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
if (++this._counter === 81) {
process.stdout.write('\n');
return;
}
if (result.status === 'skipped') {
process.stdout.write(colors.yellow('°'));
return;
}
if (this.willRetry(test, result)) {
process.stdout.write(colors.gray('×'));
return;
}
switch (test.status()) {
case 'expected': process.stdout.write(colors.green('·')); break;
case 'unexpected': process.stdout.write(colors.red(test.results[test.results.length - 1].status === 'timedOut' ? 'T' : 'F')); break;
case 'flaky': process.stdout.write(colors.yellow('±')); break;
}
}
onTimeout(timeout: number) {
super.onTimeout(timeout);
this.onEnd();
}
onEnd() {
super.onEnd();
process.stdout.write('\n');
this.epilogue(true);
}
}
export default DotReporter;

30
tests/folio/src/reporters/empty.ts
View File

@ -1,30 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FullConfig, TestResult, Test, Suite, TestError, Reporter } from '../types';
class EmptyReporter implements Reporter {
onBegin(config: FullConfig, suite: Suite) {}
onTestBegin(test: Test) {}
onStdOut(chunk: string | Buffer, test?: Test) {}
onStdErr(chunk: string | Buffer, test?: Test) {}
onTestEnd(test: Test, result: TestResult) {}
onTimeout(timeout: number) {}
onError(error: TestError) {}
onEnd() {}
}
export default EmptyReporter;

136
tests/folio/src/reporters/json.ts
View File

@ -1,136 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import EmptyReporter from './empty';
import { FullConfig, Test, Suite, Spec, TestResult, TestError } from '../types';
export interface SerializedSuite {
title: string;
file: string;
column: number;
line: number;
specs: ReturnType<JSONReporter['_serializeTestSpec']>[];
suites?: SerializedSuite[];
}
export type ReportFormat = {
config: FullConfig;
errors?: TestError[];
suites?: SerializedSuite[];
};
function toPosixPath(aPath: string): string {
return aPath.split(path.sep).join(path.posix.sep);
}
class JSONReporter extends EmptyReporter {
config: FullConfig;
suite: Suite;
private _errors: TestError[] = [];
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
}
onTimeout() {
this.onEnd();
}
onError(error: TestError): void {
this._errors.push(error);
}
onEnd() {
outputReport({
config: {
...this.config,
outputDir: toPosixPath(this.config.outputDir),
testDir: toPosixPath(this.config.testDir),
},
suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s),
errors: this._errors
});
}
private _serializeSuite(suite: Suite): null | SerializedSuite {
if (!suite.findSpec(test => true))
return null;
const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
return {
title: suite.title,
file: toPosixPath(path.relative(this.config.testDir, suite.file)),
line: suite.line,
column: suite.column,
specs: suite.specs.map(test => this._serializeTestSpec(test)),
suites: suites.length ? suites : undefined,
};
}
private _serializeTestSpec(spec: Spec) {
return {
title: spec.title,
ok: spec.ok(),
tests: spec.tests.map(r => this._serializeTest(r)),
file: toPosixPath(path.relative(this.config.testDir, spec.file)),
line: spec.line,
column: spec.column,
};
}
private _serializeTest(test: Test) {
return {
timeout: test.timeout,
annotations: test.annotations,
expectedStatus: test.expectedStatus,
results: test.results.map(r => this._serializeTestResult(r))
};
}
private _serializeTestResult(result: TestResult) {
return {
workerIndex: result.workerIndex,
status: result.status,
duration: result.duration,
error: result.error,
stdout: result.stdout.map(s => stdioEntry(s)),
stderr: result.stderr.map(s => stdioEntry(s)),
data: result.data,
retry: result.retry,
};
}
}
function outputReport(report: ReportFormat) {
const reportString = JSON.stringify(report, undefined, 2);
const outputName = process.env[`FOLIO_JSON_OUTPUT_NAME`];
if (outputName) {
fs.mkdirSync(path.dirname(outputName), { recursive: true });
fs.writeFileSync(outputName, reportString);
} else {
console.log(reportString);
}
}
function stdioEntry(s: string | Buffer): any {
if (typeof s === 'string')
return { text: s };
return { buffer: s.toString('base64') };
}
export default JSONReporter;

185
tests/folio/src/reporters/junit.ts
View File

@ -1,185 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import { FullConfig } from '../types';
import EmptyReporter from './empty';
import { Suite, Test } from '../test';
import { monotonicTime } from '../util';
import { formatFailure, stripAscii } from './base';
class JUnitReporter extends EmptyReporter {
private config: FullConfig;
private suite: Suite;
private timestamp: number;
private startTime: number;
private totalTests = 0;
private totalFailures = 0;
private totalSkipped = 0;
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
this.timestamp = Date.now();
this.startTime = monotonicTime();
}
onEnd() {
const duration = monotonicTime() - this.startTime;
const children: XMLEntry[] = [];
for (const suite of this.suite.suites)
children.push(this._buildTestSuite(suite));
const tokens: string[] = [];
const self = this;
const root: XMLEntry = {
name: 'testsuites',
attributes: {
id: process.env[`FOLIO_JUNIT_SUITE_ID`] || '',
name: process.env[`FOLIO_JUNIT_SUITE_NAME`] || '',
tests: self.totalTests,
failures: self.totalFailures,
skipped: self.totalSkipped,
errors: 0,
time: duration / 1000
},
children
};
serializeXML(root, tokens);
const reportString = tokens.join('\n');
const outputName = process.env[`FOLIO_JUNIT_OUTPUT_NAME`];
if (outputName) {
fs.mkdirSync(path.dirname(outputName), { recursive: true });
fs.writeFileSync(outputName, reportString);
} else {
console.log(reportString);
}
}
private _buildTestSuite(suite: Suite): XMLEntry {
let tests = 0;
let skipped = 0;
let failures = 0;
let duration = 0;
const children: XMLEntry[] = [];
suite.findTest(test => {
++tests;
if (test.skipped)
++skipped;
if (!test.ok())
++failures;
for (const result of test.results)
duration += result.duration;
this._addTestCase(test, children);
});
this.totalTests += tests;
this.totalSkipped += skipped;
this.totalFailures += failures;
const entry: XMLEntry = {
name: 'testsuite',
attributes: {
name: path.relative(this.config.testDir, suite.file),
timestamp: this.timestamp,
hostname: '',
tests,
failures,
skipped,
time: duration / 1000,
errors: 0,
},
children
};
return entry;
}
private _addTestCase(test: Test, entries: XMLEntry[]) {
const entry = {
name: 'testcase',
attributes: {
name: test.spec.fullTitle(),
classname: path.relative(this.config.testDir, test.spec.file) + ' ' + test.spec.parent.fullTitle(),
time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
},
children: []
};
entries.push(entry);
if (test.skipped) {
entry.children.push({ name: 'skipped'});
return;
}
if (!test.ok()) {
entry.children.push({
name: 'failure',
attributes: {
message: `${path.basename(test.spec.file)}:${test.spec.line}:${test.spec.column} ${test.spec.title}`,
type: 'FAILURE',
},
text: stripAscii(formatFailure(this.config, test))
});
}
for (const result of test.results) {
for (const stdout of result.stdout) {
entries.push({
name: 'system-out',
text: stdout.toString()
});
}
for (const stderr of result.stderr) {
entries.push({
name: 'system-err',
text: stderr.toString()
});
}
}
}
}
type XMLEntry = {
name: string;
attributes?: { [name: string]: string | number | boolean };
children?: XMLEntry[];
text?: string;
};
function serializeXML(entry: XMLEntry, tokens: string[]) {
const attrs: string[] = [];
for (const name of Object.keys(entry.attributes || {}))
attrs.push(`${name}="${escape(String(entry.attributes[name]))}"`);
tokens.push(`<${entry.name}${attrs.length ? ' ' : ''}${attrs.join(' ')}>`);
for (const child of entry.children || [])
serializeXML(child, tokens);
if (entry.text)
tokens.push(escape(entry.text));
tokens.push(`</${entry.name}>`);
}
function escape(text: string): string {
text = text.replace(/"/g, '&quot;');
text = text.replace(/&/g, '&amp;');
text = text.replace(/</g, '&lt;');
text = text.replace(/>/g, '&gt;');
return text;
}
export default JUnitReporter;

81
tests/folio/src/reporters/line.ts
View File

@ -1,81 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import * as path from 'path';
import { BaseReporter, formatFailure } from './base';
import { FullConfig, Test, Suite, TestResult } from '../types';
class LineReporter extends BaseReporter {
private _total: number;
private _current = 0;
private _failures = 0;
private _lastTest: Test;
onBegin(config: FullConfig, suite: Suite) {
super.onBegin(config, suite);
this._total = suite.totalTestCount();
console.log();
}
onStdOut(chunk: string | Buffer, test?: Test) {
this._dumpToStdio(test, chunk, process.stdout);
}
onStdErr(chunk: string | Buffer, test?: Test) {
this._dumpToStdio(test, chunk, process.stderr);
}
private _fullTitle(test: Test) {
const baseName = path.basename(test.spec.file);
const runListName = test.alias ? `[${test.alias}] ` : '';
return `${baseName} - ${runListName}${test.spec.fullTitle()}`;
}
private _dumpToStdio(test: Test | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
if (this.config.quiet)
return;
stream.write(`\u001B[1A\u001B[2K`);
if (test && this._lastTest !== test) {
// Write new header for the output.
stream.write(colors.gray(this._fullTitle(test) + `\n`));
this._lastTest = test;
}
stream.write(chunk);
console.log();
}
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
const width = process.stdout.columns - 1;
const title = `[${++this._current}/${this._total}] ${this._fullTitle(test)}`.substring(0, width);
process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
if (!this.willRetry(test, result) && !test.ok()) {
process.stdout.write(`\u001B[1A\u001B[2K`);
console.log(formatFailure(this.config, test, ++this._failures));
console.log();
}
}
onEnd() {
process.stdout.write(`\u001B[1A\u001B[2K`);
super.onEnd();
this.epilogue(false);
}
}
export default LineReporter;

79
tests/folio/src/reporters/list.ts
View File

@ -1,79 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import milliseconds from 'ms';
import { BaseReporter } from './base';
import { FullConfig, Suite, Test, TestResult } from '../types';
class ListReporter extends BaseReporter {
private _failure = 0;
private _lastRow = 0;
private _testRows = new Map<Test, number>();
onBegin(config: FullConfig, suite: Suite) {
super.onBegin(config, suite);
console.log();
}
onTestBegin(test: Test) {
super.onTestBegin(test);
if (process.stdout.isTTY)
process.stdout.write(' ' + colors.gray(test.spec.fullTitle() + ': ') + '\n');
this._testRows.set(test, this._lastRow++);
}
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
const spec = test.spec;
const duration = colors.dim(` (${milliseconds(result.duration)})`);
let text = '';
if (result.status === 'skipped') {
text = colors.green(' - ') + colors.cyan(spec.fullTitle());
} else {
const statusMark = result.status === 'passed' ? ' ✓ ' : ' x ';
if (result.status === test.expectedStatus)
text = '\u001b[2K\u001b[0G' + colors.green(statusMark) + colors.gray(spec.fullTitle()) + duration;
else
text = '\u001b[2K\u001b[0G' + colors.red(`${statusMark}${++this._failure}) ` + spec.fullTitle()) + duration;
}
const testRow = this._testRows.get(test);
// Go up if needed
if (process.stdout.isTTY && testRow !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - testRow}A`);
// Erase line
if (process.stdout.isTTY)
process.stdout.write('\u001B[2K');
process.stdout.write(text);
// Go down if needed.
if (testRow !== this._lastRow) {
if (process.stdout.isTTY)
process.stdout.write(`\u001B[${this._lastRow - testRow}E`);
else
process.stdout.write('\n');
}
}
onEnd() {
super.onEnd();
process.stdout.write('\n');
this.epilogue(true);
}
}
export default ListReporter;

65
tests/folio/src/reporters/multiplexer.ts
View File

@ -1,65 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FullConfig, Suite, Test, TestError, TestResult, Reporter } from '../types';
export class Multiplexer implements Reporter {
private _reporters: Reporter[];
constructor(reporters: Reporter[]) {
this._reporters = reporters;
}
onBegin(config: FullConfig, suite: Suite) {
for (const reporter of this._reporters)
reporter.onBegin(config, suite);
}
onTestBegin(test: Test) {
for (const reporter of this._reporters)
reporter.onTestBegin(test);
}
onStdOut(chunk: string | Buffer, test?: Test) {
for (const reporter of this._reporters)
reporter.onStdOut(chunk, test);
}
onStdErr(chunk: string | Buffer, test?: Test) {
for (const reporter of this._reporters)
reporter.onStdErr(chunk, test);
}
onTestEnd(test: Test, result: TestResult) {
for (const reporter of this._reporters)
reporter.onTestEnd(test, result);
}
onTimeout(timeout: number) {
for (const reporter of this._reporters)
reporter.onTimeout(timeout);
}
onEnd() {
for (const reporter of this._reporters)
reporter.onEnd();
}
onError(error: TestError) {
for (const reporter of this._reporters)
reporter.onError(error);
}
}

132
tests/folio/src/runner.ts
View File

@ -1,132 +0,0 @@
/**
* Copyright 2019 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import rimraf from 'rimraf';
import { promisify } from 'util';
import { Dispatcher } from './dispatcher';
import { Reporter } from './types';
import { createMatcher, monotonicTime, raceAgainstDeadline } from './util';
import { Suite } from './test';
import { Loader } from './loader';
const removeFolderAsync = promisify(rimraf);
type RunResult = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'no-tests';
export class Runner {
private _reporter: Reporter;
private _loader: Loader;
private _rootSuite: Suite;
constructor(loader: Loader, reporter: Reporter, runListFilter?: string[]) {
this._reporter = reporter;
this._loader = loader;
// This makes sure we don't generate 1000000 tests if only one spec is focused.
const filtered = new Set<Suite>();
for (const { fileSuites } of loader.runLists()) {
for (const fileSuite of fileSuites.values()) {
if (fileSuite._hasOnly())
filtered.add(fileSuite);
}
}
this._rootSuite = new Suite('');
const grepMatcher = createMatcher(loader.config().grep);
const nonEmptySuites = new Set<Suite>();
for (let runListIndex = 0; runListIndex < loader.runLists().length; runListIndex++) {
const runList = loader.runLists()[runListIndex];
if (runListFilter && !runListFilter.includes(runList.alias))
continue;
for (const fileSuite of runList.fileSuites.values()) {
if (filtered.size && !filtered.has(fileSuite))
continue;
const specs = fileSuite._allSpecs().filter(spec => grepMatcher(spec.fullTitle()));
if (!specs.length)
continue;
fileSuite._renumber();
for (const spec of specs) {
for (let i = 0; i < loader.config().repeatEach; ++i)
spec._appendTest(runListIndex, runList.alias, i);
}
nonEmptySuites.add(fileSuite);
}
}
for (const fileSuite of nonEmptySuites)
this._rootSuite._addSuite(fileSuite);
filterOnly(this._rootSuite);
}
list() {
this._reporter.onBegin(this._loader.config(), this._rootSuite);
this._reporter.onEnd();
}
async run(): Promise<RunResult> {
await removeFolderAsync(this._loader.config().outputDir).catch(e => {});
if (this._loader.config().forbidOnly) {
const hasOnly = this._rootSuite.findSpec(t => t._only) || this._rootSuite.findSuite(s => s._only);
if (hasOnly)
return 'forbid-only';
}
const total = this._rootSuite.totalTestCount();
if (!total)
return 'no-tests';
const globalDeadline = this._loader.config().globalTimeout ? this._loader.config().globalTimeout + monotonicTime() : 0;
const { result, timedOut } = await raceAgainstDeadline(this._runTests(this._rootSuite), globalDeadline);
if (timedOut) {
this._reporter.onTimeout(this._loader.config().globalTimeout);
process.exit(1);
}
return result;
}
private async _runTests(suite: Suite): Promise<RunResult> {
const dispatcher = new Dispatcher(this._loader, suite, this._reporter);
let sigint = false;
let sigintCallback: () => void;
const sigIntPromise = new Promise<void>(f => sigintCallback = f);
const sigintHandler = () => {
process.off('SIGINT', sigintHandler);
sigint = true;
sigintCallback();
};
process.on('SIGINT', sigintHandler);
this._reporter.onBegin(this._loader.config(), suite);
await Promise.race([dispatcher.run(), sigIntPromise]);
await dispatcher.stop();
this._reporter.onEnd();
if (sigint)
return 'sigint';
return dispatcher.hasWorkerErrors() || suite.findSpec(spec => !spec.ok()) ? 'failed' : 'passed';
}
}
function filterOnly(suite: Suite) {
const onlySuites = suite.suites.filter(child => filterOnly(child) || child._only);
const onlyTests = suite.specs.filter(spec => spec._only);
if (onlySuites.length || onlyTests.length) {
suite.suites = onlySuites;
suite.specs = onlyTests;
return true;
}
return false;
}

222
tests/folio/src/spec.ts
View File

@ -1,222 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { expect } from './expect';
import { currentTestInfo } from './globals';
import { Spec, Suite } from './test';
import { callLocation, errorWithCallLocation, interpretCondition } from './util';
import { Config, Env, RunWithConfig, TestInfo, TestType, WorkerInfo } from './types';
Error.stackTraceLimit = 15;
let currentFile: string | undefined;
export function setCurrentFile(file?: string) {
currentFile = file;
}
export type RunListDescription = {
alias: string;
fileSuites: Map<string, Suite>;
env: Env<any>;
config: RunWithConfig;
testType: TestType<any, any>;
};
export const configFile: {
config?: Config,
globalSetup?: () => any,
globalTeardown?: (globalSetupResult: any) => any,
runLists: RunListDescription[]
} = { runLists: [] };
function mergeEnvs(envs: any[]): any {
if (envs.length === 1)
return envs[0];
const forward = [...envs];
const backward = [...forward].reverse();
return {
beforeAll: async (workerInfo: WorkerInfo) => {
for (const env of forward) {
if (env.beforeAll)
await env.beforeAll(workerInfo);
}
},
afterAll: async (workerInfo: WorkerInfo) => {
let error: Error | undefined;
for (const env of backward) {
if (env.afterAll) {
try {
await env.afterAll(workerInfo);
} catch (e) {
error = error || e;
}
}
}
if (error)
throw error;
},
beforeEach: async (testInfo: TestInfo) => {
let result = undefined;
for (const env of forward) {
if (env.beforeEach) {
const r = await env.beforeEach(testInfo);
result = result === undefined ? r : { ...result, ...r };
}
}
return result;
},
afterEach: async (testInfo: TestInfo) => {
let error: Error | undefined;
for (const env of backward) {
if (env.afterEach) {
try {
await env.afterEach(testInfo);
} catch (e) {
error = error || e;
}
}
}
if (error)
throw error;
},
};
}
export function newTestTypeImpl(): any {
const fileSuites = new Map<string, Suite>();
let suites: Suite[] = [];
function ensureSuiteForCurrentLocation() {
const location = callLocation(currentFile);
let fileSuite = fileSuites.get(location.file);
if (!fileSuite) {
fileSuite = new Suite('');
fileSuite.file = location.file;
fileSuites.set(location.file, fileSuite);
}
if (suites[suites.length - 1] !== fileSuite)
suites = [fileSuite];
return location;
}
function spec(type: 'default' | 'only', title: string, options: Function | any, fn?: Function) {
if (!currentFile)
throw errorWithCallLocation(`Test can only be defined in a test file.`);
const location = ensureSuiteForCurrentLocation();
if (typeof fn !== 'function') {
fn = options;
options = {};
}
const spec = new Spec(title, fn, suites[0]);
spec.file = location.file;
spec.line = location.line;
spec.column = location.column;
spec.testOptions = options;
if (type === 'only')
spec._only = true;
}
function describe(type: 'default' | 'only', title: string, fn: Function) {
if (!currentFile)
throw errorWithCallLocation(`Suite can only be defined in a test file.`);
const location = ensureSuiteForCurrentLocation();
const child = new Suite(title, suites[0]);
child.file = location.file;
child.line = location.line;
child.column = location.column;
if (type === 'only')
child._only = true;
suites.unshift(child);
fn();
suites.shift();
}
function hook(name: string, fn: Function) {
if (!currentFile)
throw errorWithCallLocation(`Hook can only be defined in a test file.`);
ensureSuiteForCurrentLocation();
suites[0]._addHook(name, fn);
}
const modifier = (type: 'skip' | 'fail' | 'fixme', arg?: boolean | string, description?: string) => {
if (currentFile) {
const processed = interpretCondition(arg, description);
if (processed.condition)
suites[0]._annotations.push({ type, description: processed.description });
return;
}
const testInfo = currentTestInfo();
if (!testInfo)
throw new Error(`test.${type} can only be called inside the test`);
(testInfo[type] as any)(arg, description);
};
const test: any = spec.bind(null, 'default');
test.expect = expect;
test.only = spec.bind(null, 'only');
test.describe = describe.bind(null, 'default');
test.describe.only = describe.bind(null, 'only');
test.beforeEach = hook.bind(null, 'beforeEach');
test.afterEach = hook.bind(null, 'afterEach');
test.beforeAll = hook.bind(null, 'beforeAll');
test.afterAll = hook.bind(null, 'afterAll');
test.skip = modifier.bind(null, 'skip');
test.fixme = modifier.bind(null, 'fixme');
test.fail = modifier.bind(null, 'fail');
test.runWith = (...envs: any[]) => {
let alias = '';
if (typeof envs[0] === 'string') {
alias = envs[0];
envs = envs.slice(1);
}
let options = envs[envs.length - 1];
if (!envs.length || options.beforeAll || options.beforeEach || options.afterAll || options.afterEach)
options = {};
else
envs = envs.slice(0, envs.length - 1);
configFile.runLists.push({
fileSuites,
env: mergeEnvs(envs),
alias,
config: { timeout: options.timeout },
testType: test,
});
};
return test;
}
export function setConfig(config: Config) {
// TODO: add config validation.
configFile.config = config;
}
export function globalSetup(globalSetupFunction: () => any) {
if (typeof globalSetupFunction !== 'function')
throw errorWithCallLocation(`globalSetup takes a single function argument.`);
configFile.globalSetup = globalSetupFunction;
}
export function globalTeardown(globalTeardownFunction: (globalSetupResult: any) => any) {
if (typeof globalTeardownFunction !== 'function')
throw errorWithCallLocation(`globalTeardown takes a single function argument.`);
configFile.globalTeardown = globalTeardownFunction;
}

237
tests/folio/src/test.ts
View File

@ -1,237 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as types from './types';
class Base {
title: string;
file: string;
line: number;
column: number;
parent?: Suite;
_only = false;
_ordinal: number;
constructor(title: string, parent?: Suite) {
this.title = title;
this.parent = parent;
}
titlePath(): string[] {
if (!this.parent)
return [];
if (!this.title)
return this.parent.titlePath();
return [...this.parent.titlePath(), this.title];
}
fullTitle(): string {
return this.titlePath().join(' ');
}
}
export class Spec extends Base implements types.Spec {
fn: Function;
tests: Test[] = [];
testOptions: any = {};
constructor(title: string, fn: Function, suite: Suite) {
super(title, suite);
this.fn = fn;
suite._addSpec(this);
}
ok(): boolean {
return !this.tests.find(r => !r.ok());
}
_appendTest(runListIndex: number, runListAlias: string, repeatEachIndex: number) {
const test = new Test(this);
test.alias = runListAlias;
test._runListIndex = runListIndex;
test._workerHash = `${runListIndex}#repeat-${repeatEachIndex}`;
test._id = `${this._ordinal}@${this.file}::[${test._workerHash}]`;
test._repeatEachIndex = repeatEachIndex;
this.tests.push(test);
return test;
}
}
export class Suite extends Base implements types.Suite {
suites: Suite[] = [];
specs: Spec[] = [];
_entries: (Suite | Spec)[] = [];
_hooks: { type: string, fn: Function } [] = [];
_annotations: { type: 'skip' | 'fixme' | 'fail', description?: string }[] = [];
constructor(title: string, parent?: Suite) {
super(title, parent);
if (parent)
parent._addSuite(this);
}
_clear() {
this.suites = [];
this.specs = [];
this._entries = [];
this._hooks = [];
this._annotations = [];
}
_addSpec(spec: Spec) {
spec.parent = this;
this.specs.push(spec);
this._entries.push(spec);
}
_addSuite(suite: Suite) {
suite.parent = this;
this.suites.push(suite);
this._entries.push(suite);
}
findTest(fn: (test: Test) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.findTest(fn))
return true;
}
for (const spec of this.specs) {
for (const test of spec.tests) {
if (fn(test))
return true;
}
}
return false;
}
findSpec(fn: (spec: Spec) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.findSpec(fn))
return true;
}
for (const spec of this.specs) {
if (fn(spec))
return true;
}
return false;
}
findSuite(fn: (suite: Suite) => boolean | void): boolean {
if (fn(this))
return true;
for (const suite of this.suites) {
if (suite.findSuite(fn))
return true;
}
return false;
}
totalTestCount(): number {
let total = 0;
for (const suite of this.suites)
total += suite.totalTestCount();
for (const spec of this.specs)
total += spec.tests.length;
return total;
}
_allSpecs(): Spec[] {
const result: Spec[] = [];
this.findSpec(test => { result.push(test); });
return result;
}
_renumber() {
// All tests are identified with their ordinals.
let ordinal = 0;
this.findSpec((test: Spec) => {
test._ordinal = ordinal++;
});
}
_hasOnly(): boolean {
if (this._only)
return true;
if (this.suites.find(suite => suite._hasOnly()))
return true;
if (this.specs.find(spec => spec._only))
return true;
}
_addHook(type: string, fn: any) {
this._hooks.push({ type, fn });
}
}
export class Test implements types.Test {
spec: Spec;
results: types.TestResult[] = [];
skipped = false;
expectedStatus: types.TestStatus = 'passed';
timeout = 0;
annotations: any[] = [];
alias = '';
_id: string;
_workerHash: string;
_repeatEachIndex: number;
_runListIndex: number;
constructor(spec: Spec) {
this.spec = spec;
}
status(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
if (this.skipped)
return 'skipped';
// List mode bail out.
if (!this.results.length)
return 'skipped';
if (this.results.length === 1 && this.expectedStatus === this.results[0].status)
return 'expected';
let hasPassedResults = false;
for (const result of this.results) {
// Missing status is Ok when running in shards mode.
if (!result.status)
return 'skipped';
if (result.status === this.expectedStatus)
hasPassedResults = true;
}
if (hasPassedResults)
return 'flaky';
return 'unexpected';
}
ok(): boolean {
const status = this.status();
return status === 'expected' || status === 'flaky' || status === 'skipped';
}
_appendTestResult(): types.TestResult {
const result: types.TestResult = {
retry: this.results.length,
workerIndex: 0,
duration: 0,
stdout: [],
stderr: [],
data: {}
};
this.results.push(result);
return result;
}
}

80
tests/folio/src/transform.ts
View File

@ -1,80 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as crypto from 'crypto';
import * as os from 'os';
import * as path from 'path';
import * as fs from 'fs';
import * as pirates from 'pirates';
import * as babel from '@babel/core';
import * as sourceMapSupport from 'source-map-support';
const version = 3;
const cacheDir = path.join(os.tmpdir(), 'playwright-transform-cache');
const sourceMaps: Map<string, string> = new Map();
sourceMapSupport.install({
environment: 'node',
handleUncaughtExceptions: false,
retrieveSourceMap(source) {
if (!sourceMaps.has(source))
return null;
const sourceMapPath = sourceMaps.get(source);
if (!fs.existsSync(sourceMapPath))
return null;
return {
map: JSON.parse(fs.readFileSync(sourceMapPath, 'utf-8')),
url: source
};
}
});
function calculateCachePath(content: string, filePath: string): string {
const hash = crypto.createHash('sha1').update(content).update(filePath).update(String(version)).digest('hex');
const fileName = path.basename(filePath, path.extname(filePath)).replace(/\W/g, '') + '_' + hash;
return path.join(cacheDir, hash[0] + hash[1], fileName);
}
export function installTransform(): () => void {
return pirates.addHook((code, filename) => {
const cachePath = calculateCachePath(code, filename);
const codePath = cachePath + '.js';
const sourceMapPath = cachePath + '.map';
sourceMaps.set(filename, sourceMapPath);
if (fs.existsSync(codePath))
return fs.readFileSync(codePath, 'utf8');
const result = babel.transformFileSync(filename, {
babelrc: false,
configFile: false,
presets: [
['@babel/preset-env', { targets: {node: '10.17.0'} }],
['@babel/preset-typescript', { onlyRemoveTypeImports: true }],
],
plugins: [['@babel/plugin-proposal-class-properties', {loose: true}]],
sourceMaps: 'both',
});
if (result.code) {
fs.mkdirSync(path.dirname(cachePath), {recursive: true});
if (result.map)
fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');
fs.writeFileSync(codePath, result.code, 'utf8');
}
return result.code;
}, {
exts: ['.ts']
});
}

196
tests/folio/src/types.ts
View File

@ -1,196 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Expect } from './expectType';
export interface RunWithConfig {
timeout?: number;
// TODO: move retries, outputDir, repeatEach, snapshotDir, testPathSegment here from Config.
}
export interface Config extends RunWithConfig {
forbidOnly?: boolean;
globalTimeout?: number;
grep?: string | RegExp | (string | RegExp)[];
maxFailures?: number;
outputDir?: string;
quiet?: boolean;
repeatEach?: number;
retries?: number;
shard?: { total: number, current: number } | null;
snapshotDir?: string;
testDir?: string;
testIgnore?: string | RegExp | (string | RegExp)[];
testMatch?: string | RegExp | (string | RegExp)[];
updateSnapshots?: boolean;
workers?: number;
}
export type FullConfig = Required<Config>;
interface TestModifier {
skip(): void;
skip(condition: boolean): void;
skip(description: string): void;
skip(condition: boolean, description: string): void;
fixme(): void;
fixme(condition: boolean): void;
fixme(description: string): void;
fixme(condition: boolean, description: string): void;
fail(): void;
fail(condition: boolean): void;
fail(description: string): void;
fail(condition: boolean, description: string): void;
}
export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
export interface WorkerInfo {
config: FullConfig;
workerIndex: number;
globalSetupResult: any;
}
export interface TestInfo extends WorkerInfo, TestModifier {
// Declaration
title: string;
file: string;
line: number;
column: number;
fn: Function;
// Modifiers
expectedStatus: TestStatus;
timeout: number;
annotations: any[];
testOptions: any; // TODO: make testOptions typed.
repeatEachIndex: number;
retry: number;
// Results
duration: number;
status?: TestStatus;
error?: any;
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
data: any;
// Paths
snapshotPathSegment: string;
snapshotPath: (...pathSegments: string[]) => string;
outputPath: (...pathSegments: string[]) => string;
}
interface SuiteFunction {
(name: string, inner: () => void): void;
}
interface TestFunction<TestArgs, TestOptions> {
(name: string, inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void): void;
(name: string, options: TestOptions, fn: (args: TestArgs, testInfo: TestInfo) => any): void;
}
export interface TestType<TestArgs, TestOptions> extends TestFunction<TestArgs, TestOptions>, TestModifier {
only: TestFunction<TestArgs, TestOptions>;
describe: SuiteFunction & {
only: SuiteFunction;
};
beforeEach: (inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void) => void;
afterEach: (inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void) => void;
beforeAll: (inner: (workerInfo: WorkerInfo) => Promise<void> | void) => void;
afterAll: (inner: (workerInfo: WorkerInfo) => Promise<void> | void) => void;
expect: Expect;
runWith(config?: RunWithConfig): void;
runWith(alias: string, config?: RunWithConfig): void;
runWith(env: Env<TestArgs>, config?: RunWithConfig): void;
runWith(alias: string, env: Env<TestArgs>, config?: RunWithConfig): void;
runWith<TestArgs1, TestArgs2>(env1: Env<TestArgs1>, env2: Env<TestArgs2>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2>;
runWith<TestArgs1, TestArgs2>(alias: string, env1: Env<TestArgs1>, env2: Env<TestArgs2>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2>;
runWith<TestArgs1, TestArgs2, TestArgs3>(env1: Env<TestArgs1>, env2: Env<TestArgs2>, env3: Env<TestArgs3>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2 & TestArgs3>;
runWith<TestArgs1, TestArgs2, TestArgs3>(alias: string, env1: Env<TestArgs1>, env2: Env<TestArgs2>, env3: Env<TestArgs3>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2 & TestArgs3>;
}
export interface Env<TestArgs> {
beforeAll?(workerInfo: WorkerInfo): Promise<any>;
beforeEach?(testInfo: TestInfo): Promise<TestArgs>;
afterEach?(testInfo: TestInfo): Promise<any>;
afterAll?(workerInfo: WorkerInfo): Promise<any>;
}
type RunWithOrNever<ExpectedTestArgs, CombinedTestArgs> = CombinedTestArgs extends ExpectedTestArgs ? void : never;
// ---------- Reporters API -----------
export interface Suite {
title: string;
file: string;
line: number;
column: number;
suites: Suite[];
specs: Spec[];
findTest(fn: (test: Test) => boolean | void): boolean;
findSpec(fn: (spec: Spec) => boolean | void): boolean;
totalTestCount(): number;
}
export interface Spec {
title: string;
file: string;
line: number;
column: number;
tests: Test[];
fullTitle(): string;
ok(): boolean;
}
export interface Test {
spec: Spec;
results: TestResult[];
skipped: boolean;
expectedStatus: TestStatus;
timeout: number;
annotations: any[];
alias: string;
status(): 'skipped' | 'expected' | 'unexpected' | 'flaky';
ok(): boolean;
}
export interface TestResult {
retry: number;
workerIndex: number,
duration: number;
status?: TestStatus;
error?: TestError;
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
data: any;
}
export interface TestError {
message?: string;
stack?: string;
value?: string;
}
export interface Reporter {
onBegin(config: FullConfig, suite: Suite): void;
onTestBegin(test: Test): void;
onStdOut(chunk: string | Buffer, test?: Test): void;
onStdErr(chunk: string | Buffer, test?: Test): void;
onTestEnd(test: Test, result: TestResult): void;
onTimeout(timeout: number): void;
onError(error: TestError): void;
onEnd(): void;
}

139
tests/folio/src/util.ts
View File

@ -1,139 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import path from 'path';
import util from 'util';
import StackUtils from 'stack-utils';
import { TestError } from './types';
import { default as minimatch } from 'minimatch';
const FOLIO_DIRS = [__dirname, path.join(__dirname, '..', 'src')];
const cwd = process.cwd();
const stackUtils = new StackUtils({ cwd });
export async function raceAgainstDeadline<T>(promise: Promise<T>, deadline: number): Promise<{ result?: T, timedOut?: boolean }> {
if (!deadline)
return { result: await promise };
const timeout = deadline - monotonicTime();
if (timeout <= 0)
return { timedOut: true };
let timer: NodeJS.Timer;
let done = false;
let fulfill: (t: { result?: T, timedOut?: boolean }) => void;
let reject: (e: Error) => void;
const result = new Promise((f, r) => {
fulfill = f;
reject = r;
});
setTimeout(() => {
done = true;
fulfill({ timedOut: true });
}, timeout);
promise.then(result => {
clearTimeout(timer);
if (!done) {
done = true;
fulfill({ result });
}
}).catch(e => {
clearTimeout(timer);
if (!done)
reject(e);
});
return result;
}
export function serializeError(error: Error | any): TestError {
if (error instanceof Error) {
return {
message: error.message,
stack: error.stack
};
}
return {
value: util.inspect(error)
};
}
function callFrames(): string[] {
const obj = { stack: '' };
Error.captureStackTrace(obj);
const frames = obj.stack.split('\n').slice(1);
while (frames.length && FOLIO_DIRS.some(dir => frames[0].includes(dir)))
frames.shift();
return frames;
}
export function callLocation(fallbackFile: string): {file: string, line: number, column: number} {
const frames = callFrames();
if (!frames.length)
return {file: fallbackFile, line: 1, column: 1};
const location = stackUtils.parseLine(frames[0]);
return {
file: path.resolve(cwd, location.file),
line: location.line,
column: location.column,
};
}
export function errorWithCallLocation(message: string): Error {
const frames = callFrames();
const error = new Error(message);
error.stack = 'Error: ' + message + '\n' + frames.join('\n');
return error;
}
export function monotonicTime(): number {
const [seconds, nanoseconds] = process.hrtime();
return seconds * 1000 + (nanoseconds / 1000000 | 0);
}
export function prependErrorMessage(e: Error, message: string) {
let stack = e.stack || '';
if (stack.includes(e.message))
stack = stack.substring(stack.indexOf(e.message) + e.message.length);
let m = e.message;
if (m.startsWith('Error:'))
m = m.substring('Error:'.length);
e.message = message + m;
e.stack = e.message + stack;
}
export function interpretCondition(arg?: boolean | string, description?: string): { condition: boolean, description?: string } {
if (arg === undefined && description === undefined)
return { condition: true };
if (typeof arg === 'string')
return { condition: true, description: arg };
return { condition: !!arg, description };
}
export function createMatcher(patterns: string | RegExp | (string | RegExp)[]): (value: string) => boolean {
const list = Array.isArray(patterns) ? patterns : [patterns];
return (value: string) => {
for (const pattern of list) {
if (pattern instanceof RegExp || Object.prototype.toString.call(pattern) === '[object RegExp]') {
if ((pattern as RegExp).test(value))
return true;
} else {
if (minimatch(value, pattern))
return true;
}
}
return false;
};
}

119
tests/folio/src/worker.ts
View File

@ -1,119 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Console } from 'console';
import * as util from 'util';
import { RunPayload, TestOutputPayload, WorkerInitParams } from './ipc';
import { serializeError } from './util';
import { WorkerRunner } from './workerRunner';
let closed = false;
sendMessageToParent('ready');
global.console = new Console({
stdout: process.stdout,
stderr: process.stderr,
colorMode: process.env.FORCE_COLOR === '1',
});
process.stdout.write = chunk => {
const outPayload: TestOutputPayload = {
testId: workerRunner ? workerRunner._testId : undefined,
...chunkToParams(chunk)
};
sendMessageToParent('stdOut', outPayload);
return true;
};
if (!process.env.PW_RUNNER_DEBUG) {
process.stderr.write = chunk => {
const outPayload: TestOutputPayload = {
testId: workerRunner ? workerRunner._testId : undefined,
...chunkToParams(chunk)
};
sendMessageToParent('stdErr', outPayload);
return true;
};
}
process.on('disconnect', gracefullyCloseAndExit);
process.on('SIGINT',() => {});
process.on('SIGTERM',() => {});
let workerRunner: WorkerRunner;
process.on('unhandledRejection', (reason, promise) => {
if (workerRunner)
workerRunner.unhandledError(reason);
});
process.on('uncaughtException', error => {
if (workerRunner)
workerRunner.unhandledError(error);
});
process.on('message', async message => {
if (message.method === 'init') {
const initParams = message.params as WorkerInitParams;
workerRunner = new WorkerRunner(initParams);
for (const event of ['testBegin', 'testEnd', 'done'])
workerRunner.on(event, sendMessageToParent.bind(null, event));
return;
}
if (message.method === 'stop') {
await gracefullyCloseAndExit();
return;
}
if (message.method === 'run') {
const runPayload = message.params as RunPayload;
await workerRunner!.run(runPayload);
}
});
async function gracefullyCloseAndExit() {
if (closed)
return;
closed = true;
// Force exit after 30 seconds.
setTimeout(() => process.exit(0), 30000);
// Meanwhile, try to gracefully shutdown.
try {
if (workerRunner) {
workerRunner.stop();
await workerRunner.cleanup();
}
} catch (e) {
process.send({ method: 'teardownError', params: { error: serializeError(e) } });
}
process.exit(0);
}
function sendMessageToParent(method, params = {}) {
try {
process.send({ method, params });
} catch (e) {
// Can throw when closing.
}
}
function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: string } {
if (chunk instanceof Buffer)
return { buffer: chunk.toString('base64') };
if (typeof chunk !== 'string')
return { text: util.inspect(chunk) };
return { text: chunk };
}

462
tests/folio/src/workerRunner.ts
View File

@ -1,462 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import { EventEmitter } from 'events';
import { interpretCondition, monotonicTime, raceAgainstDeadline, serializeError } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
import { setCurrentTestInfo } from './globals';
import { Loader } from './loader';
import { Spec, Suite, Test } from './test';
import { TestInfo, WorkerInfo } from './types';
import { RunListDescription } from './spec';
import { extendAgain } from './expect';
export class WorkerRunner extends EventEmitter {
private _params: WorkerInitParams;
private _loader: Loader;
private _runList: RunListDescription;
private _outputPathSegment: string;
private _workerInfo: WorkerInfo;
private _envInitialized = false;
private _failedTestId: string | undefined;
private _fatalError: any | undefined;
private _entries: Map<string, TestEntry>;
private _remaining: Map<string, TestEntry>;
private _isStopped: any;
_testId: string | null;
private _testInfo: TestInfo | null = null;
private _file: string;
private _timeout: number;
constructor(params: WorkerInitParams) {
super();
this._params = params;
}
stop() {
this._isStopped = true;
this._testId = null;
this._setCurrentTestInfo(null);
}
async cleanup() {
if (!this._envInitialized)
return;
this._envInitialized = false;
if (this._runList.env.afterAll) {
// TODO: separate timeout for afterAll?
const result = await raceAgainstDeadline(this._runList.env.afterAll(this._workerInfo), this._deadline());
if (result.timedOut)
throw new Error(`Timeout of ${this._timeout}ms exceeded while shutting down environment`);
}
}
unhandledError(error: Error | any) {
if (this._isStopped)
return;
if (this._testInfo) {
this._testInfo.status = 'failed';
this._testInfo.error = serializeError(error);
this._failedTestId = this._testId;
this.emit('testEnd', buildTestEndPayload(this._testId, this._testInfo));
} else {
// No current test - fatal error.
this._fatalError = serializeError(error);
}
this._reportDoneAndStop();
}
private _deadline() {
return this._timeout ? monotonicTime() + this._timeout : 0;
}
private async _loadIfNeeded() {
if (this._loader)
return;
this._loader = new Loader();
this._loader.deserialize(this._params.loader);
this._runList = this._loader.runLists()[this._params.runListIndex];
const sameAliasAndTestType = this._loader.runLists().filter(runList => runList.alias === this._runList.alias && runList.testType === this._runList.testType);
if (sameAliasAndTestType.length > 1)
this._outputPathSegment = this._runList.alias + (sameAliasAndTestType.indexOf(this._runList) + 1);
else
this._outputPathSegment = this._runList.alias;
this._timeout = this._runList.config.timeout === undefined ? this._loader.config().timeout : this._runList.config.timeout;
this._workerInfo = {
workerIndex: this._params.workerIndex,
config: this._loader.config(),
globalSetupResult: this._params.globalSetupResult,
};
if (this._isStopped)
return;
if (this._runList.env.beforeAll) {
// TODO: separate timeout for beforeAll?
const result = await raceAgainstDeadline(this._runList.env.beforeAll(this._workerInfo), this._deadline());
if (result.timedOut) {
this._fatalError = serializeError(new Error(`Timeout of ${this._timeout}ms exceeded while initializing environment`));
this._reportDoneAndStop();
}
}
this._envInitialized = true;
}
async run(runPayload: RunPayload) {
this._file = runPayload.file;
this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
this._remaining = new Map(runPayload.entries.map(e => [ e.testId, e ]));
await this._loadIfNeeded();
if (this._isStopped)
return;
this._loader.loadTestFile(this._file);
extendAgain();
const fileSuite = this._runList.fileSuites.get(this._file);
if (fileSuite) {
fileSuite._renumber();
fileSuite.findSpec(spec => {
spec._appendTest(this._params.runListIndex, this._runList.alias, this._params.repeatEachIndex);
});
await this._runSuite(fileSuite);
}
if (this._isStopped)
return;
this._reportDone();
}
private async _runSuite(suite: Suite) {
if (this._isStopped)
return;
const skipHooks = !this._hasTestsToRun(suite);
for (const hook of suite._hooks) {
if (hook.type !== 'beforeAll' || skipHooks)
continue;
if (this._isStopped)
return;
// TODO: separate timeout for beforeAll?
const result = await raceAgainstDeadline(hook.fn(this._workerInfo), this._deadline());
if (result.timedOut) {
this._fatalError = serializeError(new Error(`Timeout of ${this._timeout}ms exceeded while running beforeAll hook`));
this._reportDoneAndStop();
}
}
for (const entry of suite._entries) {
if (entry instanceof Suite)
await this._runSuite(entry);
else
await this._runSpec(entry);
}
for (const hook of suite._hooks) {
if (hook.type !== 'afterAll' || skipHooks)
continue;
if (this._isStopped)
return;
// TODO: separate timeout for afterAll?
const result = await raceAgainstDeadline(hook.fn(this._workerInfo), this._deadline());
if (result.timedOut) {
this._fatalError = serializeError(new Error(`Timeout of ${this._timeout}ms exceeded while running afterAll hook`));
this._reportDoneAndStop();
}
}
}
private async _runSpec(spec: Spec) {
if (this._isStopped)
return;
const test = spec.tests[0];
if (!this._entries.has(test._id))
return;
const { retry } = this._entries.get(test._id);
// TODO: support some of test.slow(), test.setTimeout(), describe.slow() and describe.setTimeout()
const deadline = this._deadline();
this._remaining.delete(test._id);
const testId = test._id;
this._testId = testId;
const config = this._workerInfo.config;
const relativePath = path.relative(config.testDir, spec.file.replace(/\.(spec|test)\.(js|ts)/, ''));
const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '-');
const relativeTestPath = path.join(relativePath, sanitizedTitle);
const testInfo: TestInfo = {
...this._workerInfo,
title: spec.title,
file: spec.file,
line: spec.line,
column: spec.column,
fn: spec.fn,
repeatEachIndex: this._params.repeatEachIndex,
retry,
expectedStatus: 'passed',
annotations: [],
duration: 0,
status: 'passed',
stdout: [],
stderr: [],
timeout: this._timeout,
data: {},
snapshotPathSegment: '',
outputPath: (...pathSegments: string[]): string => {
let suffix = this._outputPathSegment;
if (testInfo.retry)
suffix += (suffix ? '-' : '') + 'retry' + testInfo.retry;
if (testInfo.repeatEachIndex)
suffix += (suffix ? '-' : '') + 'repeat' + testInfo.repeatEachIndex;
const basePath = path.join(config.outputDir, relativeTestPath, suffix);
fs.mkdirSync(basePath, { recursive: true });
return path.join(basePath, ...pathSegments);
},
snapshotPath: (...pathSegments: string[]): string => {
const basePath = path.join(config.testDir, config.snapshotDir, relativeTestPath, testInfo.snapshotPathSegment);
return path.join(basePath, ...pathSegments);
},
testOptions: spec.testOptions,
skip: (arg?: boolean | string, description?: string) => modifier(testInfo, 'skip', arg, description),
fixme: (arg?: boolean | string, description?: string) => modifier(testInfo, 'fixme', arg, description),
fail: (arg?: boolean | string, description?: string) => modifier(testInfo, 'fail', arg, description),
};
this._setCurrentTestInfo(testInfo);
// Preprocess suite annotations.
for (let parent = spec.parent; parent; parent = parent.parent)
testInfo.annotations.push(...parent._annotations);
if (testInfo.annotations.some(a => a.type === 'skip' || a.type === 'fixme'))
testInfo.expectedStatus = 'skipped';
else if (testInfo.annotations.some(a => a.type === 'fail'))
testInfo.expectedStatus = 'failed';
this.emit('testBegin', buildTestBeginPayload(testId, testInfo));
if (testInfo.expectedStatus === 'skipped') {
testInfo.status = 'skipped';
this.emit('testEnd', buildTestEndPayload(testId, testInfo));
return;
}
const startTime = monotonicTime();
const testArgsResult = await raceAgainstDeadline(this._runEnvBeforeEach(testInfo), deadline);
if (testArgsResult.timedOut && testInfo.status === 'passed')
testInfo.status = 'timedOut';
if (this._isStopped)
return;
const testArgs = testArgsResult.result;
// Do not run test/teardown if we failed to initialize.
if (testArgs !== undefined) {
const result = await raceAgainstDeadline(this._runTestWithBeforeHooks(test, testInfo, testArgs), deadline);
// Do not overwrite test failure upon hook timeout.
if (result.timedOut && testInfo.status === 'passed')
testInfo.status = 'timedOut';
if (this._isStopped)
return;
if (!result.timedOut) {
const hooksResult = await raceAgainstDeadline(this._runAfterHooks(test, testInfo, testArgs), deadline);
// Do not overwrite test failure upon hook timeout.
if (hooksResult.timedOut && testInfo.status === 'passed')
testInfo.status = 'timedOut';
} else {
// A timed-out test gets a full additional timeout to run after hooks.
const newDeadline = this._deadline();
await raceAgainstDeadline(this._runAfterHooks(test, testInfo, testArgs), newDeadline);
}
}
if (this._isStopped)
return;
testInfo.duration = monotonicTime() - startTime;
this.emit('testEnd', buildTestEndPayload(testId, testInfo));
if (testInfo.status !== 'passed') {
this._failedTestId = this._testId;
this._reportDoneAndStop();
}
this._setCurrentTestInfo(null);
this._testId = null;
}
private _setCurrentTestInfo(testInfo: TestInfo | null) {
this._testInfo = testInfo;
setCurrentTestInfo(testInfo);
}
// Returns TestArgs or undefined when env.beforeEach has failed.
private async _runEnvBeforeEach(testInfo: TestInfo): Promise<any> {
try {
let testArgs: any = {};
if (this._runList.env.beforeEach)
testArgs = await this._runList.env.beforeEach(testInfo);
if (testArgs === undefined)
testArgs = {};
return testArgs;
} catch (error) {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
// Failed to initialize environment - no need to run any hooks now.
return undefined;
}
}
private async _runTestWithBeforeHooks(test: Test, testInfo: TestInfo, testArgs: any) {
try {
await this._runHooks(test.spec.parent, 'beforeEach', testArgs, testInfo);
} catch (error) {
if (error instanceof SkipError) {
testInfo.status = 'skipped';
} else {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
}
// Continue running afterEach hooks even after the failure.
}
// Do not run the test when beforeEach hook fails.
if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped')
return;
try {
await test.spec.fn(testArgs, testInfo);
testInfo.status = 'passed';
} catch (error) {
if (error instanceof SkipError) {
testInfo.status = 'skipped';
} else {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
}
}
}
private async _runAfterHooks(test: Test, testInfo: TestInfo, testArgs: any) {
try {
await this._runHooks(test.spec.parent, 'afterEach', testArgs, testInfo);
} catch (error) {
// Do not overwrite test failure error.
if (!(error instanceof SkipError) && testInfo.status === 'passed') {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
// Continue running even after the failure.
}
}
try {
if (this._runList.env.afterEach)
await this._runList.env.afterEach(testInfo);
} catch (error) {
// Do not overwrite test failure error.
if (testInfo.status === 'passed') {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
}
}
}
private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testArgs: any, testInfo: TestInfo) {
if (this._isStopped)
return;
const all = [];
for (let s = suite; s; s = s.parent) {
const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn);
all.push(...funcs.reverse());
}
if (type === 'beforeEach')
all.reverse();
let error: Error | undefined;
for (const hook of all) {
try {
await hook(testArgs, testInfo);
} catch (e) {
// Always run all the hooks, and capture the first error.
error = error || e;
}
}
if (error)
throw error;
}
private _reportDone() {
const donePayload: DonePayload = {
failedTestId: this._failedTestId,
fatalError: this._fatalError,
remaining: [...this._remaining.values()],
};
this.emit('done', donePayload);
}
private _reportDoneAndStop() {
if (this._isStopped)
return;
this._reportDone();
this.stop();
}
private _hasTestsToRun(suite: Suite): boolean {
return suite.findSpec(spec => {
const entry = this._entries.get(spec.tests[0]._id);
if (!entry)
return;
for (let parent = spec.parent; parent; parent = parent.parent) {
if (parent._annotations.some(a => a.type === 'skip' || a.type === 'fixme'))
return;
}
return true;
});
}
}
function buildTestBeginPayload(testId: string, testInfo: TestInfo): TestBeginPayload {
return {
testId,
workerIndex: testInfo.workerIndex
};
}
function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload {
return {
testId,
duration: testInfo.duration,
status: testInfo.status!,
error: testInfo.error,
data: testInfo.data,
expectedStatus: testInfo.expectedStatus,
annotations: testInfo.annotations,
timeout: testInfo.timeout,
};
}
function modifier(testInfo: TestInfo, type: 'skip' | 'fail' | 'fixme', arg?: boolean | string, description?: string) {
const processed = interpretCondition(arg, description);
if (!processed.condition)
return;
testInfo.annotations.push({ type, description: processed.description });
if (type === 'skip' || type === 'fixme') {
testInfo.expectedStatus = 'skipped';
throw new SkipError(processed.description);
} else if (type === 'fail') {
if (testInfo.expectedStatus !== 'skipped')
testInfo.expectedStatus = 'failed';
}
}
class SkipError extends Error {
}

2222
tests/folio/third_party/diff_match_patch.js vendored
View File

File diff suppressed because it is too large Load Diff

22
tests/folio/tsconfig.json
View File

@ -1,22 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"moduleResolution": "node",
"module": "commonjs",
"strict": false,
"sourceMap": true,
"declarationMap": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"declaration": true,
"outDir": "./out",
"rootDir": "./src",
"lib": ["esnext", "dom", "DOM.Iterable"],
},
"exclude": [
"node_modules"
],
"include": [
"src"
]
}

20
utils/doclint/test/class-testapi.md
View File

@ -1,20 +0,0 @@
# class: DoesNotExist
## method: DoesNotExist.doesNotExist
# class: Exists
## method: Exists.doesNotExist
## method: Exists.exists
### param: Exists.exists.exists
- `exists` <[boolean]>
### param: Exists.exists.doesNotExist
- `doesNotExist` <[boolean]>
### option: Exists.exists.option
- `option` <[int]>
## method: Exists.exists2

42
utils/doclint/test/missingDocs.spec.js
View File

@ -1,42 +0,0 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const fs = require('fs');
const path = require('path');
const missingDocs = require('../missingDocs');
const { folio } = require('folio');
const { parseApi } = require('../api_parser');
const { test, expect } = folio;
test('missing docs', async ({}) => {
const documentation = parseApi(path.join(__dirname));
const tsSources = [
path.join(__dirname, 'test-api.ts'),
path.join(__dirname, 'test-api-class.ts'),
];
const errors = missingDocs(documentation, tsSources, path.join(__dirname, 'test-api.ts'));
expect(errors).toEqual([
'Missing documentation for "Exists.exists2.extra"',
'Missing documentation for "Exists.exists2.options"',
'Missing documentation for "Exists.extra"',
'Missing documentation for "Extra"',
'Documented "DoesNotExist" not found in sources',
'Documented "Exists.doesNotExist" not found is sources',
'Documented "Exists.exists.doesNotExist" not found is sources',
]);
});

35
utils/doclint/test/test-api-class.ts
View File

@ -1,35 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export class Exists {
exists(exists: boolean) {
return true;
}
exists2(extra: boolean, options: {}) {
return true;
}
extra() {
return false;
}
}
export class Extra {
exists() {
return true;
}
}

18
utils/doclint/test/test-api.ts
View File

@ -1,18 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { Exists } from './test-api-class';
export { Extra } from './test-api-class';