microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-08-17 08:00:23 +03:00
Code Issues Projects Releases Wiki Activity

test: bring new folio and migrate small amount of tests to it (#5994)

Browse Source
This commit is contained in:
Dmitry Gozman 2021-04-01 16:35:26 -07:00 committed by GitHub
parent 66541552d0
commit be79b3883b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 6903 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintignore
View File

@ -13,5 +13,5 @@ utils/generate_types/test/test.ts
node_modules/
browser_patches/*/checkout/
browser_patches/chromium/output/
packages/**/*.d.ts
**/*.d.ts
output/

53
.github/workflows/tests.yml vendored
View File

@ -38,10 +38,17 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json && node test/checkCoverage.js"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
env:
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
env:
BROWSER: ${{ matrix.browser }}
FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
# Checking coverage across two test suites is hard. Temporary disabled.
# - run: node test/checkCoverage.js
# env:
# BROWSER: ${{ matrix.browser }}
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
- uses: actions/upload-artifact@v1
@ -66,6 +73,9 @@ jobs:
- run: npm ci
- run: npm run build
- run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
- run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
env:
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
env:
BROWSER: ${{ matrix.browser }}
@ -96,6 +106,10 @@ jobs:
- run: npm ci
- run: npm run build
- run: node lib/cli/cli install-deps
- run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
shell: bash
env:
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
shell: bash
env:
@ -150,6 +164,11 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
if: ${{ always() }}
env:
HEADFUL: 1
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
if: ${{ always() }}
env:
@ -185,6 +204,10 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
env:
PWMODE: "${{ matrix.mode }}"
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
env:
BROWSER: "chromium"
@ -219,6 +242,10 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
env:
PWVIDEO: 1
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json -p video"
env:
BROWSER: ${{ matrix.browser }}
@ -249,8 +276,10 @@ jobs:
run: utils/avd_recreate.sh
- name: Start Android Emulator
run: utils/avd_start.sh
- name: Run device tests
run: npx folio test/android -p browserName=chromium --workers=1 --forbid-only --timeout=120000 --global-timeout=5400000 --retries=3 --reporter=dot,json
- name: Run tests
run: npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts
env:
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- name: Run page tests
run: npx folio test/page -p browserName=chromium --workers=1 --forbid-only --timeout=120000 --global-timeout=5400000 --retries=3 --reporter=dot,json
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@ -286,6 +315,10 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
env:
PW_CHROMIUM_CHANNEL: "chrome"
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
env:
BROWSER: "chromium"
@ -314,6 +347,11 @@ jobs:
- run: npm run build
# This only created problems, should we move ffmpeg back into npm?
- run: node lib/cli/cli install ffmpeg
- run: npm run folio -- chromium --reporter=dot,json
shell: bash
env:
PW_CHROMIUM_CHANNEL: "chrome"
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
shell: bash
env:
@ -340,6 +378,10 @@ jobs:
- run: npm run build
# This only created problems, should we move ffmpeg back into npm?
- run: node lib/cli/cli install ffmpeg
- run: npm run folio -- chromium --reporter=dot,json
env:
PW_CHROMIUM_CHANNEL: "chrome"
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
env:
BROWSER: "chromium"
@ -368,6 +410,11 @@ jobs:
- run: npm run build
# This only created problems, should we move ffmpeg back into npm?
- run: node lib/cli/cli install ffmpeg
- run: npm run folio -- chromium --reporter=dot,json
shell: bash
env:
PW_CHROMIUM_CHANNEL: "msedge"
FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
- run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
shell: bash
env:

4
package.json
View File

@ -28,7 +28,9 @@
"check-deps": "node utils/check_deps.js",
"build-android-driver": "./utils/build_android_driver.sh",
"storybook": "start-storybook -p 6006 -s public",
"build-storybook": "build-storybook -s public"
"build-storybook": "build-storybook -s public",
"build-folio": "tsc -p ./tests/folio",
"folio": "npm run build-folio && node tests/folio/cli.js --config=tests/config/default.config.ts"
},
"author": {
"name": "Microsoft Corporation"

59
test/android/browser.spec.ts
View File

@ -1,59 +0,0 @@
/**
* Copyright 2020 Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { folio } from '../fixtures';
const { it, expect } = folio;
if (process.env.PW_ANDROID_TESTS) {
it('androidDevice.model', async function({ androidDevice }) {
expect(androidDevice.model()).toBe('sdk_gphone_x86_arm');
});
it('androidDevice.launchBrowser', async function({ androidDevice }) {
const context = await androidDevice.launchBrowser();
const [page] = context.pages();
await page.goto('data:text/html,<title>Hello world!</title>');
expect(await page.title()).toBe('Hello world!');
await context.close();
});
it('should create new page', async function({ androidDevice }) {
const context = await androidDevice.launchBrowser();
const page = await context.newPage();
await page.goto('data:text/html,<title>Hello world!</title>');
expect(await page.title()).toBe('Hello world!');
await page.close();
await context.close();
});
it('should check', async function({ androidDevice }) {
const context = await androidDevice.launchBrowser();
const [page] = context.pages();
await page.setContent(`<input id='checkbox' type='checkbox'></input>`);
await page.check('input');
expect(await page.evaluate(() => window['checkbox'].checked)).toBe(true);
await page.close();
await context.close();
});
it('should be able to send CDP messages', async ({ androidDevice }) => {
const context = await androidDevice.launchBrowser();
const [page] = context.pages();
const client = await context.newCDPSession(page);
await client.send('Runtime.enable');
const evalResponse = await client.send('Runtime.evaluate', {expression: '1 + 2', returnByValue: true});
expect(evalResponse.result.value).toBe(3);
});
}

63
test/android/device.spec.ts
View File

@ -1,63 +0,0 @@
/**
* Copyright 2020 Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import { PNG } from 'pngjs';
import { folio } from '../fixtures';
const { it, expect } = folio;
if (process.env.PW_ANDROID_TESTS) {
it('androidDevice.shell', async function({ androidDevice }) {
const output = await androidDevice.shell('echo 123');
expect(output.toString()).toBe('123\n');
});
it('androidDevice.open', async function({ androidDevice }) {
const socket = await androidDevice.open('shell:/bin/cat');
await socket.write(Buffer.from('321\n'));
const output = await new Promise(resolve => socket.on('data', resolve));
expect(output.toString()).toBe('321\n');
const closedPromise = new Promise(resolve => socket.on('close', resolve));
await socket.close();
await closedPromise;
});
it('androidDevice.screenshot', async function({ androidDevice, testInfo }) {
const path = testInfo.outputPath('screenshot.png');
const result = await androidDevice.screenshot({ path });
const buffer = fs.readFileSync(path);
expect(result.length).toBe(buffer.length);
const { width, height} = PNG.sync.read(result);
expect(width).toBe(1080);
expect(height).toBe(1920);
});
it('androidDevice.push', async function({ androidDevice }) {
await androidDevice.shell('rm /data/local/tmp/hello-world');
await androidDevice.push(Buffer.from('hello world'), '/data/local/tmp/hello-world');
const data = await androidDevice.shell('cat /data/local/tmp/hello-world');
expect(data).toEqual(Buffer.from('hello world'));
});
it('androidDevice.fill', test => {
test.fixme(!!process.env.CI, 'Hangs on the bots');
}, async function({ androidDevice }) {
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
await androidDevice.fill({ res: 'org.chromium.webview_shell:id/url_field' }, 'Hello');
expect((await androidDevice.info({ res: 'org.chromium.webview_shell:id/url_field' })).text).toBe('Hello');
});
}

61
test/android/webview.spec.ts
View File

@ -1,61 +0,0 @@
/**
* Copyright 2020 Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { folio } from '../fixtures';
const { it, expect } = folio;
if (process.env.PW_ANDROID_TESTS) {
it('androidDevice.webView', async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
expect(webview.pkg()).toBe('org.chromium.webview_shell');
expect(androidDevice.webViews().length).toBe(1);
});
it('webView.page', async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
const page = await webview.page();
expect(page.url()).toBe('about:blank');
});
it('should navigate page internally', async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
const page = await webview.page();
await page.goto('data:text/html,<title>Hello world!</title>');
expect(await page.title()).toBe('Hello world!');
});
it('should navigate page externally', test => {
test.fixme(!!process.env.CI, 'Hangs on the bots');
}, async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
const page = await webview.page();
await androidDevice.fill({ res: 'org.chromium.webview_shell:id/url_field' }, 'data:text/html,<title>Hello world!</title>');
await Promise.all([
page.waitForNavigation(),
androidDevice.press({ res: 'org.chromium.webview_shell:id/url_field' }, 'Enter')
]);
expect(await page.title()).toBe('Hello world!');
});
}

12
test/fixtures.spec.ts
View File

@ -17,9 +17,6 @@
import { folio } from './remoteServer.fixture';
import { execSync } from 'child_process';
import path from 'path';
import * as stackTrace from '../src/utils/stackTrace';
import { setUnderTest } from '../src/utils/utils';
import type { Browser } from '../index';
const { it, describe, expect, beforeEach, afterEach } = folio;
@ -146,12 +143,3 @@ describe('stalling signals', (suite, { platform, headful }) => {
expect(await stallingRemoteServer.childExitCode()).toBe(130);
});
});
it('caller file path', async ({}) => {
setUnderTest();
const callme = require('./fixtures/callback');
const filePath = callme(() => {
return stackTrace.getCallerFilePath(path.join(__dirname, 'fixtures') + path.sep);
});
expect(filePath).toBe(__filename);
});

57
tests/android/browser.spec.ts Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright 2020 Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from '../config/androidTest';
test('androidDevice.model', async function({ androidDevice }) {
expect(androidDevice.model()).toBe('sdk_gphone_x86_arm');
});
test('androidDevice.launchBrowser', async function({ androidDevice }) {
const context = await androidDevice.launchBrowser();
const [page] = context.pages();
await page.goto('data:text/html,<title>Hello world!</title>');
expect(await page.title()).toBe('Hello world!');
await context.close();
});
test('should create new page', async function({ androidDevice }) {
const context = await androidDevice.launchBrowser();
const page = await context.newPage();
await page.goto('data:text/html,<title>Hello world!</title>');
expect(await page.title()).toBe('Hello world!');
await page.close();
await context.close();
});
test('should check', async function({ androidDevice }) {
const context = await androidDevice.launchBrowser();
const [page] = context.pages();
await page.setContent(`<input id='checkbox' type='checkbox'></input>`);
await page.check('input');
expect(await page.evaluate(() => window['checkbox'].checked)).toBe(true);
await page.close();
await context.close();
});
test('should be able to send CDP messages', async ({ androidDevice }) => {
const context = await androidDevice.launchBrowser();
const [page] = context.pages();
const client = await context.newCDPSession(page);
await client.send('Runtime.enable');
const evalResponse = await client.send('Runtime.evaluate', {expression: '1 + 2', returnByValue: true});
expect(evalResponse.result.value).toBe(3);
});

59
tests/android/device.spec.ts Normal file
View File

@ -0,0 +1,59 @@
/**
* Copyright 2020 Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import { PNG } from 'pngjs';
import { test, expect } from '../config/androidTest';
test('androidDevice.shell', async function({ androidDevice }) {
const output = await androidDevice.shell('echo 123');
expect(output.toString()).toBe('123\n');
});
test('androidDevice.open', async function({ androidDevice }) {
const socket = await androidDevice.open('shell:/bin/cat');
await socket.write(Buffer.from('321\n'));
const output = await new Promise(resolve => socket.on('data', resolve));
expect(output.toString()).toBe('321\n');
const closedPromise = new Promise(resolve => socket.on('close', resolve));
await socket.close();
await closedPromise;
});
test('androidDevice.screenshot', async function({ androidDevice }, testInfo) {
const path = testInfo.outputPath('screenshot.png');
const result = await androidDevice.screenshot({ path });
const buffer = fs.readFileSync(path);
expect(result.length).toBe(buffer.length);
const { width, height} = PNG.sync.read(result);
expect(width).toBe(1080);
expect(height).toBe(1920);
});
test('androidDevice.push', async function({ androidDevice }) {
await androidDevice.shell('rm /data/local/tmp/hello-world');
await androidDevice.push(Buffer.from('hello world'), '/data/local/tmp/hello-world');
const data = await androidDevice.shell('cat /data/local/tmp/hello-world');
expect(data).toEqual(Buffer.from('hello world'));
});
test('androidDevice.fill', async function({ androidDevice }) {
test.fixme(!!process.env.CI, 'Hangs on the bots');
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
await androidDevice.fill({ res: 'org.chromium.webview_shell:id/url_field' }, 'Hello');
expect((await androidDevice.info({ res: 'org.chromium.webview_shell:id/url_field' })).text).toBe('Hello');
});

58
tests/android/webview.spec.ts Normal file
View File

@ -0,0 +1,58 @@
/**
* Copyright 2020 Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from '../config/androidTest';
test('androidDevice.webView', async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
expect(webview.pkg()).toBe('org.chromium.webview_shell');
expect(androidDevice.webViews().length).toBe(1);
});
test('webView.page', async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
const page = await webview.page();
expect(page.url()).toBe('about:blank');
});
test('should navigate page internally', async function({ androidDevice }) {
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
const page = await webview.page();
await page.goto('data:text/html,<title>Hello world!</title>');
expect(await page.title()).toBe('Hello world!');
});
test('should navigate page externally', async function({ androidDevice }) {
test.fixme(!!process.env.CI, 'Hangs on the bots');
expect(androidDevice.webViews().length).toBe(0);
await androidDevice.shell('am start org.chromium.webview_shell/.WebViewBrowserActivity');
const webview = await androidDevice.webView({ pkg: 'org.chromium.webview_shell' });
const page = await webview.page();
await androidDevice.fill({ res: 'org.chromium.webview_shell:id/url_field' }, 'data:text/html,<title>Hello world!</title>');
await Promise.all([
page.waitForNavigation(),
androidDevice.press({ res: 'org.chromium.webview_shell:id/url_field' }, 'Enter')
]);
expect(await page.title()).toBe('Hello world!');
});

0
test/fixtures/callback.js → tests/assets/callback.js
View File

8
test/browser.spec.ts → tests/browser.spec.ts
View File

@ -14,9 +14,9 @@
* limitations under the License.
*/
import { it, expect } from './fixtures';
import { test, expect } from './config/browserTest';
it('should create new page', async function({browser}) {
test('should create new page', async function({browser}) {
const page1 = await browser.newPage();
expect(browser.contexts().length).toBe(1);
@ -30,7 +30,7 @@ it('should create new page', async function({browser}) {
expect(browser.contexts().length).toBe(0);
});
it('should throw upon second create new page', async function({browser}) {
test('should throw upon second create new page', async function({browser}) {
const page = await browser.newPage();
let error;
await page.context().newPage().catch(e => error = e);
@ -38,7 +38,7 @@ it('should throw upon second create new page', async function({browser}) {
expect(error.message).toContain('Please use browser.newContext()');
});
it('version should work', async function({browser, isChromium}) {
test('version should work', async function({browser, isChromium}) {
const version = browser.version();
if (isChromium)
expect(version.match(/^\d+\.\d+\.\d+\.\d+$/)).toBeTruthy();

27
test/browsertype-basic.spec.ts → tests/browsertype-basic.spec.ts
View File

@ -16,30 +16,23 @@
*/
import fs from 'fs';
import { it, expect } from './fixtures';
import { test, expect } from './config/playwrightTest';
test('browserType.executablePath should work', async ({ browserType, browserChannel, browserOptions }) => {
test.skip(!!browserChannel, 'We skip browser download when testing a channel');
test.skip(!!browserOptions.executablePath, 'Skip with custom executable path');
it('browserType.executablePath should work', (test, { browserChannel }) => {
test.fixme(!!browserChannel, 'Uncomment on roll');
test.skip(Boolean(process.env.CRPATH || process.env.FFPATH || process.env.WKPATH));
}, async ({ browserType, browserChannel }) => {
// Interesting, unless I use browserChannel in test, filter above does not work!
const executablePath = browserType.executablePath();
expect(fs.existsSync(executablePath)).toBe(true);
});
it('browserType.name should work', async ({browserType, isChromium, isFirefox, isWebKit}) => {
if (isWebKit)
expect(browserType.name()).toBe('webkit');
else if (isFirefox)
expect(browserType.name()).toBe('firefox');
else if (isChromium)
expect(browserType.name()).toBe('chromium');
else
throw new Error('Unknown browser');
test('browserType.name should work', async ({browserType, browserName}) => {
expect(browserType.name()).toBe(browserName);
});
it('should throw when trying to connect with not-chromium', (test, {browserName}) => {
test('should throw when trying to connect with not-chromium', async ({ browserType, browserName }) => {
test.skip(browserName === 'chromium');
}, async ({browserType }) => {
const error = await browserType.connectOverCDP({wsEndpoint: 'foo'}).catch(e => e);
expect(error.message).toBe('Connecting over CDP is only supported in Chromium.');
});

38
tests/config/android.config.ts Normal file
View File

@ -0,0 +1,38 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { setConfig, Config } from '../folio/out';
import * as path from 'path';
import { test as pageTest } from './pageTest';
import { test as androidTest } from './androidTest';
import { ServerEnv } from './serverEnv';
import { AndroidEnv, AndroidPageEnv } from './androidEnv';
const config: Config = {
testDir: path.join(__dirname, '..'),
timeout: 120000,
globalTimeout: 5400000,
};
if (process.env.CI) {
config.workers = 1;
config.forbidOnly = true;
config.retries = 3;
}
setConfig(config);
const serverEnv = new ServerEnv();
pageTest.runWith('android', serverEnv, new AndroidPageEnv(), {});
androidTest.runWith('android', serverEnv, new AndroidEnv(), {});

81
tests/config/androidEnv.ts Normal file
View File

@ -0,0 +1,81 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Env, WorkerInfo, TestInfo } from '../folio/out';
import type { AndroidDevice, BrowserContext } from '../../index';
import * as os from 'os';
import { AndroidTestArgs } from './androidTest';
import { PageTestArgs } from './pageTest';
require('../../lib/utils/utils').setUnderTest();
const playwright: typeof import('../../index') = require('../../index');
export class AndroidEnv implements Env<AndroidTestArgs> {
protected _device?: AndroidDevice;
async beforeAll(workerInfo: WorkerInfo) {
this._device = (await playwright._android.devices())[0];
await this._device.shell('am force-stop org.chromium.webview_shell');
await this._device.shell('am force-stop com.android.chrome');
this._device.setDefaultTimeout(120000);
}
async beforeEach(testInfo: TestInfo) {
// Use chromium screenshots.
testInfo.snapshotPathSegment = 'chromium';
return {
mode: 'default' as const,
isChromium: true,
isFirefox: false,
isWebKit: false,
isWindows: os.platform() === 'win32',
isMac: os.platform() === 'darwin',
isLinux: os.platform() === 'linux',
platform: os.platform() as ('win32' | 'darwin' | 'linux'),
video: false,
toImpl: (playwright as any)._toImpl,
playwright,
androidDevice: this._device!,
};
}
async afterAll(workerInfo: WorkerInfo) {
if (this._device)
await this._device.close();
this._device = undefined;
}
}
export class AndroidPageEnv extends AndroidEnv implements Env<PageTestArgs> {
private _context?: BrowserContext;
async beforeAll(workerInfo: WorkerInfo) {
await super.beforeAll(workerInfo);
this._context = await this._device!.launchBrowser();
}
async beforeEach(testInfo: TestInfo) {
const result = await super.beforeEach(testInfo);
for (const page of this._context!.pages())
await page.close();
const page = await this._context!.newPage();
return {
...result,
androidDevice: undefined,
page,
};
}
}

27
tests/config/androidTest.ts Normal file
View File

@ -0,0 +1,27 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import type { AndroidDevice } from '../../index';
import type { CommonTestArgs } from './pageTest';
import type { ServerTestArgs } from './serverTest';
export { expect } from 'folio';
export type AndroidTestArgs = CommonTestArgs & {
androidDevice: AndroidDevice;
};
export const test = newTestType<AndroidTestArgs & ServerTestArgs>();

261
tests/config/browserEnv.ts Normal file
View File

@ -0,0 +1,261 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Env, WorkerInfo, TestInfo } from '../folio/out';
import type { Browser, BrowserContext, BrowserContextOptions, BrowserType, LaunchOptions } from '../../index';
import { installCoverageHooks } from '../../test/coverage';
import { start } from '../../lib/outofprocess';
import { PlaywrightClient } from '../../lib/remote/playwrightClient';
import { removeFolders } from '../../lib/utils/utils';
import * as path from 'path';
import * as fs from 'fs';
import * as os from 'os';
import * as util from 'util';
import * as childProcess from 'child_process';
import { PlaywrightTestArgs } from './playwrightTest';
import { BrowserTestArgs } from './browserTest';
const mkdtempAsync = util.promisify(fs.mkdtemp);
export type BrowserName = 'chromium' | 'firefox' | 'webkit';
type TestOptions = {
mode: 'default' | 'driver' | 'service';
video?: boolean;
trace?: boolean;
};
class DriverMode {
private _playwrightObject: any;
async setup(workerInfo: WorkerInfo) {
this._playwrightObject = await start();
return this._playwrightObject;
}
async teardown() {
await this._playwrightObject.stop();
}
}
class ServiceMode {
private _playwrightObejct: any;
private _client: any;
private _serviceProcess: childProcess.ChildProcess;
async setup(workerInfo: WorkerInfo) {
const port = 9407 + workerInfo.workerIndex * 2;
this._serviceProcess = childProcess.fork(path.join(__dirname, '..', '..', 'lib', 'service.js'), [String(port)], {
stdio: 'pipe'
});
this._serviceProcess.stderr.pipe(process.stderr);
await new Promise<void>(f => {
this._serviceProcess.stdout.on('data', data => {
if (data.toString().includes('Listening on'))
f();
});
});
this._serviceProcess.unref();
this._serviceProcess.on('exit', this._onExit);
this._client = await PlaywrightClient.connect(`ws://localhost:${port}/ws`);
this._playwrightObejct = this._client.playwright();
return this._playwrightObejct;
}
async teardown() {
await this._client.close();
this._serviceProcess.removeListener('exit', this._onExit);
const processExited = new Promise(f => this._serviceProcess.on('exit', f));
this._serviceProcess.kill();
await processExited;
}
private _onExit(exitCode, signal) {
throw new Error(`Server closed with exitCode=${exitCode} signal=${signal}`);
}
}
class DefaultMode {
async setup(workerInfo: WorkerInfo) {
return require('../../index');
}
async teardown() {
}
}
export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
private _mode: DriverMode | ServiceMode | DefaultMode;
private _browserName: BrowserName;
protected _options: LaunchOptions & TestOptions;
protected _browserOptions: LaunchOptions;
private _playwright: typeof import('../../index');
protected _browserType: BrowserType<Browser>;
private _coverage: ReturnType<typeof installCoverageHooks> | undefined;
private _userDataDirs: string[] = [];
private _persistentContext: BrowserContext | undefined;
constructor(browserName: BrowserName, options: LaunchOptions & TestOptions) {
this._browserName = browserName;
this._options = options;
this._mode = {
default: new DefaultMode(),
service: new ServiceMode(),
driver: new DriverMode(),
}[this._options.mode];
}
async beforeAll(workerInfo: WorkerInfo) {
this._coverage = installCoverageHooks(this._browserName);
require('../../lib/utils/utils').setUnderTest();
this._playwright = await this._mode.setup(workerInfo);
this._browserType = this._playwright[this._browserName];
this._browserOptions = {
...this._options,
handleSIGINT: false,
};
}
private async _createUserDataDir() {
// We do not put user data dir in testOutputPath,
// because we do not want to upload them as test result artifacts.
//
// Additionally, it is impossible to upload user data dir after test run:
// - Firefox removes lock file later, presumably from another watchdog process?
// - WebKit has circular symlinks that makes CI go crazy.
const dir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
this._userDataDirs.push(dir);
return dir;
}
private async _launchPersistent(options?: Parameters<BrowserType<Browser>['launchPersistentContext']>[1]) {
if (this._persistentContext)
throw new Error('can only launch one persitent context');
const userDataDir = await this._createUserDataDir();
this._persistentContext = await this._browserType.launchPersistentContext(userDataDir, { ...this._browserOptions, ...options });
const page = this._persistentContext.pages()[0];
return { context: this._persistentContext, page };
}
async beforeEach(testInfo: TestInfo) {
// Different screenshots per browser.
testInfo.snapshotPathSegment = this._browserName;
return {
playwright: this._playwright,
browserName: this._browserName,
browserType: this._browserType,
browserChannel: this._options.channel,
browserOptions: this._browserOptions,
isChromium: this._browserName === 'chromium',
isFirefox: this._browserName === 'firefox',
isWebKit: this._browserName === 'webkit',
isWindows: os.platform() === 'win32',
isMac: os.platform() === 'darwin',
isLinux: os.platform() === 'linux',
headful: !this._browserOptions.headless,
video: !!this._options.video,
mode: this._options.mode,
platform: os.platform() as ('win32' | 'darwin' | 'linux'),
createUserDataDir: this._createUserDataDir.bind(this),
launchPersistent: this._launchPersistent.bind(this),
toImpl: (this._playwright as any)._toImpl,
};
}
async afterEach(testInfo: TestInfo) {
await removeFolders(this._userDataDirs);
this._userDataDirs = [];
if (this._persistentContext) {
await this._persistentContext.close();
this._persistentContext = undefined;
}
}
async afterAll(workerInfo: WorkerInfo) {
const { coverage, uninstall } = this._coverage!;
uninstall();
const coveragePath = path.join(__dirname, '..', '..', 'test', 'coverage-report', workerInfo.workerIndex + '.json');
const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key));
await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
}
}
export class BrowserEnv extends PlaywrightEnv implements Env<BrowserTestArgs> {
private _browser: Browser | undefined;
private _contextOptions: BrowserContextOptions;
private _contexts: BrowserContext[] = [];
constructor(browserName: BrowserName, options: LaunchOptions & BrowserContextOptions & TestOptions) {
super(browserName, options);
this._contextOptions = options;
}
async beforeAll(workerInfo: WorkerInfo) {
await super.beforeAll(workerInfo);
this._browser = await this._browserType.launch(this._browserOptions);
}
async beforeEach(testInfo: TestInfo) {
const result = await super.beforeEach(testInfo);
const contextOptions = {
recordVideo: this._options.video ? { dir: testInfo.outputPath('') } : undefined,
_traceDir: this._options.trace ? testInfo.outputPath('') : undefined,
...this._contextOptions,
} as BrowserContextOptions;
const contextFactory = async (options: BrowserContextOptions = {}) => {
const context = await this._browser.newContext({ ...contextOptions, ...options });
this._contexts.push(context);
return context;
};
return {
...result,
browser: this._browser,
contextOptions: this._contextOptions as BrowserContextOptions,
contextFactory,
};
}
async afterEach(testInfo: TestInfo) {
for (const context of this._contexts)
await context.close();
this._contexts = [];
await super.afterEach(testInfo);
}
async afterAll(workerInfo: WorkerInfo) {
if (this._browser)
await this._browser.close();
this._browser = undefined;
await super.afterAll(workerInfo);
}
}
export class PageEnv extends BrowserEnv {
async beforeEach(testInfo: TestInfo) {
const result = await super.beforeEach(testInfo);
const context = await result.contextFactory();
const page = await context.newPage();
return {
...result,
context,
page,
};
}
}

28
tests/config/browserTest.ts Normal file
View File

@ -0,0 +1,28 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import type { Browser, BrowserContextOptions, BrowserContext } from '../../index';
import type { PlaywrightTestArgs } from './playwrightTest';
export { expect } from 'folio';
export type BrowserTestArgs = PlaywrightTestArgs & {
browser: Browser;
contextOptions: BrowserContextOptions;
contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;
};
export const test = newTestType<BrowserTestArgs>();

62
tests/config/default.config.ts Normal file
View File

@ -0,0 +1,62 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { setConfig, Config } from '../folio/out';
import * as path from 'path';
import { test as playwrightTest } from './playwrightTest';
import { test as browserTest } from './browserTest';
import { test as pageTest } from './pageTest';
import { PlaywrightEnv, BrowserEnv, PageEnv, BrowserName } from './browserEnv';
import { ServerEnv } from './serverEnv';
const config: Config = {
testDir: path.join(__dirname, '..'),
timeout: process.env.PWVIDEO ? 60000 : 30000,
globalTimeout: 5400000,
};
if (process.env.CI) {
config.workers = 1;
config.forbidOnly = true;
config.retries = 3;
}
setConfig(config);
const getExecutablePath = (browserName: BrowserName) => {
if (browserName === 'chromium' && process.env.CRPATH)
return process.env.CRPATH;
if (browserName === 'firefox' && process.env.FFPATH)
return process.env.FFPATH;
if (browserName === 'webkit' && process.env.WKPATH)
return process.env.WKPATH;
};
const serverEnv = new ServerEnv();
const browsers = ['chromium', 'webkit', 'firefox'] as BrowserName[];
for (const browserName of browsers) {
const executablePath = getExecutablePath(browserName);
const options = {
mode: (process.env.PWMODE || 'default') as ('default' | 'driver' | 'service'),
executablePath,
trace: !!process.env.PWTRACE,
headless: !process.env.HEADFUL,
channel: process.env.PW_CHROMIUM_CHANNEL as any,
video: !!process.env.PWVIDEO,
};
playwrightTest.runWith(browserName, serverEnv, new PlaywrightEnv(browserName, options), {});
browserTest.runWith(browserName, serverEnv, new BrowserEnv(browserName, options), {});
pageTest.runWith(browserName, serverEnv, new PageEnv(browserName, options), {});
}

42
tests/config/pageTest.ts Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import type { Page } from '../../index';
import type { ServerTestArgs } from './serverTest';
export { expect } from 'folio';
export type CommonTestArgs = {
mode: 'default' | 'driver' | 'service';
platform: 'win32' | 'darwin' | 'linux';
video: boolean;
playwright: typeof import('../../index');
toImpl: (rpcObject: any) => any;
isChromium: boolean;
isFirefox: boolean;
isWebKit: boolean;
isWindows: boolean;
isMac: boolean;
isLinux: boolean;
};
export type PageTestArgs = CommonTestArgs & {
page: Page;
};
export const test = newTestType<PageTestArgs & ServerTestArgs>();

33
tests/config/playwrightTest.ts Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { newTestType } from '../folio/out';
import type { Browser, BrowserType, LaunchOptions, BrowserContext, Page } from '../../index';
import { CommonTestArgs } from './pageTest';
import type { ServerTestArgs } from './serverTest';
export { expect } from 'folio';
export type PlaywrightTestArgs = CommonTestArgs & {
browserName: 'chromium' | 'firefox' | 'webkit';
browserType: BrowserType<Browser>;
browserChannel: string | undefined;
browserOptions: LaunchOptions;
headful: boolean;
createUserDataDir: () => Promise<string>;
launchPersistent: (options?: Parameters<BrowserType<Browser>['launchPersistentContext']>[1]) => Promise<{ context: BrowserContext, page: Page }>;
};
export const test = newTestType<PlaywrightTestArgs & ServerTestArgs>();

77
tests/config/serverEnv.ts Normal file
View File

@ -0,0 +1,77 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { WorkerInfo, TestInfo } from '../folio/out';
import { TestServer } from '../../utils/testserver';
import * as path from 'path';
import socks from 'socksv5';
export class ServerEnv {
private _server: TestServer;
private _httpsServer: TestServer;
private _socksServer: any;
async beforeAll(workerInfo: WorkerInfo) {
const assetsPath = path.join(__dirname, '..', 'assets');
const cachedPath = path.join(__dirname, '..', 'assets', 'cached');
const port = 8907 + workerInfo.workerIndex * 2;
this._server = await TestServer.create(assetsPath, port);
this._server.enableHTTPCache(cachedPath);
const httpsPort = port + 1;
this._httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort);
this._httpsServer.enableHTTPCache(cachedPath);
this._socksServer = socks.createServer((info, accept, deny) => {
let socket;
if ((socket = accept(true))) {
// Catch and ignore ECONNRESET errors.
socket.on('error', () => {});
const body = '<html><title>Served by the SOCKS proxy</title></html>';
socket.end([
'HTTP/1.1 200 OK',
'Connection: close',
'Content-Type: text/html',
'Content-Length: ' + Buffer.byteLength(body),
'',
body
].join('\r\n'));
}
});
const socksPort = 9107 + workerInfo.workerIndex * 2;
this._socksServer.listen(socksPort, 'localhost');
this._socksServer.useAuth(socks.auth.None());
}
async beforeEach(testInfo: TestInfo) {
this._server.reset();
this._httpsServer.reset();
return {
asset: (p: string) => path.join(__dirname, '..', 'assets', p),
server: this._server,
httpsServer: this._httpsServer,
};
}
async afterAll(workerInfo: WorkerInfo) {
await Promise.all([
this._server.stop(),
this._httpsServer.stop(),
this._socksServer.close(),
]);
}
}

24
tests/config/serverTest.ts Normal file
View File

@ -0,0 +1,24 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TestServer } from '../../utils/testserver';
export type ServerTestArgs = {
asset: (path: string) => string;
socksPort: number,
server: TestServer;
httpsServer: TestServer;
};

1
tests/folio/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
out/

19
tests/folio/cli.js Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env node
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('./out/cli');

278
tests/folio/src/cli.ts Normal file
View File

@ -0,0 +1,278 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { default as ignore } from 'fstream-ignore';
import * as commander from 'commander';
import * as fs from 'fs';
import * as path from 'path';
import EmptyReporter from './reporters/empty';
import DotReporter from './reporters/dot';
import JSONReporter from './reporters/json';
import JUnitReporter from './reporters/junit';
import LineReporter from './reporters/line';
import ListReporter from './reporters/list';
import { Multiplexer } from './reporters/multiplexer';
import { Runner } from './runner';
import { Config, FullConfig, Reporter } from './types';
import { Loader } from './loader';
import { createMatcher } from './util';
export const reporters: { [name: string]: new () => Reporter } = {
'dot': DotReporter,
'json': JSONReporter,
'junit': JUnitReporter,
'line': LineReporter,
'list': ListReporter,
'null': EmptyReporter,
};
const availableReporters = Object.keys(reporters).map(r => `"${r}"`).join();
const defaultConfig: FullConfig = {
forbidOnly: false,
globalTimeout: 0,
grep: /.*/,
maxFailures: 0,
outputDir: path.resolve(process.cwd(), 'test-results'),
quiet: false,
repeatEach: 1,
retries: 0,
shard: null,
snapshotDir: '__snapshots__',
testDir: path.resolve(process.cwd()),
testIgnore: 'node_modules/**',
testMatch: '**/?(*.)+(spec|test).[jt]s',
timeout: 10000,
updateSnapshots: false,
workers: Math.ceil(require('os').cpus().length / 2),
};
const loadProgram = new commander.Command();
loadProgram.helpOption(false);
addRunnerOptions(loadProgram);
loadProgram.action(async command => {
try {
await runTests(command);
} catch (e) {
console.log(e);
process.exit(1);
}
});
loadProgram.parse(process.argv);
async function runTests(command: any) {
if (command.help === undefined) {
console.log(loadProgram.helpInformation());
process.exit(0);
}
const reporterList: string[] = command.reporter.split(',');
const reporterObjects: Reporter[] = reporterList.map(c => {
if (reporters[c])
return new reporters[c]();
try {
const p = path.resolve(process.cwd(), c);
return new (require(p).default)();
} catch (e) {
console.error('Invalid reporter ' + c, e);
process.exit(1);
}
});
const loader = new Loader();
loader.addConfig(defaultConfig);
function loadConfig(configName: string) {
const configFile = path.resolve(process.cwd(), configName);
if (fs.existsSync(configFile)) {
loader.loadConfigFile(configFile);
return true;
}
return false;
}
if (command.config) {
if (!loadConfig(command.config))
throw new Error(`${command.config} does not exist`);
} else if (!loadConfig('folio.config.ts') && !loadConfig('folio.config.js')) {
throw new Error(`Configuration file not found. Either pass --config, or create folio.config.(js|ts) file`);
}
loader.addConfig(configFromCommand(command));
loader.addConfig({ testMatch: normalizeFilePatterns(loader.config().testMatch) });
loader.addConfig({ testIgnore: normalizeFilePatterns(loader.config().testIgnore) });
const testDir = loader.config().testDir;
if (!fs.existsSync(testDir))
throw new Error(`${testDir} does not exist`);
if (!fs.statSync(testDir).isDirectory())
throw new Error(`${testDir} is not a directory`);
const allAliases = new Set(loader.runLists().map(s => s.alias));
const runListFilter: string[] = [];
const testFileFilter: string[] = [];
for (const arg of command.args) {
if (allAliases.has(arg))
runListFilter.push(arg);
else
testFileFilter.push(arg);
}
const allFiles = await collectFiles(testDir);
const testFiles = filterFiles(testDir, allFiles, testFileFilter, createMatcher(loader.config().testMatch), createMatcher(loader.config().testIgnore));
for (const file of testFiles)
loader.loadTestFile(file);
const reporter = new Multiplexer(reporterObjects);
const runner = new Runner(loader, reporter, runListFilter.length ? runListFilter : undefined);
if (command.list) {
runner.list();
return;
}
const result = await runner.run();
if (result === 'sigint')
process.exit(130);
if (result === 'forbid-only') {
console.error('=====================================');
console.error(' --forbid-only found a focused test.');
console.error('=====================================');
process.exit(1);
}
if (result === 'no-tests') {
console.error('=================');
console.error(' no tests found.');
console.error('=================');
process.exit(1);
}
process.exit(result === 'failed' ? 1 : 0);
}
async function collectFiles(testDir: string): Promise<string[]> {
const entries: any[] = [];
let callback = () => {};
const promise = new Promise<void>(f => callback = f);
ignore({ path: testDir, ignoreFiles: ['.gitignore'] })
.on('child', (entry: any) => entries.push(entry))
.on('end', callback);
await promise;
return entries.filter(e => e.type === 'File').sort((a, b) => {
if (a.depth !== b.depth && (a.dirname.startsWith(b.dirname) || b.dirname.startsWith(a.dirname)))
return a.depth - b.depth;
return a.path > b.path ? 1 : (a.path < b.path ? -1 : 0);
}).map(e => e.path);
}
function filterFiles(base: string, files: string[], filters: string[], filesMatch: (value: string) => boolean, filesIgnore: (value: string) => boolean): string[] {
return files.filter(file => {
file = path.relative(base, file);
if (filesIgnore(file))
return false;
if (!filesMatch(file))
return false;
if (filters.length && !filters.find(filter => file.includes(filter)))
return false;
return true;
});
}
function addRunnerOptions(program: commander.Command) {
program = program
.version('Version alpha')
.option('-c, --config <file>', `Configuration file (default: "folio.config.ts" or "folio.config.js")`)
.option('--forbid-only', `Fail if exclusive test(s) encountered (default: ${defaultConfig.forbidOnly})`)
.option('-g, --grep <grep>', `Only run tests matching this string or regexp (default: "${defaultConfig.grep}")`)
.option('--global-timeout <timeout>', `Specify maximum time this test suite can run in milliseconds (default: 0 for unlimited)`)
.option('-h, --help', `Display help`)
.option('-j, --workers <workers>', `Number of concurrent workers, use 1 to run in single worker (default: number of CPU cores / 2)`)
.option('--list', `Only collect all the test and report them`)
.option('--max-failures <N>', `Stop after the first N failures (default: ${defaultConfig.maxFailures})`)
.option('--output <dir>', `Folder for output artifacts (default: "test-results")`)
.option('--quiet', `Suppress stdio`)
.option('--repeat-each <repeat-each>', `Specify how many times to run the tests (default: ${defaultConfig.repeatEach})`)
.option('--reporter <reporter>', `Specify reporter to use, comma-separated, can be ${availableReporters}`, process.env.CI ? 'dot' : 'line')
.option('--retries <retries>', `Specify retry count (default: ${defaultConfig.retries})`)
.option('--shard <shard>', `Shard tests and execute only selected shard, specify in the form "current/all", 1-based, for example "3/5"`)
.option('--snapshot-dir <dir>', `Snapshot directory, relative to tests directory (default: "${defaultConfig.snapshotDir}"`)
.option('--test-dir <dir>', `Directory containing test files (default: current directory)`)
.option('--test-ignore <pattern>', `Pattern used to ignore test files (default: "${defaultConfig.testIgnore}")`)
.option('--test-match <pattern>', `Pattern used to find test files (default: "${defaultConfig.testMatch}")`)
.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds (default: ${defaultConfig.timeout})`)
.option('-u, --update-snapshots', `Whether to update snapshots with actual results (default: ${defaultConfig.updateSnapshots})`)
.option('-x', `Stop after the first failure`);
}
function configFromCommand(command: any): Config {
const config: Config = {};
if (command.forbidOnly)
config.forbidOnly = true;
if (command.globalTimeout)
config.globalTimeout = parseInt(command.globalTimeout, 10);
if (command.grep)
config.grep = maybeRegExp(command.grep);
if (command.maxFailures || command.x)
config.maxFailures = command.x ? 1 : parseInt(command.maxFailures, 10);
if (command.output)
config.outputDir = path.resolve(process.cwd(), command.output);
if (command.quiet)
config.quiet = command.quiet;
if (command.repeatEach)
config.repeatEach = parseInt(command.repeatEach, 10);
if (command.retries)
config.retries = parseInt(command.retries, 10);
if (command.shard) {
const pair = command.shard.split('/').map((t: string) => parseInt(t, 10));
config.shard = { current: pair[0] - 1, total: pair[1] };
}
if (command.snapshotDir)
config.snapshotDir = command.snapshotDir;
if (command.testDir)
config.testDir = path.resolve(process.cwd(), command.testDir);
if (command.testMatch)
config.testMatch = maybeRegExp(command.testMatch);
if (command.testIgnore)
config.testIgnore = maybeRegExp(command.testIgnore);
if (command.timeout)
config.timeout = parseInt(command.timeout, 10);
if (command.updateSnapshots)
config.updateSnapshots = !!command.updateSnapshots;
if (command.workers)
config.workers = parseInt(command.workers, 10);
return config;
}
function normalizeFilePattern(pattern: string): string {
if (!pattern.includes('/') && !pattern.includes('\\'))
pattern = '**/' + pattern;
return pattern;
}
function normalizeFilePatterns(patterns: string | RegExp | (string | RegExp)[]) {
if (typeof patterns === 'string')
patterns = normalizeFilePattern(patterns);
else if (Array.isArray(patterns))
patterns = patterns.map(item => typeof item === 'string' ? normalizeFilePattern(item) : item);
return patterns;
}
function maybeRegExp(pattern: string): string | RegExp {
const match = pattern.match(/^\/(.*)\/([gi]*)$/);
if (match)
return new RegExp(match[1], match[2]);
return pattern;
}

387
tests/folio/src/dispatcher.ts Normal file
View File

@ -0,0 +1,387 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import child_process from 'child_process';
import path from 'path';
import { EventEmitter } from 'events';
import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, TestStatus, WorkerInitParams } from './ipc';
import { TestResult, Reporter } from './types';
import { Suite, Test } from './test';
import { Loader } from './loader';
type DispatcherEntry = {
runPayload: RunPayload;
hash: string;
repeatEachIndex: number;
runListIndex: number;
};
export class Dispatcher {
private _workers = new Set<Worker>();
private _freeWorkers: Worker[] = [];
private _workerClaimers: (() => void)[] = [];
private _testById = new Map<string, { test: Test, result: TestResult }>();
private _queue: DispatcherEntry[] = [];
private _stopCallback: () => void;
readonly _loader: Loader;
private _suite: Suite;
private _reporter: Reporter;
private _hasWorkerErrors = false;
private _isStopped = false;
private _failureCount = 0;
private _didRunGlobalSetup = false;
_globalSetupResult: any = undefined;
constructor(loader: Loader, suite: Suite, reporter: Reporter) {
this._loader = loader;
this._reporter = reporter;
this._suite = suite;
for (const suite of this._suite.suites) {
for (const spec of suite._allSpecs()) {
for (const test of spec.tests)
this._testById.set(test._id, { test, result: test._appendTestResult() });
}
}
this._queue = this._filesSortedByWorkerHash();
// Shard tests.
let total = this._suite.totalTestCount();
let shardDetails = '';
const shard = this._loader.config().shard;
if (shard) {
total = 0;
const shardSize = Math.ceil(total / shard.total);
const from = shardSize * shard.current;
const to = shardSize * (shard.current + 1);
shardDetails = `, shard ${shard.current + 1} of ${shard.total}`;
let current = 0;
const filteredQueue: DispatcherEntry[] = [];
for (const entry of this._queue) {
if (current >= from && current < to) {
filteredQueue.push(entry);
total += entry.runPayload.entries.length;
}
current += entry.runPayload.entries.length;
}
this._queue = filteredQueue;
}
if (process.stdout.isTTY) {
const workers = new Set<string>();
suite.findSpec(test => {
for (const variant of test.tests)
workers.add(test.file + variant._workerHash);
});
console.log();
const jobs = Math.min(this._loader.config().workers, workers.size);
console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`);
}
}
_filesSortedByWorkerHash(): DispatcherEntry[] {
const entriesByWorkerHashAndFile = new Map<string, Map<string, DispatcherEntry>>();
for (const suite of this._suite.suites) {
for (const spec of suite._allSpecs()) {
for (const test of spec.tests) {
let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
if (!entriesByFile) {
entriesByFile = new Map();
entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
}
let entry = entriesByFile.get(spec.file);
if (!entry) {
entry = {
runPayload: {
entries: [],
file: spec.file,
},
repeatEachIndex: test._repeatEachIndex,
runListIndex: test._runListIndex,
hash: test._workerHash,
};
entriesByFile.set(spec.file, entry);
}
entry.runPayload.entries.push({
retry: this._testById.get(test._id).result.retry,
testId: test._id,
});
}
}
}
const result: DispatcherEntry[] = [];
for (const entriesByFile of entriesByWorkerHashAndFile.values()) {
for (const entry of entriesByFile.values())
result.push(entry);
}
result.sort((a, b) => a.hash < b.hash ? -1 : (a.hash === b.hash ? 0 : 1));
return result;
}
async run() {
if (this._loader.globalSetup) {
this._didRunGlobalSetup = true;
this._globalSetupResult = await this._loader.globalSetup();
}
// Loop in case job schedules more jobs
while (this._queue.length && !this._isStopped)
await this._dispatchQueue();
}
async _dispatchQueue() {
const jobs = [];
while (this._queue.length) {
if (this._isStopped)
break;
const entry = this._queue.shift();
const requiredHash = entry.hash;
let worker = await this._obtainWorker(entry);
while (!this._isStopped && worker.hash && worker.hash !== requiredHash) {
worker.stop();
worker = await this._obtainWorker(entry);
}
if (this._isStopped)
break;
jobs.push(this._runJob(worker, entry));
}
await Promise.all(jobs);
}
async _runJob(worker: Worker, entry: DispatcherEntry) {
worker.run(entry.runPayload);
let doneCallback;
const result = new Promise(f => doneCallback = f);
worker.once('done', (params: DonePayload) => {
// We won't file remaining if:
// - there are no remaining
// - we are here not because something failed
// - no unrecoverable worker error
if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
this._freeWorkers.push(worker);
this._notifyWorkerClaimer();
doneCallback();
return;
}
// When worker encounters error, we will stop it and create a new one.
worker.stop();
// In case of fatal error, we are done with the entry.
if (params.fatalError) {
// Report all the tests are failing with this error.
for (const { testId } of entry.runPayload.entries) {
const { test, result } = this._testById.get(testId);
this._reporter.onTestBegin(test);
result.error = params.fatalError;
this._reportTestEnd(test, result, 'failed');
}
doneCallback();
return;
}
const remaining = params.remaining;
// Only retry expected failures, not passes and only if the test failed.
if (this._loader.config().retries && params.failedTestId) {
const pair = this._testById.get(params.failedTestId);
if (pair.test.expectedStatus === 'passed' && pair.test.results.length < this._loader.config().retries + 1) {
pair.result = pair.test._appendTestResult();
remaining.unshift({
retry: pair.result.retry,
testId: pair.test._id,
});
}
}
if (remaining.length)
this._queue.unshift({ ...entry, runPayload: { ...entry.runPayload, entries: remaining } });
// This job is over, we just scheduled another one.
doneCallback();
});
return result;
}
async _obtainWorker(entry: DispatcherEntry) {
const claimWorker = (): Promise<Worker> | null => {
// Use available worker.
if (this._freeWorkers.length)
return Promise.resolve(this._freeWorkers.pop());
// Create a new worker.
if (this._workers.size < this._loader.config().workers)
return this._createWorker(entry);
return null;
};
// Note: it is important to claim the worker synchronously,
// so that we won't miss a _notifyWorkerClaimer call while awaiting.
let worker = claimWorker();
if (!worker) {
// Wait for available or stopped worker.
await new Promise<void>(f => this._workerClaimers.push(f));
worker = claimWorker();
}
return worker;
}
async _notifyWorkerClaimer() {
if (this._isStopped || !this._workerClaimers.length)
return;
const callback = this._workerClaimers.shift();
callback();
}
_createWorker(entry: DispatcherEntry) {
const worker = new Worker(this);
worker.on('testBegin', (params: TestBeginPayload) => {
const { test, result: testRun } = this._testById.get(params.testId);
testRun.workerIndex = params.workerIndex;
this._reporter.onTestBegin(test);
});
worker.on('testEnd', (params: TestEndPayload) => {
const { test, result } = this._testById.get(params.testId);
result.data = params.data;
result.duration = params.duration;
result.error = params.error;
test.expectedStatus = params.expectedStatus;
test.annotations = params.annotations;
test.timeout = params.timeout;
if (params.expectedStatus === 'skipped')
test.skipped = true;
this._reportTestEnd(test, result, params.status);
});
worker.on('stdOut', (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
const pair = this._testById.get(params.testId);
if (pair)
pair.result.stdout.push(chunk);
this._reporter.onStdOut(chunk, pair ? pair.test : undefined);
});
worker.on('stdErr', (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
const pair = this._testById.get(params.testId);
if (pair)
pair.result.stderr.push(chunk);
this._reporter.onStdErr(chunk, pair ? pair.test : undefined);
});
worker.on('teardownError', ({error}) => {
this._hasWorkerErrors = true;
this._reporter.onError(error);
});
worker.on('exit', () => {
this._workers.delete(worker);
this._notifyWorkerClaimer();
if (this._stopCallback && !this._workers.size)
this._stopCallback();
});
this._workers.add(worker);
return worker.init(entry).then(() => worker);
}
async stop() {
this._isStopped = true;
if (this._workers.size) {
const result = new Promise<void>(f => this._stopCallback = f);
for (const worker of this._workers)
worker.stop();
await result;
}
if (this._didRunGlobalSetup && this._loader.globalTeardown)
await this._loader.globalTeardown(this._globalSetupResult);
}
private _reportTestEnd(test: Test, result: TestResult, status: TestStatus) {
if (this._isStopped)
return;
result.status = status;
if (result.status !== 'skipped' && result.status !== test.expectedStatus)
++this._failureCount;
const maxFailures = this._loader.config().maxFailures;
if (!maxFailures || this._failureCount <= maxFailures)
this._reporter.onTestEnd(test, result);
if (maxFailures && this._failureCount === maxFailures)
this._isStopped = true;
}
hasWorkerErrors(): boolean {
return this._hasWorkerErrors;
}
}
let lastWorkerIndex = 0;
class Worker extends EventEmitter {
process: child_process.ChildProcess;
runner: Dispatcher;
hash: string;
index: number;
stdout: any[];
stderr: any[];
constructor(runner: Dispatcher) {
super();
this.runner = runner;
this.index = lastWorkerIndex++;
this.process = child_process.fork(path.join(__dirname, 'worker.js'), {
detached: false,
env: {
FORCE_COLOR: process.stdout.isTTY ? '1' : '0',
DEBUG_COLORS: process.stdout.isTTY ? '1' : '0',
...process.env
},
// Can't pipe since piping slows down termination for some reason.
stdio: ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc']
});
this.process.on('exit', () => this.emit('exit'));
this.process.on('error', e => {}); // do not yell at a send to dead process.
this.process.on('message', (message: any) => {
const { method, params } = message;
this.emit(method, params);
});
}
async init(entry: DispatcherEntry) {
this.hash = entry.hash;
const params: WorkerInitParams = {
workerIndex: this.index,
repeatEachIndex: entry.repeatEachIndex,
runListIndex: entry.runListIndex,
globalSetupResult: this.runner._globalSetupResult,
loader: this.runner._loader.serialize(),
};
this.process.send({ method: 'init', params });
await new Promise(f => this.process.once('message', f)); // Ready ack
}
run(runPayload: RunPayload) {
this.process.send({ method: 'run', params: runPayload });
}
stop() {
this.process.send({ method: 'stop' });
}
}
function chunkFromParams(params: TestOutputPayload): string | Buffer {
if (typeof params.text === 'string')
return params.text;
return Buffer.from(params.buffer, 'base64');
}

53
tests/folio/src/expect.ts Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Expect } from './expectType';
import expectLibrary from 'expect';
import { currentTestInfo } from './globals';
import { compare } from './golden';
export const expect: Expect = expectLibrary;
const snapshotOrdinalSymbol = Symbol('snapshotOrdinalSymbol');
function toMatchSnapshot(received: Buffer | string, nameOrOptions?: string | { name?: string, threshold?: number }, optOptions: { threshold?: number } = {}) {
let options: { name?: string, threshold?: number };
const testInfo = currentTestInfo();
if (typeof nameOrOptions === 'string')
options = { name: nameOrOptions, ...optOptions };
else
options = { ...nameOrOptions };
let name = options.name;
if (!name) {
const ordinal = (testInfo as any)[snapshotOrdinalSymbol] || 0;
(testInfo as any)[snapshotOrdinalSymbol] = ordinal + 1;
let extension: string;
if (typeof received === 'string')
extension = '.txt';
else if (received[0] === 0x89 && received[1] === 0x50 && received[2] === 0x4E && received[3] === 0x47)
extension = '.png';
else if (received[0] === 0xFF && received[1] === 0xD8 && received[2] === 0xFF)
extension = '.jpeg';
else
extension = '.dat';
name = 'snapshot' + (ordinal ? '_' + ordinal : '') + extension;
}
const { pass, message } = compare(received, name, testInfo.snapshotPath, testInfo.outputPath, testInfo.config.updateSnapshots, options);
return { pass, message: () => message };
}
expectLibrary.extend({ toMatchSnapshot });

150
tests/folio/src/expectType.ts Normal file
View File

@ -0,0 +1,150 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
*/
export declare type AsymmetricMatcher = Record<string, any>;
export declare type Expect = {
<T = unknown>(actual: T): Matchers<T>;
[id: string]: AsymmetricMatcher;
not: {
[id: string]: AsymmetricMatcher;
};
};
export interface Matchers<R> {
/**
* If you know how to test something, `.not` lets you test its opposite.
*/
not: Matchers<R>;
/**
* Use resolves to unwrap the value of a fulfilled promise so any other
* matcher can be chained. If the promise is rejected the assertion fails.
*/
resolves: Matchers<Promise<R>>;
/**
* Unwraps the reason of a rejected promise so any other matcher can be chained.
* If the promise is fulfilled the assertion fails.
*/
rejects: Matchers<Promise<R>>;
/**
* Checks that a value is what you expect. It uses `===` to check strict equality.
* Don't use `toBe` with floating-point numbers.
*/
toBe(expected: unknown): R;
/**
* Using exact equality with floating point numbers is a bad idea.
* Rounding means that intuitive things fail.
* The default for numDigits is 2.
*/
toBeCloseTo(expected: number, numDigits?: number): R;
/**
* Ensure that a variable is not undefined.
*/
toBeDefined(): R;
/**
* When you don't care what a value is, you just want to
* ensure a value is false in a boolean context.
*/
toBeFalsy(): R;
/**
* For comparing floating point numbers.
*/
toBeGreaterThan(expected: number | bigint): R;
/**
* For comparing floating point numbers.
*/
toBeGreaterThanOrEqual(expected: number | bigint): R;
/**
* Ensure that an object is an instance of a class.
* This matcher uses `instanceof` underneath.
*/
toBeInstanceOf(expected: Function): R;
/**
* For comparing floating point numbers.
*/
toBeLessThan(expected: number | bigint): R;
/**
* For comparing floating point numbers.
*/
toBeLessThanOrEqual(expected: number | bigint): R;
/**
* This is the same as `.toBe(null)` but the error messages are a bit nicer.
* So use `.toBeNull()` when you want to check that something is null.
*/
toBeNull(): R;
/**
* Use when you don't care what a value is, you just want to ensure a value
* is true in a boolean context. In JavaScript, there are six falsy values:
* `false`, `0`, `''`, `null`, `undefined`, and `NaN`. Everything else is truthy.
*/
toBeTruthy(): R;
/**
* Used to check that a variable is undefined.
*/
toBeUndefined(): R;
/**
* Used to check that a variable is NaN.
*/
toBeNaN(): R;
/**
* Used when you want to check that an item is in a list.
* For testing the items in the list, this uses `===`, a strict equality check.
*/
toContain(expected: unknown): R;
/**
* Used when you want to check that an item is in a list.
* For testing the items in the list, this matcher recursively checks the
* equality of all fields, rather than checking for object identity.
*/
toContainEqual(expected: unknown): R;
/**
* Used when you want to check that two objects have the same value.
* This matcher recursively checks the equality of all fields, rather than checking for object identity.
*/
toEqual(expected: unknown): R;
/**
* Use to check if property at provided reference keyPath exists for an object.
* For checking deeply nested properties in an object you may use dot notation or an array containing
* the keyPath for deep references.
*
* Optionally, you can provide a value to check if it's equal to the value present at keyPath
* on the target object. This matcher uses 'deep equality' (like `toEqual()`) and recursively checks
* the equality of all fields.
*
* @example
*
* expect(houseForSale).toHaveProperty('kitchen.area', 20);
*/
toHaveProperty(keyPath: string | Array<string>, value?: unknown): R;
/**
* Check that a string matches a regular expression.
*/
toMatch(expected: string | RegExp): R;
/**
* Used to check that a JavaScript object matches a subset of the properties of an object
*/
toMatchObject(expected: Record<string, unknown> | Array<unknown>): R;
/**
* Use to test that objects have the same types as well as structure.
*/
toStrictEqual(expected: unknown): R;
/**
* Match snapshot
*/
toMatchSnapshot(options?: {
name?: string,
threshold?: number
}): R;
/**
* Match snapshot
*/
toMatchSnapshot(name: string, options?: {
threshold?: number
}): R;
}
export {};

25
tests/folio/src/globals.ts Normal file
View File

@ -0,0 +1,25 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TestInfo } from './types';
let currentTestInfoValue: TestInfo | null = null;
export function setCurrentTestInfo(testInfo: TestInfo | null) {
currentTestInfoValue = testInfo;
}
export function currentTestInfo(): TestInfo | null {
return currentTestInfoValue;
}

171
tests/folio/src/golden.ts Normal file
View File

@ -0,0 +1,171 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import fs from 'fs';
import path from 'path';
import jpeg from 'jpeg-js';
import pixelmatch from 'pixelmatch';
import { PNG } from 'pngjs';
import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
const extensionToMimeType: { [key: string]: string } = {
'dat': 'application/octet-string',
'jpeg': 'image/jpeg',
'jpg': 'image/jpeg',
'png': 'image/png',
'txt': 'text/plain',
};
const GoldenComparators: { [key: string]: any } = {
'application/octet-string': compareBuffers,
'image/png': compareImages,
'image/jpeg': compareImages,
'text/plain': compareText,
};
function compareBuffers(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string): { diff?: object; errorMessage?: string; } | null {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be Buffer.' };
if (Buffer.compare(actualBuffer, expectedBuffer))
return { errorMessage: 'Buffers differ' };
return null;
}
function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: object; errorMessage?: string; } | null {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be Buffer.' };
const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpeg.decode(actualBuffer);
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
if (expected.width !== actual.width || expected.height !== actual.height) {
return {
errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
};
}
const diff = new PNG({width: expected.width, height: expected.height});
const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options });
return count > 0 ? { diff: PNG.sync.write(diff) } : null;
}
function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: object; errorMessage?: string; diffExtension?: string; } | null {
if (typeof actual !== 'string')
return { errorMessage: 'Actual result should be string' };
const expected = expectedBuffer.toString('utf-8');
if (expected === actual)
return null;
const dmp = new diff_match_patch();
const d = dmp.diff_main(expected, actual);
dmp.diff_cleanupSemantic(d);
return {
errorMessage: diff_prettyTerminal(d)
};
}
export function compare(actual: Buffer | string, name: string, snapshotPath: (name: string) => string, outputPath: (name: string) => string, updateSnapshots: boolean, options?: { threshold?: number }): { pass: boolean; message?: string; } {
const snapshotFile = snapshotPath(name);
if (!fs.existsSync(snapshotFile)) {
fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
fs.writeFileSync(snapshotFile, actual);
return {
pass: false,
message: snapshotFile + ' is missing in golden results, writing actual.'
};
}
const expected = fs.readFileSync(snapshotFile);
const extension = path.extname(snapshotFile).substring(1);
const mimeType = extensionToMimeType[extension] || 'application/octet-string';
const comparator = GoldenComparators[mimeType];
if (!comparator) {
return {
pass: false,
message: 'Failed to find comparator with type ' + mimeType + ': ' + snapshotFile,
};
}
const result = comparator(actual, expected, mimeType, options);
if (!result)
return { pass: true };
if (updateSnapshots) {
fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
fs.writeFileSync(snapshotFile, actual);
console.log('Updating snapshot at ' + snapshotFile);
return {
pass: true,
message: snapshotFile + ' running with --p-update-snapshots, writing actual.'
};
}
const outputFile = outputPath(name);
const expectedPath = addSuffix(outputFile, '-expected');
const actualPath = addSuffix(outputFile, '-actual');
const diffPath = addSuffix(outputFile, '-diff');
fs.writeFileSync(expectedPath, expected);
fs.writeFileSync(actualPath, actual);
if (result.diff)
fs.writeFileSync(diffPath, result.diff);
const output = [
colors.red(`Snapshot comparison failed:`),
];
if (result.errorMessage) {
output.push('');
output.push(indent(result.errorMessage, ' '));
}
output.push('');
output.push(`Expected: ${colors.yellow(expectedPath)}`);
output.push(`Received: ${colors.yellow(actualPath)}`);
if (result.diff)
output.push(` Diff: ${colors.yellow(diffPath)}`);
return {
pass: false,
message: output.join('\n'),
};
}
function indent(lines: string, tab: string) {
return lines.replace(/^(?=.+$)/gm, tab);
}
function addSuffix(filePath: string, suffix: string, customExtension?: string): string {
const dirname = path.dirname(filePath);
const ext = path.extname(filePath);
const name = path.basename(filePath, ext);
return path.join(dirname, name + suffix + (customExtension || ext));
}
function diff_prettyTerminal(diffs) {
const html = [];
for (let x = 0; x < diffs.length; x++) {
const op = diffs[x][0]; // Operation (insert, delete, equal)
const data = diffs[x][1]; // Text of change.
const text = data;
switch (op) {
case DIFF_INSERT:
html[x] = colors.green(text);
break;
case DIFF_DELETE:
html[x] = colors.strikethrough(colors.red(text));
break;
case DIFF_EQUAL:
html[x] = text;
break;
}
}
return html.join('');
}

27
tests/folio/src/index.ts Normal file
View File

@ -0,0 +1,27 @@
/**
* Copyright 2019 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { TestType } from './types';
import { newTestTypeImpl } from './spec';
export * from './types';
export { expect } from './expect';
export { setConfig, globalSetup, globalTeardown } from './spec';
export function newTestType<TestArgs = {}, TestOptions = {}>(): TestType<TestArgs, TestOptions> {
return newTestTypeImpl();
}

66
tests/folio/src/ipc.ts Normal file
View File

@ -0,0 +1,66 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Config, TestStatus, TestError } from './types';
export type { TestStatus } from './types';
export type WorkerInitParams = {
workerIndex: number;
repeatEachIndex: number;
runListIndex: number;
globalSetupResult: any;
loader: {
configs: (string | Config)[];
};
};
export type TestBeginPayload = {
testId: string;
workerIndex: number,
};
export type TestEndPayload = {
testId: string;
duration: number;
status: TestStatus;
error?: TestError;
data: any;
expectedStatus: TestStatus;
annotations: any[];
timeout: number;
};
export type TestEntry = {
testId: string;
retry: number;
};
export type RunPayload = {
file: string;
entries: TestEntry[];
};
export type DonePayload = {
failedTestId?: string;
fatalError?: any;
remaining: TestEntry[];
};
export type TestOutputPayload = {
testId?: string;
text?: string;
buffer?: string;
};

94
tests/folio/src/loader.ts Normal file
View File

@ -0,0 +1,94 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { installTransform } from './transform';
import { Config, FullConfig } from './types';
import { prependErrorMessage } from './util';
import { configFile, setCurrentFile, RunListDescription } from './spec';
type SerializedLoaderData = {
configs: (string | Config)[];
};
export class Loader {
globalSetup?: () => any;
globalTeardown?: (globalSetupResult: any) => any;
private _mergedConfig: FullConfig;
private _layeredConfigs: { config: Config, source?: string }[] = [];
constructor() {
this._mergedConfig = {} as any;
}
deserialize(data: SerializedLoaderData) {
for (const config of data.configs) {
if (typeof config === 'string')
this.loadConfigFile(config);
else
this.addConfig(config);
}
}
loadConfigFile(file: string) {
const revertBabelRequire = installTransform();
try {
require(file);
this.addConfig(configFile.config || {});
this._layeredConfigs[this._layeredConfigs.length - 1].source = file;
this.globalSetup = configFile.globalSetup;
this.globalTeardown = configFile.globalTeardown;
} catch (e) {
// Drop the stack.
throw new Error(e.message);
} finally {
revertBabelRequire();
}
}
addConfig(config: Config) {
this._layeredConfigs.push({ config });
this._mergedConfig = { ...this._mergedConfig, ...config };
}
loadTestFile(file: string) {
const revertBabelRequire = installTransform();
setCurrentFile(file);
try {
require(file);
} catch (e) {
prependErrorMessage(e, `Error while reading ${file}:\n`);
throw e;
} finally {
setCurrentFile();
revertBabelRequire();
}
}
config(): FullConfig {
return this._mergedConfig;
}
runLists(): RunListDescription[] {
return configFile.runLists;
}
serialize(): SerializedLoaderData {
return {
configs: this._layeredConfigs.map(c => c.source || c.config),
};
}
}

246
tests/folio/src/reporters/base.ts Normal file
View File

@ -0,0 +1,246 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { codeFrameColumns } from '@babel/code-frame';
import colors from 'colors/safe';
import fs from 'fs';
import milliseconds from 'ms';
import path from 'path';
import StackUtils from 'stack-utils';
import { TestStatus, Test, Suite, TestResult, TestError, Reporter } from '../types';
import { FullConfig } from '../types';
const stackUtils = new StackUtils();
export class BaseReporter implements Reporter {
duration = 0;
config: FullConfig;
suite: Suite;
timeout: number;
fileDurations = new Map<string, number>();
monotonicStartTime: number;
constructor() {
}
onBegin(config: FullConfig, suite: Suite) {
this.monotonicStartTime = monotonicTime();
this.config = config;
this.suite = suite;
}
onTestBegin(test: Test) {
}
onStdOut(chunk: string | Buffer) {
if (!this.config.quiet)
process.stdout.write(chunk);
}
onStdErr(chunk: string | Buffer) {
if (!this.config.quiet)
process.stderr.write(chunk);
}
onTestEnd(test: Test, result: TestResult) {
const spec = test.spec;
let duration = this.fileDurations.get(spec.file) || 0;
duration += result.duration;
this.fileDurations.set(spec.file, duration);
}
onError(error: TestError) {
console.log(formatError(error));
}
onTimeout(timeout: number) {
this.timeout = timeout;
}
onEnd() {
this.duration = monotonicTime() - this.monotonicStartTime;
}
private _printSlowTests() {
const fileDurations = [...this.fileDurations.entries()];
fileDurations.sort((a, b) => b[1] - a[1]);
let insertedGap = false;
for (let i = 0; i < 10 && i < fileDurations.length; ++i) {
const baseName = path.basename(fileDurations[i][0]);
const duration = fileDurations[i][1];
if (duration < 15000)
break;
if (!insertedGap) {
insertedGap = true;
console.log();
}
console.log(colors.yellow(' Slow test: ') + baseName + colors.yellow(` (${milliseconds(duration)})`));
}
console.log();
}
epilogue(full: boolean) {
let skipped = 0;
let expected = 0;
const unexpected: Test[] = [];
const flaky: Test[] = [];
this.suite.findTest(test => {
switch (test.status()) {
case 'skipped': ++skipped; break;
case 'expected': ++expected; break;
case 'unexpected': unexpected.push(test); break;
case 'flaky': flaky.push(test); break;
}
});
if (expected)
console.log(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
if (skipped)
console.log(colors.yellow(` ${skipped} skipped`));
if (unexpected.length) {
console.log(colors.red(` ${unexpected.length} failed`));
this._printTestHeaders(unexpected);
}
if (flaky.length) {
console.log(colors.red(` ${flaky.length} flaky`));
this._printTestHeaders(flaky);
}
if (this.timeout)
console.log(colors.red(` Timed out waiting ${this.timeout / 1000}s for the entire test run`));
if (full && unexpected.length) {
console.log('');
this._printFailures(unexpected);
}
this._printSlowTests();
}
private _printTestHeaders(tests: Test[]) {
tests.forEach(test => {
console.log(formatTestHeader(this.config, test, ' '));
});
}
private _printFailures(failures: Test[]) {
failures.forEach((test, index) => {
console.log(formatFailure(this.config, test, index + 1));
});
}
hasResultWithStatus(test: Test, status: TestStatus): boolean {
return !!test.results.find(r => r.status === status);
}
willRetry(test: Test, result: TestResult): boolean {
return result.status !== 'passed' && result.status !== test.expectedStatus && test.results.length <= this.config.retries;
}
}
export function formatFailure(config: FullConfig, test: Test, index?: number): string {
const tokens: string[] = [];
tokens.push(formatTestHeader(config, test, ' ', index));
for (const result of test.results) {
if (result.status === 'passed')
continue;
tokens.push(formatResult(test, result));
}
tokens.push('');
return tokens.join('\n');
}
function formatTestHeader(config: FullConfig, test: Test, indent: string, index?: number): string {
const tokens: string[] = [];
const spec = test.spec;
let relativePath = path.relative(config.testDir, spec.file) || path.basename(spec.file);
relativePath += ':' + spec.line + ':' + spec.column;
const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
const runListName = test.alias ? `[${test.alias}] ` : '';
const header = `${indent}${index ? index + ') ' : ''}${relativePath} › ${runListName}${spec.fullTitle()}${passedUnexpectedlySuffix}`;
tokens.push(colors.red(pad(header, '=')));
return tokens.join('\n');
}
function formatResult(test: Test, result: TestResult): string {
const tokens: string[] = [];
if (result.retry)
tokens.push(colors.gray(pad(`\n Retry #${result.retry}`, '-')));
if (result.status === 'timedOut') {
tokens.push('');
tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), ' '));
} else {
tokens.push(indent(formatError(result.error, test.spec.file), ' '));
}
return tokens.join('\n');
}
function formatError(error: TestError, file?: string) {
const stack = error.stack;
const tokens = [];
if (stack) {
tokens.push('');
const messageLocation = error.stack.indexOf(error.message);
const preamble = error.stack.substring(0, messageLocation + error.message.length);
tokens.push(preamble);
const position = file ? positionInFile(stack, file) : null;
if (position) {
const source = fs.readFileSync(file, 'utf8');
tokens.push('');
tokens.push(codeFrameColumns(source, {
start: position,
},
{ highlightCode: true}
));
}
tokens.push('');
tokens.push(colors.dim(stack.substring(preamble.length + 1)));
} else {
tokens.push('');
tokens.push(error.value);
}
return tokens.join('\n');
}
function pad(line: string, char: string): string {
return line + ' ' + colors.gray(char.repeat(Math.max(0, 100 - line.length - 1)));
}
function indent(lines: string, tab: string) {
return lines.replace(/^(?=.+$)/gm, tab);
}
function positionInFile(stack: string, file: string): { column: number; line: number; } {
// Stack will have /private/var/folders instead of /var/folders on Mac.
file = fs.realpathSync(file);
for (const line of stack.split('\n')) {
const parsed = stackUtils.parseLine(line);
if (!parsed)
continue;
if (path.resolve(process.cwd(), parsed.file) === file)
return {column: parsed.column, line: parsed.line};
}
return null;
}
function monotonicTime(): number {
const [seconds, nanoseconds] = process.hrtime();
return seconds * 1000 + (nanoseconds / 1000000 | 0);
}
const asciiRegex = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', 'g');
export function stripAscii(str: string): string {
return str.replace(asciiRegex, '');
}

57
tests/folio/src/reporters/dot.ts Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import { BaseReporter } from './base';
import { Test, TestResult } from '../types';
class DotReporter extends BaseReporter {
private _counter = 0;
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
if (++this._counter === 81) {
process.stdout.write('\n');
return;
}
if (result.status === 'skipped') {
process.stdout.write(colors.yellow('°'));
return;
}
if (this.willRetry(test, result)) {
process.stdout.write(colors.gray('×'));
return;
}
switch (test.status()) {
case 'expected': process.stdout.write(colors.green('·')); break;
case 'unexpected': process.stdout.write(colors.red(test.results[test.results.length - 1].status === 'timedOut' ? 'T' : 'F')); break;
case 'flaky': process.stdout.write(colors.yellow('±')); break;
}
}
onTimeout(timeout: number) {
super.onTimeout(timeout);
this.onEnd();
}
onEnd() {
super.onEnd();
process.stdout.write('\n');
this.epilogue(true);
}
}
export default DotReporter;

30
tests/folio/src/reporters/empty.ts Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FullConfig, TestResult, Test, Suite, TestError, Reporter } from '../types';
class EmptyReporter implements Reporter {
onBegin(config: FullConfig, suite: Suite) {}
onTestBegin(test: Test) {}
onStdOut(chunk: string | Buffer, test?: Test) {}
onStdErr(chunk: string | Buffer, test?: Test) {}
onTestEnd(test: Test, result: TestResult) {}
onTimeout(timeout: number) {}
onError(error: TestError) {}
onEnd() {}
}
export default EmptyReporter;

136
tests/folio/src/reporters/json.ts Normal file
View File

@ -0,0 +1,136 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import EmptyReporter from './empty';
import { FullConfig, Test, Suite, Spec, TestResult, TestError } from '../types';
export interface SerializedSuite {
title: string;
file: string;
column: number;
line: number;
specs: ReturnType<JSONReporter['_serializeTestSpec']>[];
suites?: SerializedSuite[];
}
export type ReportFormat = {
config: FullConfig;
errors?: TestError[];
suites?: SerializedSuite[];
};
function toPosixPath(aPath: string): string {
return aPath.split(path.sep).join(path.posix.sep);
}
class JSONReporter extends EmptyReporter {
config: FullConfig;
suite: Suite;
private _errors: TestError[] = [];
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
}
onTimeout() {
this.onEnd();
}
onError(error: TestError): void {
this._errors.push(error);
}
onEnd() {
outputReport({
config: {
...this.config,
outputDir: toPosixPath(this.config.outputDir),
testDir: toPosixPath(this.config.testDir),
},
suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s),
errors: this._errors
});
}
private _serializeSuite(suite: Suite): null | SerializedSuite {
if (!suite.findSpec(test => true))
return null;
const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
return {
title: suite.title,
file: toPosixPath(path.relative(this.config.testDir, suite.file)),
line: suite.line,
column: suite.column,
specs: suite.specs.map(test => this._serializeTestSpec(test)),
suites: suites.length ? suites : undefined,
};
}
private _serializeTestSpec(spec: Spec) {
return {
title: spec.title,
ok: spec.ok(),
tests: spec.tests.map(r => this._serializeTest(r)),
file: toPosixPath(path.relative(this.config.testDir, spec.file)),
line: spec.line,
column: spec.column,
};
}
private _serializeTest(test: Test) {
return {
timeout: test.timeout,
annotations: test.annotations,
expectedStatus: test.expectedStatus,
results: test.results.map(r => this._serializeTestResult(r))
};
}
private _serializeTestResult(result: TestResult) {
return {
workerIndex: result.workerIndex,
status: result.status,
duration: result.duration,
error: result.error,
stdout: result.stdout.map(s => stdioEntry(s)),
stderr: result.stderr.map(s => stdioEntry(s)),
data: result.data,
retry: result.retry,
};
}
}
function outputReport(report: ReportFormat) {
const reportString = JSON.stringify(report, undefined, 2);
const outputName = process.env[`FOLIO_JSON_OUTPUT_NAME`];
if (outputName) {
fs.mkdirSync(path.dirname(outputName), { recursive: true });
fs.writeFileSync(outputName, reportString);
} else {
console.log(reportString);
}
}
function stdioEntry(s: string | Buffer): any {
if (typeof s === 'string')
return { text: s };
return { buffer: s.toString('base64') };
}
export default JSONReporter;

185
tests/folio/src/reporters/junit.ts Normal file
View File

@ -0,0 +1,185 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import { FullConfig } from '../types';
import EmptyReporter from './empty';
import { Suite, Test } from '../test';
import { monotonicTime } from '../util';
import { formatFailure, stripAscii } from './base';
class JUnitReporter extends EmptyReporter {
private config: FullConfig;
private suite: Suite;
private timestamp: number;
private startTime: number;
private totalTests = 0;
private totalFailures = 0;
private totalSkipped = 0;
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
this.timestamp = Date.now();
this.startTime = monotonicTime();
}
onEnd() {
const duration = monotonicTime() - this.startTime;
const children: XMLEntry[] = [];
for (const suite of this.suite.suites)
children.push(this._buildTestSuite(suite));
const tokens: string[] = [];
const self = this;
const root: XMLEntry = {
name: 'testsuites',
attributes: {
id: process.env[`FOLIO_JUNIT_SUITE_ID`] || '',
name: process.env[`FOLIO_JUNIT_SUITE_NAME`] || '',
tests: self.totalTests,
failures: self.totalFailures,
skipped: self.totalSkipped,
errors: 0,
time: duration / 1000
},
children
};
serializeXML(root, tokens);
const reportString = tokens.join('\n');
const outputName = process.env[`FOLIO_JUNIT_OUTPUT_NAME`];
if (outputName) {
fs.mkdirSync(path.dirname(outputName), { recursive: true });
fs.writeFileSync(outputName, reportString);
} else {
console.log(reportString);
}
}
private _buildTestSuite(suite: Suite): XMLEntry {
let tests = 0;
let skipped = 0;
let failures = 0;
let duration = 0;
const children: XMLEntry[] = [];
suite.findTest(test => {
++tests;
if (test.skipped)
++skipped;
if (!test.ok())
++failures;
for (const result of test.results)
duration += result.duration;
this._addTestCase(test, children);
});
this.totalTests += tests;
this.totalSkipped += skipped;
this.totalFailures += failures;
const entry: XMLEntry = {
name: 'testsuite',
attributes: {
name: path.relative(this.config.testDir, suite.file),
timestamp: this.timestamp,
hostname: '',
tests,
failures,
skipped,
time: duration / 1000,
errors: 0,
},
children
};
return entry;
}
private _addTestCase(test: Test, entries: XMLEntry[]) {
const entry = {
name: 'testcase',
attributes: {
name: test.spec.fullTitle(),
classname: path.relative(this.config.testDir, test.spec.file) + ' ' + test.spec.parent.fullTitle(),
time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
},
children: []
};
entries.push(entry);
if (test.skipped) {
entry.children.push({ name: 'skipped'});
return;
}
if (!test.ok()) {
entry.children.push({
name: 'failure',
attributes: {
message: `${path.basename(test.spec.file)}:${test.spec.line}:${test.spec.column} ${test.spec.title}`,
type: 'FAILURE',
},
text: stripAscii(formatFailure(this.config, test))
});
}
for (const result of test.results) {
for (const stdout of result.stdout) {
entries.push({
name: 'system-out',
text: stdout.toString()
});
}
for (const stderr of result.stderr) {
entries.push({
name: 'system-err',
text: stderr.toString()
});
}
}
}
}
type XMLEntry = {
name: string;
attributes?: { [name: string]: string | number | boolean };
children?: XMLEntry[];
text?: string;
};
function serializeXML(entry: XMLEntry, tokens: string[]) {
const attrs: string[] = [];
for (const name of Object.keys(entry.attributes || {}))
attrs.push(`${name}="${escape(String(entry.attributes[name]))}"`);
tokens.push(`<${entry.name}${attrs.length ? ' ' : ''}${attrs.join(' ')}>`);
for (const child of entry.children || [])
serializeXML(child, tokens);
if (entry.text)
tokens.push(escape(entry.text));
tokens.push(`</${entry.name}>`);
}
function escape(text: string): string {
text = text.replace(/"/g, '&quot;');
text = text.replace(/&/g, '&amp;');
text = text.replace(/</g, '&lt;');
text = text.replace(/>/g, '&gt;');
return text;
}
export default JUnitReporter;

81
tests/folio/src/reporters/line.ts Normal file
View File

@ -0,0 +1,81 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import * as path from 'path';
import { BaseReporter, formatFailure } from './base';
import { FullConfig, Test, Suite, TestResult } from '../types';
class LineReporter extends BaseReporter {
private _total: number;
private _current = 0;
private _failures = 0;
private _lastTest: Test;
onBegin(config: FullConfig, suite: Suite) {
super.onBegin(config, suite);
this._total = suite.totalTestCount();
console.log();
}
onStdOut(chunk: string | Buffer, test?: Test) {
this._dumpToStdio(test, chunk, process.stdout);
}
onStdErr(chunk: string | Buffer, test?: Test) {
this._dumpToStdio(test, chunk, process.stderr);
}
private _fullTitle(test: Test) {
const baseName = path.basename(test.spec.file);
const runListName = test.alias ? `[${test.alias}] ` : '';
return `${baseName} - ${runListName}${test.spec.fullTitle()}`;
}
private _dumpToStdio(test: Test | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
if (this.config.quiet)
return;
stream.write(`\u001B[1A\u001B[2K`);
if (test && this._lastTest !== test) {
// Write new header for the output.
stream.write(colors.gray(this._fullTitle(test) + `\n`));
this._lastTest = test;
}
stream.write(chunk);
console.log();
}
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
const width = process.stdout.columns - 1;
const title = `[${++this._current}/${this._total}] ${this._fullTitle(test)}`.substring(0, width);
process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
if (!this.willRetry(test, result) && !test.ok()) {
process.stdout.write(`\u001B[1A\u001B[2K`);
console.log(formatFailure(this.config, test, ++this._failures));
console.log();
}
}
onEnd() {
process.stdout.write(`\u001B[1A\u001B[2K`);
super.onEnd();
this.epilogue(false);
}
}
export default LineReporter;

79
tests/folio/src/reporters/list.ts Normal file
View File

@ -0,0 +1,79 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import milliseconds from 'ms';
import { BaseReporter } from './base';
import { FullConfig, Suite, Test, TestResult } from '../types';
class ListReporter extends BaseReporter {
private _failure = 0;
private _lastRow = 0;
private _testRows = new Map<Test, number>();
onBegin(config: FullConfig, suite: Suite) {
super.onBegin(config, suite);
console.log();
}
onTestBegin(test: Test) {
super.onTestBegin(test);
if (process.stdout.isTTY)
process.stdout.write(' ' + colors.gray(test.spec.fullTitle() + ': ') + '\n');
this._testRows.set(test, this._lastRow++);
}
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
const spec = test.spec;
const duration = colors.dim(` (${milliseconds(result.duration)})`);
let text = '';
if (result.status === 'skipped') {
text = colors.green(' - ') + colors.cyan(spec.fullTitle());
} else {
const statusMark = result.status === 'passed' ? ' ✓ ' : ' x ';
if (result.status === test.expectedStatus)
text = '\u001b[2K\u001b[0G' + colors.green(statusMark) + colors.gray(spec.fullTitle()) + duration;
else
text = '\u001b[2K\u001b[0G' + colors.red(`${statusMark}${++this._failure}) ` + spec.fullTitle()) + duration;
}
const testRow = this._testRows.get(test);
// Go up if needed
if (process.stdout.isTTY && testRow !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - testRow}A`);
// Erase line
if (process.stdout.isTTY)
process.stdout.write('\u001B[2K');
process.stdout.write(text);
// Go down if needed.
if (testRow !== this._lastRow) {
if (process.stdout.isTTY)
process.stdout.write(`\u001B[${this._lastRow - testRow}E`);
else
process.stdout.write('\n');
}
}
onEnd() {
super.onEnd();
process.stdout.write('\n');
this.epilogue(true);
}
}
export default ListReporter;

65
tests/folio/src/reporters/multiplexer.ts Normal file
View File

@ -0,0 +1,65 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FullConfig, Suite, Test, TestError, TestResult, Reporter } from '../types';
export class Multiplexer implements Reporter {
private _reporters: Reporter[];
constructor(reporters: Reporter[]) {
this._reporters = reporters;
}
onBegin(config: FullConfig, suite: Suite) {
for (const reporter of this._reporters)
reporter.onBegin(config, suite);
}
onTestBegin(test: Test) {
for (const reporter of this._reporters)
reporter.onTestBegin(test);
}
onStdOut(chunk: string | Buffer, test?: Test) {
for (const reporter of this._reporters)
reporter.onStdOut(chunk, test);
}
onStdErr(chunk: string | Buffer, test?: Test) {
for (const reporter of this._reporters)
reporter.onStdErr(chunk, test);
}
onTestEnd(test: Test, result: TestResult) {
for (const reporter of this._reporters)
reporter.onTestEnd(test, result);
}
onTimeout(timeout: number) {
for (const reporter of this._reporters)
reporter.onTimeout(timeout);
}
onEnd() {
for (const reporter of this._reporters)
reporter.onEnd();
}
onError(error: TestError) {
for (const reporter of this._reporters)
reporter.onError(error);
}
}

132
tests/folio/src/runner.ts Normal file
View File

@ -0,0 +1,132 @@
/**
* Copyright 2019 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import rimraf from 'rimraf';
import { promisify } from 'util';
import { Dispatcher } from './dispatcher';
import { Reporter } from './types';
import { createMatcher, monotonicTime, raceAgainstDeadline } from './util';
import { Suite } from './test';
import { Loader } from './loader';
const removeFolderAsync = promisify(rimraf);
type RunResult = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'no-tests';
export class Runner {
private _reporter: Reporter;
private _loader: Loader;
private _rootSuite: Suite;
constructor(loader: Loader, reporter: Reporter, runListFilter?: string[]) {
this._reporter = reporter;
this._loader = loader;
// This makes sure we don't generate 1000000 tests if only one spec is focused.
const filtered = new Set<Suite>();
for (const { fileSuites } of loader.runLists()) {
for (const fileSuite of fileSuites.values()) {
if (fileSuite._hasOnly())
filtered.add(fileSuite);
}
}
this._rootSuite = new Suite('');
const grepMatcher = createMatcher(loader.config().grep);
const nonEmptySuites = new Set<Suite>();
for (let runListIndex = 0; runListIndex < loader.runLists().length; runListIndex++) {
const runList = loader.runLists()[runListIndex];
if (runListFilter && !runListFilter.includes(runList.alias))
continue;
for (const fileSuite of runList.fileSuites.values()) {
if (filtered.size && !filtered.has(fileSuite))
continue;
const specs = fileSuite._allSpecs().filter(spec => grepMatcher(spec.fullTitle()));
if (!specs.length)
continue;
fileSuite._renumber();
for (const spec of specs) {
for (let i = 0; i < loader.config().repeatEach; ++i)
spec._appendTest(runListIndex, runList.alias, i);
}
nonEmptySuites.add(fileSuite);
}
}
for (const fileSuite of nonEmptySuites)
this._rootSuite._addSuite(fileSuite);
filterOnly(this._rootSuite);
}
list() {
this._reporter.onBegin(this._loader.config(), this._rootSuite);
this._reporter.onEnd();
}
async run(): Promise<RunResult> {
await removeFolderAsync(this._loader.config().outputDir).catch(e => {});
if (this._loader.config().forbidOnly) {
const hasOnly = this._rootSuite.findSpec(t => t._only) || this._rootSuite.findSuite(s => s._only);
if (hasOnly)
return 'forbid-only';
}
const total = this._rootSuite.totalTestCount();
if (!total)
return 'no-tests';
const globalDeadline = this._loader.config().globalTimeout ? this._loader.config().globalTimeout + monotonicTime() : 0;
const { result, timedOut } = await raceAgainstDeadline(this._runTests(this._rootSuite), globalDeadline);
if (timedOut) {
this._reporter.onTimeout(this._loader.config().globalTimeout);
process.exit(1);
}
return result;
}
private async _runTests(suite: Suite): Promise<RunResult> {
const dispatcher = new Dispatcher(this._loader, suite, this._reporter);
let sigint = false;
let sigintCallback: () => void;
const sigIntPromise = new Promise<void>(f => sigintCallback = f);
const sigintHandler = () => {
process.off('SIGINT', sigintHandler);
sigint = true;
sigintCallback();
};
process.on('SIGINT', sigintHandler);
this._reporter.onBegin(this._loader.config(), suite);
await Promise.race([dispatcher.run(), sigIntPromise]);
await dispatcher.stop();
this._reporter.onEnd();
if (sigint)
return 'sigint';
return dispatcher.hasWorkerErrors() || suite.findSpec(spec => !spec.ok()) ? 'failed' : 'passed';
}
}
function filterOnly(suite: Suite) {
const onlySuites = suite.suites.filter(child => filterOnly(child) || child._only);
const onlyTests = suite.specs.filter(spec => spec._only);
if (onlySuites.length || onlyTests.length) {
suite.suites = onlySuites;
suite.specs = onlyTests;
return true;
}
return false;
}

218
tests/folio/src/spec.ts Normal file
View File

@ -0,0 +1,218 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { expect } from './expect';
import { currentTestInfo } from './globals';
import { Spec, Suite } from './test';
import { callLocation, errorWithCallLocation, interpretCondition } from './util';
import { Config, Env, RunWithConfig, TestInfo, TestType, WorkerInfo } from './types';
Error.stackTraceLimit = 15;
let currentFile: string | undefined;
export function setCurrentFile(file?: string) {
currentFile = file;
}
export type RunListDescription = {
alias: string;
fileSuites: Map<string, Suite>;
env: Env<any>;
config: RunWithConfig;
testType: TestType<any, any>;
};
export const configFile: {
config?: Config,
globalSetup?: () => any,
globalTeardown?: (globalSetupResult: any) => any,
runLists: RunListDescription[]
} = { runLists: [] };
function mergeEnvs(envs: any[]): any {
if (envs.length === 1)
return envs[0];
const forward = [...envs];
const backward = [...forward].reverse();
return {
beforeAll: async (workerInfo: WorkerInfo) => {
for (const env of forward) {
if (env.beforeAll)
await env.beforeAll(workerInfo);
}
},
afterAll: async (workerInfo: WorkerInfo) => {
for (const env of backward) {
if (env.afterAll)
await env.afterAll(workerInfo);
}
},
beforeEach: async (testInfo: TestInfo) => {
let result = undefined;
for (const env of forward) {
if (env.beforeEach) {
const r = await env.beforeEach(testInfo);
result = result === undefined ? r : { ...result, ...r };
}
}
return result;
},
afterEach: async (testInfo: TestInfo) => {
for (const env of backward) {
if (env.afterEach)
await env.afterEach(testInfo);
}
},
};
}
export function newTestTypeImpl(): any {
const fileSuites = new Map<string, Suite>();
let suites: Suite[] = [];
function ensureSuiteForCurrentLocation() {
const location = callLocation(currentFile);
let fileSuite = fileSuites.get(location.file);
if (!fileSuite) {
fileSuite = new Suite('');
fileSuite.file = location.file;
fileSuites.set(location.file, fileSuite);
}
if (suites[suites.length - 1] !== fileSuite)
suites = [fileSuite];
return location;
}
function spec(type: 'default' | 'only', title: string, options: Function | any, fn?: Function) {
if (!currentFile)
throw errorWithCallLocation(`Test can only be defined in a test file.`);
const location = ensureSuiteForCurrentLocation();
if (typeof fn !== 'function') {
fn = options;
options = {};
}
const spec = new Spec(title, fn, suites[0]);
spec.file = location.file;
spec.line = location.line;
spec.column = location.column;
spec.testOptions = options;
if (type === 'only')
spec._only = true;
}
function describe(type: 'default' | 'only', title: string, fn: Function) {
if (!currentFile)
throw errorWithCallLocation(`Suite can only be defined in a test file.`);
const location = ensureSuiteForCurrentLocation();
const child = new Suite(title, suites[0]);
child.file = location.file;
child.line = location.line;
child.column = location.column;
if (type === 'only')
child._only = true;
suites.unshift(child);
fn();
suites.shift();
}
function hook(name: string, fn: Function) {
if (!currentFile)
throw errorWithCallLocation(`Hook can only be defined in a test file.`);
suites[0]._addHook(name, fn);
}
const modifier = (type: 'skip' | 'fail' | 'fixme', arg?: boolean | string, description?: string) => {
const processed = interpretCondition(arg, description);
if (!processed.condition)
return;
if (currentFile) {
suites[0]._annotations.push({ type, description: processed.description });
return;
}
const testInfo = currentTestInfo();
if (!testInfo)
throw new Error(`test.${type} can only be called inside the test`);
testInfo.annotations.push({ type, description: processed.description });
if (type === 'skip' || type === 'fixme') {
testInfo.expectedStatus = 'skipped';
throw new SkipError(processed.description);
} else if (type === 'fail') {
if (testInfo.expectedStatus !== 'skipped')
testInfo.expectedStatus = 'failed';
}
};
const test: any = spec.bind(null, 'default');
test.expect = expect;
test.only = spec.bind(null, 'only');
test.describe = describe.bind(null, 'default');
test.describe.only = describe.bind(null, 'only');
test.beforeEach = hook.bind(null, 'beforeEach');
test.afterEach = hook.bind(null, 'afterEach');
test.beforeAll = hook.bind(null, 'beforeAll');
test.afterAll = hook.bind(null, 'afterAll');
test.skip = modifier.bind(null, 'skip');
test.fixme = modifier.bind(null, 'fixme');
test.fail = modifier.bind(null, 'fail');
test.runWith = (...envs: any[]) => {
let alias = '';
if (typeof envs[0] === 'string') {
alias = envs[0];
envs = envs.slice(1);
}
let options = envs[envs.length - 1];
if (!envs.length || options.beforeAll || options.beforeEach || options.afterAll || options.afterEach)
options = {};
else
envs = envs.slice(0, envs.length - 1);
configFile.runLists.push({
fileSuites,
env: mergeEnvs(envs),
alias,
config: { timeout: options.timeout },
testType: test,
});
};
return test;
}
export class SkipError extends Error {
}
export function setConfig(config: Config) {
// TODO: add config validation.
configFile.config = config;
}
export function globalSetup(globalSetupFunction: () => any) {
if (typeof globalSetupFunction !== 'function')
throw errorWithCallLocation(`globalSetup takes a single function argument.`);
configFile.globalSetup = globalSetupFunction;
}
export function globalTeardown(globalTeardownFunction: (globalSetupResult: any) => any) {
if (typeof globalTeardownFunction !== 'function')
throw errorWithCallLocation(`globalTeardown takes a single function argument.`);
configFile.globalTeardown = globalTeardownFunction;
}

237
tests/folio/src/test.ts Normal file
View File

@ -0,0 +1,237 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as types from './types';
class Base {
title: string;
file: string;
line: number;
column: number;
parent?: Suite;
_only = false;
_ordinal: number;
constructor(title: string, parent?: Suite) {
this.title = title;
this.parent = parent;
}
titlePath(): string[] {
if (!this.parent)
return [];
if (!this.title)
return this.parent.titlePath();
return [...this.parent.titlePath(), this.title];
}
fullTitle(): string {
return this.titlePath().join(' ');
}
}
export class Spec extends Base implements types.Spec {
fn: Function;
tests: Test[] = [];
testOptions: any = {};
constructor(title: string, fn: Function, suite: Suite) {
super(title, suite);
this.fn = fn;
suite._addSpec(this);
}
ok(): boolean {
return !this.tests.find(r => !r.ok());
}
_appendTest(runListIndex: number, runListAlias: string, repeatEachIndex: number) {
const test = new Test(this);
test.alias = runListAlias;
test._runListIndex = runListIndex;
test._workerHash = `${runListIndex}#repeat-${repeatEachIndex}`;
test._id = `${this._ordinal}@${this.file}::[${test._workerHash}]`;
test._repeatEachIndex = repeatEachIndex;
this.tests.push(test);
return test;
}
}
export class Suite extends Base implements types.Suite {
suites: Suite[] = [];
specs: Spec[] = [];
_entries: (Suite | Spec)[] = [];
_hooks: { type: string, fn: Function } [] = [];
_annotations: { type: 'skip' | 'fixme' | 'fail', description?: string }[] = [];
constructor(title: string, parent?: Suite) {
super(title, parent);
if (parent)
parent._addSuite(this);
}
_clear() {
this.suites = [];
this.specs = [];
this._entries = [];
this._hooks = [];
this._annotations = [];
}
_addSpec(spec: Spec) {
spec.parent = this;
this.specs.push(spec);
this._entries.push(spec);
}
_addSuite(suite: Suite) {
suite.parent = this;
this.suites.push(suite);
this._entries.push(suite);
}
findTest(fn: (test: Test) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.findTest(fn))
return true;
}
for (const spec of this.specs) {
for (const test of spec.tests) {
if (fn(test))
return true;
}
}
return false;
}
findSpec(fn: (spec: Spec) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.findSpec(fn))
return true;
}
for (const spec of this.specs) {
if (fn(spec))
return true;
}
return false;
}
findSuite(fn: (suite: Suite) => boolean | void): boolean {
if (fn(this))
return true;
for (const suite of this.suites) {
if (suite.findSuite(fn))
return true;
}
return false;
}
totalTestCount(): number {
let total = 0;
for (const suite of this.suites)
total += suite.totalTestCount();
for (const spec of this.specs)
total += spec.tests.length;
return total;
}
_allSpecs(): Spec[] {
const result: Spec[] = [];
this.findSpec(test => { result.push(test); });
return result;
}
_renumber() {
// All tests are identified with their ordinals.
let ordinal = 0;
this.findSpec((test: Spec) => {
test._ordinal = ordinal++;
});
}
_hasOnly(): boolean {
if (this._only)
return true;
if (this.suites.find(suite => suite._hasOnly()))
return true;
if (this.specs.find(spec => spec._only))
return true;
}
_addHook(type: string, fn: any) {
this._hooks.push({ type, fn });
}
}
export class Test implements types.Test {
spec: Spec;
results: types.TestResult[] = [];
skipped = false;
expectedStatus: types.TestStatus = 'passed';
timeout = 0;
annotations: any[] = [];
alias = '';
_id: string;
_workerHash: string;
_repeatEachIndex: number;
_runListIndex: number;
constructor(spec: Spec) {
this.spec = spec;
}
status(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
if (this.skipped)
return 'skipped';
// List mode bail out.
if (!this.results.length)
return 'skipped';
if (this.results.length === 1 && this.expectedStatus === this.results[0].status)
return 'expected';
let hasPassedResults = false;
for (const result of this.results) {
// Missing status is Ok when running in shards mode.
if (!result.status)
return 'skipped';
if (result.status === this.expectedStatus)
hasPassedResults = true;
}
if (hasPassedResults)
return 'flaky';
return 'unexpected';
}
ok(): boolean {
const status = this.status();
return status === 'expected' || status === 'flaky' || status === 'skipped';
}
_appendTestResult(): types.TestResult {
const result: types.TestResult = {
retry: this.results.length,
workerIndex: 0,
duration: 0,
stdout: [],
stderr: [],
data: {}
};
this.results.push(result);
return result;
}
}

80
tests/folio/src/transform.ts Normal file
View File

@ -0,0 +1,80 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as crypto from 'crypto';
import * as os from 'os';
import * as path from 'path';
import * as fs from 'fs';
import * as pirates from 'pirates';
import * as babel from '@babel/core';
import * as sourceMapSupport from 'source-map-support';
const version = 3;
const cacheDir = path.join(os.tmpdir(), 'playwright-transform-cache');
const sourceMaps: Map<string, string> = new Map();
sourceMapSupport.install({
environment: 'node',
handleUncaughtExceptions: false,
retrieveSourceMap(source) {
if (!sourceMaps.has(source))
return null;
const sourceMapPath = sourceMaps.get(source);
if (!fs.existsSync(sourceMapPath))
return null;
return {
map: JSON.parse(fs.readFileSync(sourceMapPath, 'utf-8')),
url: source
};
}
});
function calculateCachePath(content: string, filePath: string): string {
const hash = crypto.createHash('sha1').update(content).update(filePath).update(String(version)).digest('hex');
const fileName = path.basename(filePath, path.extname(filePath)).replace(/\W/g, '') + '_' + hash;
return path.join(cacheDir, hash[0] + hash[1], fileName);
}
export function installTransform(): () => void {
return pirates.addHook((code, filename) => {
const cachePath = calculateCachePath(code, filename);
const codePath = cachePath + '.js';
const sourceMapPath = cachePath + '.map';
sourceMaps.set(filename, sourceMapPath);
if (fs.existsSync(codePath))
return fs.readFileSync(codePath, 'utf8');
const result = babel.transformFileSync(filename, {
babelrc: false,
configFile: false,
presets: [
['@babel/preset-env', { targets: {node: '10.17.0'} }],
['@babel/preset-typescript', { onlyRemoveTypeImports: true }],
],
plugins: [['@babel/plugin-proposal-class-properties', {loose: true}]],
sourceMaps: 'both',
});
if (result.code) {
fs.mkdirSync(path.dirname(cachePath), {recursive: true});
if (result.map)
fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');
fs.writeFileSync(codePath, result.code, 'utf8');
}
return result.code;
}, {
exts: ['.ts']
});
}

194
tests/folio/src/types.ts Normal file
View File

@ -0,0 +1,194 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Expect } from './expectType';
export interface RunWithConfig {
timeout?: number;
// TODO: move retries, outputDir, repeatEach, snapshotDir, testPathSegment here from Config.
}
export interface Config extends RunWithConfig {
forbidOnly?: boolean;
globalTimeout?: number;
grep?: string | RegExp | (string | RegExp)[];
maxFailures?: number;
outputDir?: string;
quiet?: boolean;
repeatEach?: number;
retries?: number;
shard?: { total: number, current: number } | null;
snapshotDir?: string;
testDir?: string;
testIgnore?: string | RegExp | (string | RegExp)[];
testMatch?: string | RegExp | (string | RegExp)[];
updateSnapshots?: boolean;
workers?: number;
}
export type FullConfig = Required<Config>;
export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
export interface WorkerInfo {
config: FullConfig;
workerIndex: number;
globalSetupResult: any;
}
export interface TestInfo extends WorkerInfo {
// Declaration
title: string;
file: string;
line: number;
column: number;
fn: Function;
// Modifiers
expectedStatus: TestStatus;
timeout: number;
annotations: any[];
testOptions: any; // TODO: make testOptions typed.
repeatEachIndex: number;
retry: number;
// Results
duration: number;
status?: TestStatus;
error?: any;
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
data: any;
// Paths
snapshotPathSegment: string;
snapshotPath: (...pathSegments: string[]) => string;
outputPath: (...pathSegments: string[]) => string;
}
interface SuiteFunction {
(name: string, inner: () => void): void;
}
interface TestFunction<TestArgs, TestOptions> {
(name: string, inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void): void;
(name: string, options: TestOptions, fn: (args: TestArgs, testInfo: TestInfo) => any): void;
}
export interface TestType<TestArgs, TestOptions> extends TestFunction<TestArgs, TestOptions> {
only: TestFunction<TestArgs, TestOptions>;
describe: SuiteFunction & {
only: SuiteFunction;
};
beforeEach: (inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void) => void;
afterEach: (inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void) => void;
beforeAll: (inner: (workerInfo: WorkerInfo) => Promise<void> | void) => void;
afterAll: (inner: (workerInfo: WorkerInfo) => Promise<void> | void) => void;
expect: Expect;
skip(): void;
skip(condition: boolean): void;
skip(description: string): void;
skip(condition: boolean, description: string): void;
fixme(): void;
fixme(condition: boolean): void;
fixme(description: string): void;
fixme(condition: boolean, description: string): void;
fail(): void;
fail(condition: boolean): void;
fail(description: string): void;
fail(condition: boolean, description: string): void;
runWith(config?: RunWithConfig): void;
runWith(alias: string, config?: RunWithConfig): void;
runWith(env: Env<TestArgs>, config?: RunWithConfig): void;
runWith(alias: string, env: Env<TestArgs>, config?: RunWithConfig): void;
runWith<TestArgs1, TestArgs2>(env1: Env<TestArgs1>, env2: Env<TestArgs2>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2>;
runWith<TestArgs1, TestArgs2>(alias: string, env1: Env<TestArgs1>, env2: Env<TestArgs2>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2>;
runWith<TestArgs1, TestArgs2, TestArgs3>(env1: Env<TestArgs1>, env2: Env<TestArgs2>, env3: Env<TestArgs3>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2 & TestArgs3>;
runWith<TestArgs1, TestArgs2, TestArgs3>(alias: string, env1: Env<TestArgs1>, env2: Env<TestArgs2>, env3: Env<TestArgs3>, config?: RunWithConfig): RunWithOrNever<TestArgs, TestArgs1 & TestArgs2 & TestArgs3>;
}
export interface Env<TestArgs> {
beforeAll?(workerInfo: WorkerInfo): Promise<any>;
beforeEach?(testInfo: TestInfo): Promise<TestArgs>;
afterEach?(testInfo: TestInfo): Promise<any>;
afterAll?(workerInfo: WorkerInfo): Promise<any>;
}
type RunWithOrNever<ExpectedTestArgs, CombinedTestArgs> = CombinedTestArgs extends ExpectedTestArgs ? void : never;
// ---------- Reporters API -----------
export interface Suite {
title: string;
file: string;
line: number;
column: number;
suites: Suite[];
specs: Spec[];
findTest(fn: (test: Test) => boolean | void): boolean;
findSpec(fn: (spec: Spec) => boolean | void): boolean;
totalTestCount(): number;
}
export interface Spec {
title: string;
file: string;
line: number;
column: number;
tests: Test[];
fullTitle(): string;
ok(): boolean;
}
export interface Test {
spec: Spec;
results: TestResult[];
skipped: boolean;
expectedStatus: TestStatus;
timeout: number;
annotations: any[];
alias: string;
status(): 'skipped' | 'expected' | 'unexpected' | 'flaky';
ok(): boolean;
}
export interface TestResult {
retry: number;
workerIndex: number,
duration: number;
status?: TestStatus;
error?: TestError;
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
data: any;
}
export interface TestError {
message?: string;
stack?: string;
value?: string;
}
export interface Reporter {
onBegin(config: FullConfig, suite: Suite): void;
onTestBegin(test: Test): void;
onStdOut(chunk: string | Buffer, test?: Test): void;
onStdErr(chunk: string | Buffer, test?: Test): void;
onTestEnd(test: Test, result: TestResult): void;
onTimeout(timeout: number): void;
onError(error: TestError): void;
onEnd(): void;
}

139
tests/folio/src/util.ts Normal file
View File

@ -0,0 +1,139 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import path from 'path';
import util from 'util';
import StackUtils from 'stack-utils';
import { TestError } from './types';
import { default as minimatch } from 'minimatch';
const FOLIO_DIRS = [__dirname, path.join(__dirname, '..', 'src')];
const cwd = process.cwd();
const stackUtils = new StackUtils({ cwd });
export async function raceAgainstDeadline<T>(promise: Promise<T>, deadline: number): Promise<{ result?: T, timedOut?: boolean }> {
if (!deadline)
return { result: await promise };
const timeout = deadline - monotonicTime();
if (timeout <= 0)
return { timedOut: true };
let timer: NodeJS.Timer;
let done = false;
let fulfill: (t: { result?: T, timedOut?: boolean }) => void;
let reject: (e: Error) => void;
const result = new Promise((f, r) => {
fulfill = f;
reject = r;
});
setTimeout(() => {
done = true;
fulfill({ timedOut: true });
}, timeout);
promise.then(result => {
clearTimeout(timer);
if (!done) {
done = true;
fulfill({ result });
}
}).catch(e => {
clearTimeout(timer);
if (!done)
reject(e);
});
return result;
}
export function serializeError(error: Error | any): TestError {
if (error instanceof Error) {
return {
message: error.message,
stack: error.stack
};
}
return {
value: util.inspect(error)
};
}
function callFrames(): string[] {
const obj = { stack: '' };
Error.captureStackTrace(obj);
const frames = obj.stack.split('\n').slice(1);
while (frames.length && FOLIO_DIRS.some(dir => frames[0].includes(dir)))
frames.shift();
return frames;
}
export function callLocation(fallbackFile: string): {file: string, line: number, column: number} {
const frames = callFrames();
if (!frames.length)
return {file: fallbackFile, line: 1, column: 1};
const location = stackUtils.parseLine(frames[0]);
return {
file: path.resolve(cwd, location.file),
line: location.line,
column: location.column,
};
}
export function errorWithCallLocation(message: string): Error {
const frames = callFrames();
const error = new Error(message);
error.stack = 'Error: ' + message + '\n' + frames.join('\n');
return error;
}
export function monotonicTime(): number {
const [seconds, nanoseconds] = process.hrtime();
return seconds * 1000 + (nanoseconds / 1000000 | 0);
}
export function prependErrorMessage(e: Error, message: string) {
let stack = e.stack || '';
if (stack.includes(e.message))
stack = stack.substring(stack.indexOf(e.message) + e.message.length);
let m = e.message;
if (m.startsWith('Error:'))
m = m.substring('Error:'.length);
e.message = message + m;
e.stack = e.message + stack;
}
export function interpretCondition(arg?: boolean | string, description?: string): { condition: boolean, description?: string } {
if (arg === undefined && description === undefined)
return { condition: true };
if (typeof arg === 'string')
return { condition: true, description: arg };
return { condition: !!arg, description };
}
export function createMatcher(patterns: string | RegExp | (string | RegExp)[]): (value: string) => boolean {
const list = Array.isArray(patterns) ? patterns : [patterns];
return (value: string) => {
for (const pattern of list) {
if (pattern instanceof RegExp || Object.prototype.toString.call(pattern) === '[object RegExp]') {
if ((pattern as RegExp).test(value))
return true;
} else {
if (minimatch(value, pattern))
return true;
}
}
return false;
};
}

119
tests/folio/src/worker.ts Normal file
View File

@ -0,0 +1,119 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Console } from 'console';
import * as util from 'util';
import { RunPayload, TestOutputPayload, WorkerInitParams } from './ipc';
import { serializeError } from './util';
import { WorkerRunner } from './workerRunner';
let closed = false;
sendMessageToParent('ready');
global.console = new Console({
stdout: process.stdout,
stderr: process.stderr,
colorMode: process.env.FORCE_COLOR === '1',
});
process.stdout.write = chunk => {
const outPayload: TestOutputPayload = {
testId: workerRunner ? workerRunner._testId : undefined,
...chunkToParams(chunk)
};
sendMessageToParent('stdOut', outPayload);
return true;
};
if (!process.env.PW_RUNNER_DEBUG) {
process.stderr.write = chunk => {
const outPayload: TestOutputPayload = {
testId: workerRunner ? workerRunner._testId : undefined,
...chunkToParams(chunk)
};
sendMessageToParent('stdErr', outPayload);
return true;
};
}
process.on('disconnect', gracefullyCloseAndExit);
process.on('SIGINT',() => {});
process.on('SIGTERM',() => {});
let workerRunner: WorkerRunner;
process.on('unhandledRejection', (reason, promise) => {
if (workerRunner)
workerRunner.unhandledError(reason);
});
process.on('uncaughtException', error => {
if (workerRunner)
workerRunner.unhandledError(error);
});
process.on('message', async message => {
if (message.method === 'init') {
const initParams = message.params as WorkerInitParams;
workerRunner = new WorkerRunner(initParams);
for (const event of ['testBegin', 'testEnd', 'done'])
workerRunner.on(event, sendMessageToParent.bind(null, event));
return;
}
if (message.method === 'stop') {
await gracefullyCloseAndExit();
return;
}
if (message.method === 'run') {
const runPayload = message.params as RunPayload;
await workerRunner!.run(runPayload);
}
});
async function gracefullyCloseAndExit() {
if (closed)
return;
closed = true;
// Force exit after 30 seconds.
setTimeout(() => process.exit(0), 30000);
// Meanwhile, try to gracefully shutdown.
try {
if (workerRunner) {
workerRunner.stop();
await workerRunner.cleanup();
}
} catch (e) {
process.send({ method: 'teardownError', params: { error: serializeError(e) } });
}
process.exit(0);
}
function sendMessageToParent(method, params = {}) {
try {
process.send({ method, params });
} catch (e) {
// Can throw when closing.
}
}
function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: string } {
if (chunk instanceof Buffer)
return { buffer: chunk.toString('base64') };
if (typeof chunk !== 'string')
return { text: util.inspect(chunk) };
return { text: chunk };
}

440
tests/folio/src/workerRunner.ts Normal file
View File

@ -0,0 +1,440 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import { EventEmitter } from 'events';
import { monotonicTime, raceAgainstDeadline, serializeError } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
import { setCurrentTestInfo } from './globals';
import { Loader } from './loader';
import { Spec, Suite, Test } from './test';
import { TestInfo, WorkerInfo } from './types';
import { SkipError, RunListDescription } from './spec';
export class WorkerRunner extends EventEmitter {
private _params: WorkerInitParams;
private _loader: Loader;
private _runList: RunListDescription;
private _outputPathSegment: string;
private _workerInfo: WorkerInfo;
private _envInitialized = false;
private _failedTestId: string | undefined;
private _fatalError: any | undefined;
private _entries: Map<string, TestEntry>;
private _remaining: Map<string, TestEntry>;
private _isStopped: any;
_testId: string | null;
private _testInfo: TestInfo | null = null;
private _file: string;
private _timeout: number;
constructor(params: WorkerInitParams) {
super();
this._params = params;
}
stop() {
this._isStopped = true;
this._testId = null;
this._setCurrentTestInfo(null);
}
async cleanup() {
if (!this._envInitialized)
return;
this._envInitialized = false;
if (this._runList.env.afterAll) {
// TODO: separate timeout for afterAll?
const result = await raceAgainstDeadline(this._runList.env.afterAll(this._workerInfo), this._deadline());
if (result.timedOut)
throw new Error(`Timeout of ${this._timeout}ms exceeded while shutting down environment`);
}
}
unhandledError(error: Error | any) {
if (this._isStopped)
return;
if (this._testInfo) {
this._testInfo.status = 'failed';
this._testInfo.error = serializeError(error);
this._failedTestId = this._testId;
this.emit('testEnd', buildTestEndPayload(this._testId, this._testInfo));
} else {
// No current test - fatal error.
this._fatalError = serializeError(error);
}
this._reportDoneAndStop();
}
private _deadline() {
return this._timeout ? monotonicTime() + this._timeout : 0;
}
private async _loadIfNeeded() {
if (this._loader)
return;
this._loader = new Loader();
this._loader.deserialize(this._params.loader);
this._runList = this._loader.runLists()[this._params.runListIndex];
const sameAliasAndTestType = this._loader.runLists().filter(runList => runList.alias === this._runList.alias && runList.testType === this._runList.testType);
if (sameAliasAndTestType.length > 1)
this._outputPathSegment = this._runList.alias + (sameAliasAndTestType.indexOf(this._runList) + 1);
else
this._outputPathSegment = this._runList.alias;
this._timeout = this._runList.config.timeout === undefined ? this._loader.config().timeout : this._runList.config.timeout;
this._workerInfo = {
workerIndex: this._params.workerIndex,
config: this._loader.config(),
globalSetupResult: this._params.globalSetupResult,
};
if (this._isStopped)
return;
if (this._runList.env.beforeAll) {
// TODO: separate timeout for beforeAll?
const result = await raceAgainstDeadline(this._runList.env.beforeAll(this._workerInfo), this._deadline());
if (result.timedOut) {
this._fatalError = serializeError(new Error(`Timeout of ${this._timeout}ms exceeded while initializing environment`));
this._reportDoneAndStop();
}
}
this._envInitialized = true;
}
async run(runPayload: RunPayload) {
this._file = runPayload.file;
this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
this._remaining = new Map(runPayload.entries.map(e => [ e.testId, e ]));
await this._loadIfNeeded();
if (this._isStopped)
return;
this._loader.loadTestFile(this._file);
const fileSuite = this._runList.fileSuites.get(this._file);
if (fileSuite) {
fileSuite._renumber();
fileSuite.findSpec(spec => {
spec._appendTest(this._params.runListIndex, this._runList.alias, this._params.repeatEachIndex);
});
await this._runSuite(fileSuite);
}
if (this._isStopped)
return;
this._reportDone();
}
private async _runSuite(suite: Suite) {
if (this._isStopped)
return;
const skipHooks = !this._hasTestsToRun(suite);
for (const hook of suite._hooks) {
if (hook.type !== 'beforeAll' || skipHooks)
continue;
if (this._isStopped)
return;
// TODO: separate timeout for beforeAll?
const result = await raceAgainstDeadline(hook.fn(this._workerInfo), this._deadline());
if (result.timedOut) {
this._fatalError = serializeError(new Error(`Timeout of ${this._timeout}ms exceeded while running beforeAll hook`));
this._reportDoneAndStop();
}
}
for (const entry of suite._entries) {
if (entry instanceof Suite)
await this._runSuite(entry);
else
await this._runSpec(entry);
}
for (const hook of suite._hooks) {
if (hook.type !== 'afterAll' || skipHooks)
continue;
if (this._isStopped)
return;
// TODO: separate timeout for afterAll?
const result = await raceAgainstDeadline(hook.fn(this._workerInfo), this._deadline());
if (result.timedOut) {
this._fatalError = serializeError(new Error(`Timeout of ${this._timeout}ms exceeded while running afterAll hook`));
this._reportDoneAndStop();
}
}
}
private async _runSpec(spec: Spec) {
if (this._isStopped)
return;
const test = spec.tests[0];
if (!this._entries.has(test._id))
return;
const { retry } = this._entries.get(test._id);
// TODO: support some of test.slow(), test.setTimeout(), describe.slow() and describe.setTimeout()
const deadline = this._deadline();
this._remaining.delete(test._id);
const testId = test._id;
this._testId = testId;
const config = this._workerInfo.config;
const relativePath = path.relative(config.testDir, spec.file.replace(/\.(spec|test)\.(js|ts)/, ''));
const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '-');
const relativeTestPath = path.join(relativePath, sanitizedTitle);
const testInfo: TestInfo = {
...this._workerInfo,
title: spec.title,
file: spec.file,
line: spec.line,
column: spec.column,
fn: spec.fn,
repeatEachIndex: this._params.repeatEachIndex,
retry,
expectedStatus: 'passed',
annotations: [],
duration: 0,
status: 'passed',
stdout: [],
stderr: [],
timeout: this._timeout,
data: {},
snapshotPathSegment: '',
outputPath: (...pathSegments: string[]): string => {
let suffix = this._outputPathSegment;
if (testInfo.retry)
suffix += (suffix ? '-' : '') + 'retry' + testInfo.retry;
if (testInfo.repeatEachIndex)
suffix += (suffix ? '-' : '') + 'repeat' + testInfo.repeatEachIndex;
const basePath = path.join(config.outputDir, relativeTestPath, suffix);
fs.mkdirSync(basePath, { recursive: true });
return path.join(basePath, ...pathSegments);
},
snapshotPath: (...pathSegments: string[]): string => {
const basePath = path.join(config.testDir, config.snapshotDir, relativeTestPath, testInfo.snapshotPathSegment);
return path.join(basePath, ...pathSegments);
},
testOptions: spec.testOptions,
};
this._setCurrentTestInfo(testInfo);
// Preprocess suite annotations.
for (let parent = spec.parent; parent; parent = parent.parent)
testInfo.annotations.push(...parent._annotations);
if (testInfo.annotations.some(a => a.type === 'skip' || a.type === 'fixme'))
testInfo.expectedStatus = 'skipped';
else if (testInfo.annotations.some(a => a.type === 'fail'))
testInfo.expectedStatus = 'failed';
this.emit('testBegin', buildTestBeginPayload(testId, testInfo));
if (testInfo.expectedStatus === 'skipped') {
testInfo.status = 'skipped';
this.emit('testEnd', buildTestEndPayload(testId, testInfo));
return;
}
const startTime = monotonicTime();
const testArgsResult = await raceAgainstDeadline(this._runEnvBeforeEach(testInfo), deadline);
if (testArgsResult.timedOut && testInfo.status === 'passed')
testInfo.status = 'timedOut';
if (this._isStopped)
return;
const testArgs = testArgsResult.result;
// Do not run test/teardown if we failed to initialize.
if (testArgs !== undefined) {
const result = await raceAgainstDeadline(this._runTestWithBeforeHooks(test, testInfo, testArgs), deadline);
// Do not overwrite test failure upon hook timeout.
if (result.timedOut && testInfo.status === 'passed')
testInfo.status = 'timedOut';
if (this._isStopped)
return;
if (!result.timedOut) {
const hooksResult = await raceAgainstDeadline(this._runAfterHooks(test, testInfo, testArgs), deadline);
// Do not overwrite test failure upon hook timeout.
if (hooksResult.timedOut && testInfo.status === 'passed')
testInfo.status = 'timedOut';
} else {
// A timed-out test gets a full additional timeout to run after hooks.
const newDeadline = this._deadline();
await raceAgainstDeadline(this._runAfterHooks(test, testInfo, testArgs), newDeadline);
}
}
if (this._isStopped)
return;
testInfo.duration = monotonicTime() - startTime;
this.emit('testEnd', buildTestEndPayload(testId, testInfo));
if (testInfo.status !== 'passed') {
this._failedTestId = this._testId;
this._reportDoneAndStop();
}
this._setCurrentTestInfo(null);
this._testId = null;
}
private _setCurrentTestInfo(testInfo: TestInfo | null) {
this._testInfo = testInfo;
setCurrentTestInfo(testInfo);
}
// Returns TestArgs or undefined when env.beforeEach has failed.
private async _runEnvBeforeEach(testInfo: TestInfo): Promise<any> {
try {
let testArgs: any = {};
if (this._runList.env.beforeEach)
testArgs = await this._runList.env.beforeEach(testInfo);
if (testArgs === undefined)
testArgs = {};
return testArgs;
} catch (error) {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
// Failed to initialize environment - no need to run any hooks now.
return undefined;
}
}
private async _runTestWithBeforeHooks(test: Test, testInfo: TestInfo, testArgs: any) {
try {
await this._runHooks(test.spec.parent, 'beforeEach', testArgs, testInfo);
} catch (error) {
if (error instanceof SkipError) {
testInfo.status = 'skipped';
} else {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
}
// Continue running afterEach hooks even after the failure.
}
// Do not run the test when beforeEach hook fails.
if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped')
return;
try {
await test.spec.fn(testArgs, testInfo);
testInfo.status = 'passed';
} catch (error) {
if (error instanceof SkipError) {
testInfo.status = 'skipped';
} else {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
}
}
}
private async _runAfterHooks(test: Test, testInfo: TestInfo, testArgs: any) {
try {
await this._runHooks(test.spec.parent, 'afterEach', testArgs, testInfo);
} catch (error) {
// Do not overwrite test failure error.
if (!(error instanceof SkipError) && testInfo.status === 'passed') {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
// Continue running even after the failure.
}
}
try {
if (this._runList.env.afterEach)
await this._runList.env.afterEach(testInfo);
} catch (error) {
// Do not overwrite test failure error.
if (testInfo.status === 'passed') {
testInfo.status = 'failed';
testInfo.error = serializeError(error);
}
}
}
private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testArgs: any, testInfo: TestInfo) {
if (this._isStopped)
return;
const all = [];
for (let s = suite; s; s = s.parent) {
const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn);
all.push(...funcs.reverse());
}
if (type === 'beforeEach')
all.reverse();
let error: Error | undefined;
for (const hook of all) {
try {
await hook(testArgs, testInfo);
} catch (e) {
// Always run all the hooks, and capture the first error.
error = error || e;
}
}
if (error)
throw error;
}
private _reportDone() {
const donePayload: DonePayload = {
failedTestId: this._failedTestId,
fatalError: this._fatalError,
remaining: [...this._remaining.values()],
};
this.emit('done', donePayload);
}
private _reportDoneAndStop() {
if (this._isStopped)
return;
this._reportDone();
this.stop();
}
private _hasTestsToRun(suite: Suite): boolean {
return suite.findSpec(spec => {
const entry = this._entries.get(spec.tests[0]._id);
if (!entry)
return;
for (let parent = spec.parent; parent; parent = parent.parent) {
if (parent._annotations.some(a => a.type === 'skip' || a.type === 'fixme'))
return;
}
return true;
});
}
}
function buildTestBeginPayload(testId: string, testInfo: TestInfo): TestBeginPayload {
return {
testId,
workerIndex: testInfo.workerIndex
};
}
function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload {
return {
testId,
duration: testInfo.duration,
status: testInfo.status!,
error: testInfo.error,
data: testInfo.data,
expectedStatus: testInfo.expectedStatus,
annotations: testInfo.annotations,
timeout: testInfo.timeout,
};
}

2222
tests/folio/third_party/diff_match_patch.js vendored Normal file
View File

File diff suppressed because it is too large Load Diff

22
tests/folio/tsconfig.json Normal file
View File

@ -0,0 +1,22 @@
{
"compilerOptions": {
"target": "ESNext",
"moduleResolution": "node",
"module": "commonjs",
"strict": false,
"sourceMap": true,
"declarationMap": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"declaration": true,
"outDir": "./out",
"rootDir": "./src",
"lib": ["esnext", "dom", "DOM.Iterable"],
},
"exclude": [
"node_modules"
],
"include": [
"src"
]
}

10
test/page/jshandle-as-element.spec.ts → tests/jshandle-as-element.spec.ts
View File

@ -15,21 +15,21 @@
* limitations under the License.
*/
import { it, expect } from '../fixtures';
import { test, expect } from './config/pageTest';
it('should work', async ({page}) => {
test('should work', async ({page}) => {
const aHandle = await page.evaluateHandle(() => document.body);
const element = aHandle.asElement();
expect(element).toBeTruthy();
});
it('should return null for non-elements', async ({page}) => {
test('should return null for non-elements', async ({page}) => {
const aHandle = await page.evaluateHandle(() => 2);
const element = aHandle.asElement();
expect(element).toBeFalsy();
});
it('should return ElementHandle for TextNodes', async ({page}) => {
test('should return ElementHandle for TextNodes', async ({page}) => {
await page.setContent('<div>ee!</div>');
const aHandle = await page.evaluateHandle(() => document.querySelector('div').firstChild);
const element = aHandle.asElement();
@ -37,7 +37,7 @@ it('should return ElementHandle for TextNodes', async ({page}) => {
expect(await page.evaluate(e => e.nodeType === Node.TEXT_NODE, element)).toBeTruthy();
});
it('should work with nullified Node', async ({page}) => {
test('should work with nullified Node', async ({page}) => {
await page.setContent('<section>test</section>');
await page.evaluate('delete Node');
const handle = await page.evaluateHandle(() => document.querySelector('section'));

30
tests/stack-trace.spec.ts Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright 2019 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from './config/playwrightTest';
import path from 'path';
import * as stackTrace from '../src/utils/stackTrace';
import { setUnderTest } from '../src/utils/utils';
test('caller file path', async ({}) => {
setUnderTest();
const callme = require('./assets/callback');
const filePath = callme(() => {
return stackTrace.getCallerFilePath(path.join(__dirname, 'assets') + path.sep);
});
expect(filePath).toBe(__filename);
});