From b01ccc282fec6a7c9c5f06c7712f528297a29873 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Tue, 11 May 2021 06:40:06 -0700 Subject: [PATCH] test: roll to folio@0.4.0-alpha11 (#6496) --- .github/workflows/tests.yml | 40 ++++++--------------- .github/workflows/tests_chrome_beta.yml | 12 ++----- package-lock.json | 6 ++-- package.json | 2 +- tests/config/android.config.ts | 1 + tests/config/default.config.ts | 18 ++++++---- tests/config/electron.config.ts | 1 + tests/page/elementhandle-screenshot.spec.ts | 8 +++-- tests/page/page-screenshot.spec.ts | 4 ++- 9 files changed, 40 insertions(+), 52 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 07d1c1ecf4..942c5dbcf4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -140,10 +140,8 @@ jobs: - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }}" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }} --headed" if: ${{ always() }} - env: - HEADFUL: 1 - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -169,9 +167,7 @@ jobs: - run: node lib/cli/cli install-deps chromium # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest" - env: - PWTEST_MODE: "${{ matrix.mode }}" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --mode=${{ matrix.mode }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -197,9 +193,7 @@ jobs: - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }}" - env: - PWTEST_VIDEO: 1 + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }} --video" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -256,9 +250,7 @@ jobs: - run: node lib/cli/cli install ffmpeg # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest" - env: - PWTEST_CHANNEL: "chrome" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --channel=chrome" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -284,10 +276,8 @@ jobs: - run: npm run build # This only created problems, should we move ffmpeg back into npm? - run: node lib/cli/cli install ffmpeg - - run: npm run ctest + - run: npm run ctest -- --channel=chrome shell: bash - env: - PWTEST_CHANNEL: "chrome" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -311,9 +301,7 @@ jobs: - run: npm run build # This only created problems, should we move ffmpeg back into npm? - run: node lib/cli/cli install ffmpeg - - run: npm run ctest - env: - PWTEST_CHANNEL: "chrome" + - run: npm run ctest -- --channel=chrome - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -338,9 +326,7 @@ jobs: - run: node lib/cli/cli install ffmpeg firefox-stable chromium # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest" - env: - PWTEST_CHANNEL: "firefox-stable" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest -- --channel=firefox-stable" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -365,10 +351,8 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: node lib/cli/cli install ffmpeg firefox-stable chromium - - run: npm run ftest + - run: npm run ftest -- --channel=firefox-stable shell: bash - env: - PWTEST_CHANNEL: "firefox-stable" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -391,9 +375,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: node lib/cli/cli install ffmpeg firefox-stable chromium - - run: npm run ftest - env: - PWTEST_CHANNEL: "firefox-stable" + - run: npm run ftest -- --channel=firefox-stable - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -419,10 +401,8 @@ jobs: - run: npm run build # This only created problems, should we move ffmpeg back into npm? - run: node lib/cli/cli install ffmpeg - - run: npm run ctest + - run: npm run ctest -- --channel=msedge shell: bash - env: - PWTEST_CHANNEL: "msedge" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: diff --git a/.github/workflows/tests_chrome_beta.yml b/.github/workflows/tests_chrome_beta.yml index 027ff92ed5..ecee744999 100644 --- a/.github/workflows/tests_chrome_beta.yml +++ b/.github/workflows/tests_chrome_beta.yml @@ -33,9 +33,7 @@ jobs: - run: node lib/cli/cli install ffmpeg # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest" - env: - PWTEST_CHANNEL: "chrome-beta" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --channel=chrome-beta" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 @@ -63,10 +61,8 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: node lib/cli/cli install ffmpeg - - run: npm run ctest + - run: npm run ctest -- --channel=chrome-beta shell: bash - env: - PWTEST_CHANNEL: "chrome-beta" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -90,9 +86,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: node lib/cli/cli install ffmpeg - - run: npm run ctest - env: - PWTEST_CHANNEL: "chrome-beta" + - run: npm run ctest -- --channel=chrome-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 diff --git a/package-lock.json b/package-lock.json index feb44d0f76..03aa6e3d5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8351,9 +8351,9 @@ } }, "folio": { - "version": "0.4.0-alpha10", - "resolved": "https://registry.npmjs.org/folio/-/folio-0.4.0-alpha10.tgz", - "integrity": "sha512-UnmWFIgYP/NaU9nVFzbBP5Uj473miWj3K3XpmpVIa42TFePSRdyeogFAAbhdb60C8h5E32NkTQbquGDaMGabLg==", + "version": "0.4.0-alpha11", + "resolved": "https://registry.npmjs.org/folio/-/folio-0.4.0-alpha11.tgz", + "integrity": "sha512-4kXdO+Ndbn++vCbzuMbn8bGqQLQ9J/Vni/1r9UwvirE4HydfxP1PHkvx9qb7wsB2hQbXmPkU5qM0eyGWFKpmog==", "dev": true, "requires": { "@babel/code-frame": "^7.12.13", diff --git a/package.json b/package.json index abd1cb9885..7720e2ccf3 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "eslint-plugin-notice": "^0.9.10", "eslint-plugin-react-hooks": "^4.2.0", "file-loader": "^6.1.0", - "folio": "=0.4.0-alpha10", + "folio": "=0.4.0-alpha11", "formidable": "^1.2.2", "html-webpack-plugin": "^4.4.1", "ncp": "^2.0.0", diff --git a/tests/config/android.config.ts b/tests/config/android.config.ts index 0fe774bbaf..b72588f48d 100644 --- a/tests/config/android.config.ts +++ b/tests/config/android.config.ts @@ -48,6 +48,7 @@ const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: folio.Config = { testDir, + snapshotDir: '__snapshots__', outputDir, timeout: 120000, globalTimeout: 7200000, diff --git a/tests/config/default.config.ts b/tests/config/default.config.ts index 433e5ec34a..aaae09f334 100644 --- a/tests/config/default.config.ts +++ b/tests/config/default.config.ts @@ -78,12 +78,18 @@ class PageEnv { } } +const mode = folio.registerCLIOption('mode', 'Transport mode: default, driver or service').value as ('default' | 'driver' | 'service' | undefined); +const headful = folio.registerCLIOption('headed', 'Run tests in headed mode (default: headless)', { type: 'boolean' }).value || !!process.env.HEADFUL; +const channel = folio.registerCLIOption('channel', 'Browser channel (default: no channel)').value; +const video = !!folio.registerCLIOption('video', 'Record videos for all tests', { type: 'boolean' }).value; + const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: folio.Config = { testDir, + snapshotDir: '__snapshots__', outputDir, - timeout: process.env.PWTEST_VIDEO || process.env.PWTRACE ? 60000 : 30000, + timeout: video || process.env.PWTRACE ? 60000 : 30000, globalTimeout: 5400000, workers: process.env.CI ? 1 : undefined, forbidOnly: !!process.env.CI, @@ -98,7 +104,7 @@ const config: folio.Config = { const browserNames = ['chromium', 'webkit', 'firefox'] as BrowserName[]; for (const browserName of browserNames) { const executablePath = getExecutablePath(browserName); - if (executablePath && (process.env.FOLIO_WORKER_INDEX === undefined || process.env.FOLIO_WORKER_INDEX === '')) + if (executablePath && !process.env.FOLIO_WORKER_INDEX) console.error(`Using executable at ${executablePath}`); const testIgnore: RegExp[] = browserNames.filter(b => b !== browserName).map(b => new RegExp(b)); testIgnore.push(/android/, /electron/); @@ -107,11 +113,11 @@ for (const browserName of browserNames) { testDir, testIgnore, options: { - mode: (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service'), + mode, browserName, - headful: !!process.env.HEADFUL, - channel: process.env.PWTEST_CHANNEL as any, - video: !!process.env.PWTEST_VIDEO, + headful, + channel, + video, traceDir: process.env.PWTRACE ? path.join(outputDir, 'trace') : undefined, launchOptions: { executablePath, diff --git a/tests/config/electron.config.ts b/tests/config/electron.config.ts index 3babc06902..424650b4c5 100644 --- a/tests/config/electron.config.ts +++ b/tests/config/electron.config.ts @@ -42,6 +42,7 @@ const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: folio.Config = { testDir, + snapshotDir: '__snapshots__', outputDir, timeout: 30000, globalTimeout: 5400000, diff --git a/tests/page/elementhandle-screenshot.spec.ts b/tests/page/elementhandle-screenshot.spec.ts index f292535279..83c52d1a44 100644 --- a/tests/page/elementhandle-screenshot.spec.ts +++ b/tests/page/elementhandle-screenshot.spec.ts @@ -203,14 +203,18 @@ it.describe('element screenshot', () => { expect(screenshot).toMatchSnapshot('screenshot-element-bounding-box.png'); }); - it('should work for an element with fractional dimensions', async ({page}) => { + it('should work for an element with fractional dimensions', async ({page, isElectron}) => { + it.fixme(isElectron, 'Scale is wrong'); + await page.setContent('
'); const elementHandle = await page.$('div'); const screenshot = await elementHandle.screenshot(); expect(screenshot).toMatchSnapshot('screenshot-element-fractional.png'); }); - it('should work for an element with an offset', async ({page}) => { + it('should work for an element with an offset', async ({page, isElectron}) => { + it.fixme(isElectron, 'Scale is wrong'); + await page.setContent('
'); const elementHandle = await page.$('div'); const screenshot = await elementHandle.screenshot(); diff --git a/tests/page/page-screenshot.spec.ts b/tests/page/page-screenshot.spec.ts index 28c7b5c68e..7c0b830622 100644 --- a/tests/page/page-screenshot.spec.ts +++ b/tests/page/page-screenshot.spec.ts @@ -150,7 +150,9 @@ it.describe('page screenshot', () => { expect(screenshot).toMatchSnapshot('white.jpg'); }); - it('should work with odd clip size on Retina displays', async ({page}) => { + it('should work with odd clip size on Retina displays', async ({page, isElectron}) => { + it.fixme(isElectron, 'Scale is wrong'); + const screenshot = await page.screenshot({ clip: { x: 0,