test: roll to folio@0.4.0-alpha21 (#6789)

.github/workflows/tests_secondary.yml

@@ -148,8 +148,10 @@ jobs:
     - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }} --headed"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }}"
       if: ${{ always() }}
+      env:
+        HEADFUL: 1
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -177,7 +179,9 @@ jobs:
     - run: node lib/cli/cli install-deps chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --mode=${{ matrix.mode }}"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
+      env:
+        PWTEST_MODE: ${{ matrix.mode }}
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -205,7 +209,9 @@ jobs:
     - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }} --video"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --project=${{ matrix.browser }}"
+      env:
+        PWTEST_VIDEO: 1
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -264,7 +270,9 @@ jobs:
     - run: node lib/cli/cli install ffmpeg
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --channel=chrome"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
+      env:
+        PWTEST_CHANNEL: chrome
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -290,8 +298,10 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome
+    - run: npm run ctest
       shell: bash
+      env:
+        PWTEST_CHANNEL: chrome
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
@@ -315,7 +325,9 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome
+    - run: npm run ctest
+      env:
+        PWTEST_CHANNEL: chrome
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -340,7 +352,9 @@ jobs:
     - run: node lib/cli/cli install ffmpeg firefox-stable chromium
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest -- --channel=firefox-stable"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest"
+      env:
+        PWTEST_CHANNEL: firefox-stable
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -365,8 +379,10 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg firefox-stable chromium
-    - run: npm run ftest -- --channel=firefox-stable
+    - run: npm run ftest
       shell: bash
+      env:
+        PWTEST_CHANNEL: firefox-stable
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
@@ -389,7 +405,9 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg firefox-stable chromium
-    - run: npm run ftest -- --channel=firefox-stable
+    - run: npm run ftest
+      env:
+        PWTEST_CHANNEL: firefox-stable
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -415,8 +433,10 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=msedge
+    - run: npm run ctest
       shell: bash
+      env:
+        PWTEST_CHANNEL: msedge
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -463,7 +483,9 @@ jobs:
     - run: node lib/cli/cli install ffmpeg
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest -- --channel=chrome-beta"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
+      env:
+        PWTEST_CHANNEL: chrome-beta
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1
@@ -490,8 +512,10 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome-beta
+    - run: npm run ctest
       shell: bash
+      env:
+        PWTEST_CHANNEL: chrome-beta
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
@@ -515,7 +539,9 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     - run: npm run build
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run ctest -- --channel=chrome-beta
+    - run: npm run ctest
+      env:
+        PWTEST_CHANNEL: chrome-beta
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
     - uses: actions/upload-artifact@v1

package-lock.json

@@ -14,9 +14,9 @@
       }
     },
     "@babel/compat-data": {
-      "version": "7.14.0",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.14.0.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.14.4.tgz",
-      "integrity": "sha512-vu9V3uMM/1o5Hl5OekMUowo3FqXLJSw+s+66nt0fSWVWTtmosdzn45JHOB3cPtZoe6CTBDzvSw0RdOY85Q37+Q==",
+      "integrity": "sha512-i2wXrWQNkH6JplJQGn3Rd2I4Pij8GdHkXwHMxm+zV5YG/Jci+bCNrWZEWC4o+umiDkRrRs4dVzH3X4GP7vyjQQ==",
       "dev": true
     },
     "@babel/core": {
@@ -94,14 +94,14 @@
       }
     },
     "@babel/helper-compilation-targets": {
-      "version": "7.13.16",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.13.16.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.14.4.tgz",
-      "integrity": "sha512-3gmkYIrpqsLlieFwjkGgLaSHmhnvlAYzZLlYVjlW+QwI+1zE17kGxuJGmIqDQdYp56XdmGeD+Bswx0UTyG18xA==",
+      "integrity": "sha512-JgdzOYZ/qGaKTVkn5qEDV/SXAh8KcyUVkCoSWGN8T3bwrgd6m+/dJa2kVGi6RJYJgEYPBdZ84BZp9dUjNWkBaA==",
       "dev": true,
       "requires": {
-        "@babel/compat-data": "^7.13.15",
+        "@babel/compat-data": "^7.14.4",
         "@babel/helper-validator-option": "^7.12.17",
-        "browserslist": "^4.14.5",
+        "browserslist": "^4.16.6",
         "semver": "^6.3.0"
       },
       "dependencies": {
@@ -114,16 +114,16 @@
       }
     },
     "@babel/helper-create-class-features-plugin": {
-      "version": "7.14.3",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.3.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.4.tgz",
-      "integrity": "sha512-BnEfi5+6J2Lte9LeiL6TxLWdIlEv9Woacc1qXzXBgbikcOzMRM2Oya5XGg/f/ngotv1ej2A/b+3iJH8wbS1+lQ==",
+      "integrity": "sha512-idr3pthFlDCpV+p/rMgGLGYIVtazeatrSOQk8YzO2pAepIjQhCN3myeihVg58ax2bbbGK9PUE1reFi7axOYIOw==",
       "dev": true,
       "requires": {
         "@babel/helper-annotate-as-pure": "^7.12.13",
         "@babel/helper-function-name": "^7.14.2",
         "@babel/helper-member-expression-to-functions": "^7.13.12",
         "@babel/helper-optimise-call-expression": "^7.12.13",
-        "@babel/helper-replace-supers": "^7.14.3",
+        "@babel/helper-replace-supers": "^7.14.4",
         "@babel/helper-split-export-declaration": "^7.12.13"
       }
     },
@@ -197,15 +197,15 @@
       "dev": true
     },
     "@babel/helper-replace-supers": {
-      "version": "7.14.3",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.14.3.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.14.4.tgz",
-      "integrity": "sha512-Rlh8qEWZSTfdz+tgNV/N4gz1a0TMNwCUcENhMjHTHKp3LseYH5Jha0NSlyTQWMnjbYcwFt+bqAMqSLHVXkQ6UA==",
+      "integrity": "sha512-zZ7uHCWlxfEAAOVDYQpEf/uyi1dmeC7fX4nCf2iz9drnCwi1zvwXL3HwWWNXUQEJ1k23yVn3VbddiI9iJEXaTQ==",
       "dev": true,
       "requires": {
         "@babel/helper-member-expression-to-functions": "^7.13.12",
         "@babel/helper-optimise-call-expression": "^7.12.13",
         "@babel/traverse": "^7.14.2",
-        "@babel/types": "^7.14.2"
+        "@babel/types": "^7.14.4"
       }
     },
     "@babel/helper-simple-access": {
@@ -270,9 +270,9 @@
       }
     },
     "@babel/parser": {
-      "version": "7.14.3",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.14.3.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.14.4.tgz",
-      "integrity": "sha512-7MpZDIfI7sUC5zWo2+foJ50CSI5lcqDehZ0lVgIhSi4bFEk94fLAKlF3Q0nzSQQ+ca0lm+O6G9ztKVBeu8PMRQ==",
+      "integrity": "sha512-ArliyUsWDUqEGfWcmzpGUzNfLxTdTp6WU4IuP6QFSp9gGfWS6boxFCkJSJ/L4+RG8z/FnIU3WxCk6hPL9SSWeA==",
       "dev": true
     },
     "@babel/plugin-proposal-class-properties": {
@@ -489,12 +489,12 @@
       }
     },
     "@babel/plugin-transform-typescript": {
-      "version": "7.14.3",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.14.3.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.14.4.tgz",
-      "integrity": "sha512-G5Bb5pY6tJRTC4ag1visSgiDoGgJ1u1fMUgmc2ijLkcIdzP83Q1qyZX4ggFQ/SkR+PNOatkaYC+nKcTlpsX4ag==",
+      "integrity": "sha512-WYdcGNEO7mCCZ2XzRlxwGj3PgeAr50ifkofOUC/+IN/GzKLB+biDPVBUAQN2C/dVZTvEXCp80kfQ1FFZPrwykQ==",
       "dev": true,
       "requires": {
-        "@babel/helper-create-class-features-plugin": "^7.14.3",
+        "@babel/helper-create-class-features-plugin": "^7.14.4",
         "@babel/helper-plugin-utils": "^7.13.0",
         "@babel/plugin-syntax-typescript": "^7.12.13"
       }
@@ -566,9 +566,9 @@
       }
     },
     "@babel/types": {
-      "version": "7.14.2",
+      "version": "7.14.4",
-      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.2.tgz",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.4.tgz",
-      "integrity": "sha512-SdjAG/3DikRHpUOjxZgnkbR11xUlyDMUFJdvnIgZEE16mqmY0BINMmc4//JMJglEmn6i7sq6p+mGrFWyZ98EEw==",
+      "integrity": "sha512-lCj4aIs0xUefJFQnwwQv2Bxg7Omd6bgquZ6LGC+gGMh6/s5qDVfjuCMlDmYQ15SLsWHd9n+X3E75lKIhl5Lkiw==",
       "dev": true,
       "requires": {
         "@babel/helper-validator-identifier": "^7.14.0",
@@ -2777,9 +2777,9 @@
       }
     },
     "electron-to-chromium": {
-      "version": "1.3.740",
+      "version": "1.3.741",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.740.tgz",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.741.tgz",
-      "integrity": "sha512-Mi2m55JrX2BFbNZGKYR+2ItcGnR4O5HhrvgoRRyZQlaMGQULqDhoGkLWHzJoshSzi7k1PUofxcDbNhlFrDZNhg==",
+      "integrity": "sha512-4i3T0cwnHo1O4Mnp9JniEco8bZiXoqbm3PhW5hv7uu8YLg35iajYrRnNyKFaN8/8SSTskU2hYqVTeYVPceSpUA==",
       "dev": true
     },
     "elliptic": {
@@ -3624,9 +3624,9 @@
       }
     },
     "folio": {
-      "version": "0.4.0-alpha19",
+      "version": "0.4.0-alpha21",
-      "resolved": "https://registry.npmjs.org/folio/-/folio-0.4.0-alpha19.tgz",
+      "resolved": "https://registry.npmjs.org/folio/-/folio-0.4.0-alpha21.tgz",
-      "integrity": "sha512-gWNfyQTF41nOHiH2BwsqXBRYhr/oDRxC1cQN2VTnVH+KQEbyic1wIMQuvK2a8Nm4j0Mokyg20H3cSMdg6wzXhQ==",
+      "integrity": "sha512-0xfSvwZStmE4nbSkeryBPRM5qen/58U/h+ca9SncPjFHFACtE7zpiCXgRlDb6UvdUD1x5fPnkfa2UKwzmwjMcA==",
       "dev": true,
       "requires": {
         "@babel/code-frame": "^7.12.13",

package.json

@@ -80,7 +80,7 @@
     "eslint-plugin-notice": "^0.9.10",
     "eslint-plugin-react-hooks": "^4.2.0",
     "file-loader": "^6.1.0",
-    "folio": "=0.4.0-alpha19",
+    "folio": "=0.4.0-alpha21",
     "formidable": "^1.2.2",
     "html-webpack-plugin": "^4.4.1",
     "ncp": "^2.0.0",

tests/config/android.config.ts

@@ -25,7 +25,6 @@ const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: folio.Config<CommonOptions & PlaywrightOptions> = {
   testDir,
-  snapshotDir: '__snapshots__',
   outputDir,
   timeout: 120000,
   globalTimeout: 7200000,

tests/config/baseTest.ts

@@ -124,7 +124,7 @@ const baseFixtures: folio.Fixtures<{ __baseSetup: void }, BaseOptions & BaseFixt
   isMac: [ process.platform === 'darwin', { scope: 'worker' } ],
   isLinux: [ process.platform === 'linux', { scope: 'worker' } ],
   __baseSetup: [ async ({ browserName }, run, testInfo) => {
-    testInfo.snapshotPathSegment = browserName;
+    testInfo.snapshotSuffix = browserName;
     await run();
   }, { auto: true } ],
 };

tests/config/default.config.ts

@@ -38,16 +38,15 @@ const pageFixtures = {
   isElectron: false,
 };
 
-const mode = (folio.registerCLIOption('mode', 'Transport mode: default, driver or service').value || 'default') as ('default' | 'driver' | 'service');
+const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service');
-const headed = folio.registerCLIOption('headed', 'Run tests in headed mode (default: headless)', { type: 'boolean' }).value || !!process.env.HEADFUL;
+const headed = !!process.env.HEADFUL;
-const channel = folio.registerCLIOption('channel', 'Browser channel (default: no channel)').value as any;
+const channel = process.env.PWTEST_CHANNEL as any;
-const video = !!folio.registerCLIOption('video', 'Record videos for all tests', { type: 'boolean' }).value;
+const video = !!process.env.PWTEST_VIDEO;
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: folio.Config<CommonOptions & PlaywrightOptions> = {
   testDir,
-  snapshotDir: '__snapshots__',
   outputDir,
   timeout: video || process.env.PWTRACE ? 60000 : 30000,
   globalTimeout: 5400000,

tests/config/electron.config.ts

@@ -25,7 +25,6 @@ const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
 const config: folio.Config<CommonOptions & PlaywrightOptions> = {
   testDir,
-  snapshotDir: '__snapshots__',
   outputDir,
   timeout: 30000,
   globalTimeout: 5400000,

tests/headful.spec.ts

@@ -154,7 +154,7 @@ it('focused input should produce the same screenshot', async ({browserType, brow
   it.skip(browserName === 'webkit' && platform === 'linux', 'gtk vs wpe');
   it.skip(!!process.env.CRPATH);
 
-  testInfo.snapshotPathSegment = browserName + '-' + platform;
+  testInfo.snapshotSuffix = browserName + '-' + platform;
 
   const headful = await browserType.launch({...browserOptions, headless: false });
   const headfulPage = await headful.newPage();