test: roll to folio 0.3.21-alpha (#6108)

.github/workflows/tests.yml

@@ -14,7 +14,6 @@ env:
   # Force terminal colors. @see https://www.npmjs.com/package/colors
   FORCE_COLOR: 1
   FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }}
-  FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
 
 jobs:
   test_linux:
@@ -39,7 +38,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- --tag=${{ matrix.browser }}"
     - run: node tests/config/checkCoverage.js ${{ matrix.browser }}
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
@@ -65,7 +64,7 @@ jobs:
     - run: npm ci
     - run: npm run build
     - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
-    - run: npm run test -- ${{ matrix.browser }} --reporter=dot,json
+    - run: npm run test -- --tag=${{ matrix.browser }}
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
     - uses: actions/upload-artifact@v1
@@ -92,7 +91,7 @@ jobs:
     - run: npm ci
     - run: npm run build
     - run: node lib/cli/cli install-deps
-    - run: npm run test -- ${{ matrix.browser }} --reporter=dot,json
+    - run: npm run test -- --tag=${{ matrix.browser }}
       shell: bash
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
@@ -143,7 +142,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- --tag=${{ matrix.browser }}"
       if: ${{ always() }}
       env:
         HEADFUL: 1
@@ -176,7 +175,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- chromium --reporter=dot,json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run ctest"
       env:
         PWMODE: "${{ matrix.mode }}"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@@ -208,7 +207,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- ${{ matrix.browser }} --reporter=dot,json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- --tag=${{ matrix.browser }}"
       env:
         PWVIDEO: 1
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@@ -241,7 +240,7 @@ jobs:
     - name: Start Android Emulator
       run: utils/avd_start.sh
     - name: Run tests
-      run: npm run atest -- --reporter=dot,json --shard=${{ matrix.shard }}/2
+      run: npm run atest -- --shard=${{ matrix.shard }}/2
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
     - uses: actions/upload-artifact@v1
@@ -273,7 +272,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- chromium --reporter=dot,json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run ctest"
       env:
         PW_CHROMIUM_CHANNEL: "chrome"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@@ -301,7 +300,7 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run test -- chromium --reporter=dot,json
+    - run: npm run ctest
       shell: bash
       env:
         PW_CHROMIUM_CHANNEL: "chrome"
@@ -328,7 +327,7 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run test -- chromium --reporter=dot,json
+    - run: npm run ctest
       env:
         PW_CHROMIUM_CHANNEL: "chrome"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@@ -356,7 +355,7 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run test -- chromium --reporter=dot,json
+    - run: npm run ctest
       shell: bash
       env:
         PW_CHROMIUM_CHANNEL: "msedge"

package-lock.json

@@ -8131,9 +8131,9 @@
       }
     },
     "folio": {
-      "version": "0.3.20-alpha",
+      "version": "0.3.21-alpha",
-      "resolved": "https://registry.npmjs.org/folio/-/folio-0.3.20-alpha.tgz",
+      "resolved": "https://registry.npmjs.org/folio/-/folio-0.3.21-alpha.tgz",
-      "integrity": "sha512-uJWYgfLa1l91NSbBV2pxKR7g91/Ti7cLuvfVI6M9uI5H7JaDhEEgPsLM9RG7rwoJ7AtIOFRJSbdMIwyRwvzWxw==",
+      "integrity": "sha512-gCP36QBOAfVWOWydY2vJi32mV+7mUYr+o0c+Jw25edAfZCZndkztIMpvB9dXAKct1g5f7wY5n1bTBH0ZunMWXg==",
       "dev": true,
       "requires": {
         "@babel/code-frame": "^7.10.4",

package.json

@@ -9,9 +9,9 @@
     "node": ">=10.17.0"
   },
   "scripts": {
-    "ctest": "folio --config=tests/config/default.config.ts chromium",
+    "ctest": "folio --config=tests/config/default.config.ts --tag=chromium",
-    "ftest": "folio --config=tests/config/default.config.ts firefox",
+    "ftest": "folio --config=tests/config/default.config.ts --tag=firefox",
-    "wtest": "folio --config=tests/config/default.config.ts webkit",
+    "wtest": "folio --config=tests/config/default.config.ts --tag=webkit",
     "atest": "folio --config=tests/config/android.config.ts",
     "test": "folio --config=tests/config/default.config.ts",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
@@ -79,7 +79,7 @@
     "eslint-plugin-notice": "^0.9.10",
     "eslint-plugin-react-hooks": "^4.2.0",
     "file-loader": "^6.1.0",
-    "folio": "=0.3.20-alpha",
+    "folio": "=0.3.21-alpha",
     "formidable": "^1.2.2",
     "html-webpack-plugin": "^4.4.1",
     "ncp": "^2.0.0",

tests/config/android.config.ts

@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-import { setConfig, Config } from 'folio';
+import * as folio from 'folio';
 import * as path from 'path';
 import { test as pageTest } from './pageTest';
 import { test as androidTest } from './androidTest';
 import { ServerEnv } from './serverEnv';
 import { AndroidEnv, AndroidPageEnv } from './androidEnv';
 
-const config: Config = {
+const config: folio.Config = {
   testDir: path.join(__dirname, '..'),
   timeout: 120000,
   globalTimeout: 7200000,
@@ -31,8 +31,15 @@ if (process.env.CI) {
   config.forbidOnly = true;
   config.retries = 1;  // Multiple retries are too slow on Android.
 }
-setConfig(config);
+folio.setConfig(config);
+
+if (process.env.CI) {
+  folio.setReporters([
+    new folio.reporters.dot(),
+    new folio.reporters.json({ outputFile: path.join(__dirname, '..', 'test-results', 'report.json') }),
+  ]);
+}
 
 const serverEnv = new ServerEnv();
-pageTest.runWith('android', serverEnv, new AndroidPageEnv(), {});
+pageTest.runWith(folio.merge(serverEnv, new AndroidPageEnv()), { tag: 'android' });
-androidTest.runWith('android', serverEnv, new AndroidEnv(), {});
+androidTest.runWith(folio.merge(serverEnv, new AndroidEnv()), { tag: 'android' });

tests/config/default.config.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { setConfig, Config } from 'folio';
+import * as folio from 'folio';
 import * as path from 'path';
 import { test as playwrightTest, slowTest as playwrightSlowTest } from './playwrightTest';
 import { test as browserTest, slowTest as browserSlowTest } from './browserTest';
@@ -27,7 +27,7 @@ import { ServerEnv } from './serverEnv';
 import { ElectronEnv } from './electronEnv';
 import { CLIEnv } from './cliEnv';
 
-const config: Config = {
+const config: folio.Config = {
   testDir: path.join(__dirname, '..'),
   timeout: process.env.PWVIDEO ? 60000 : 30000,
   globalTimeout: 5400000,
@@ -37,7 +37,14 @@ if (process.env.CI) {
   config.forbidOnly = true;
   config.retries = 3;
 }
-setConfig(config);
+folio.setConfig(config);
+
+if (process.env.CI) {
+  folio.setReporters([
+    new folio.reporters.dot(),
+    new folio.reporters.json({ outputFile: path.join(__dirname, '..', 'test-results', 'report.json') }),
+  ]);
+}
 
 const getExecutablePath = (browserName: BrowserName) => {
   if (browserName === 'chromium' && process.env.CRPATH)
@@ -64,14 +71,14 @@ for (const browserName of browsers) {
     channel: process.env.PW_CHROMIUM_CHANNEL as any,
     video: !!process.env.PWVIDEO,
   };
-  playwrightTest.runWith(browserName, serverEnv, new PlaywrightEnv(browserName, options), {});
+  playwrightTest.runWith(folio.merge(serverEnv, new PlaywrightEnv(browserName, options)), { tag: browserName });
-  playwrightSlowTest.runWith(browserName, serverEnv, new PlaywrightEnv(browserName, options), { timeout: config.timeout * 3 });
+  playwrightSlowTest.runWith(folio.merge(serverEnv, new PlaywrightEnv(browserName, options)), { timeout: config.timeout * 3, tag: browserName });
-  browserTest.runWith(browserName, serverEnv, new BrowserEnv(browserName, options), {});
+  browserTest.runWith(folio.merge(serverEnv, new BrowserEnv(browserName, options)), { tag: browserName });
-  browserSlowTest.runWith(browserName, serverEnv, new BrowserEnv(browserName, options), { timeout: config.timeout * 3 });
+  browserSlowTest.runWith(folio.merge(serverEnv, new BrowserEnv(browserName, options)), { timeout: config.timeout * 3, tag: browserName });
-  pageTest.runWith(browserName, serverEnv, new PageEnv(browserName, options), {});
+  pageTest.runWith(folio.merge(serverEnv, new PageEnv(browserName, options)), { tag: browserName });
-  contextTest.runWith(browserName, serverEnv, new PageEnv(browserName, options), {});
+  contextTest.runWith(folio.merge(serverEnv, new PageEnv(browserName, options)), { tag: browserName });
   if (mode !== 'service')
-    cliTest.runWith(browserName, serverEnv, new CLIEnv(browserName, options), {});
+    cliTest.runWith(folio.merge(serverEnv, new CLIEnv(browserName, options)), { tag: browserName });
   if (browserName === 'chromium')
-    electronTest.runWith(browserName, serverEnv, new ElectronEnv({ mode }));
+    electronTest.runWith(folio.merge(serverEnv, new ElectronEnv({ mode })), { tag: browserName });
 }