devops: add bot for Chromium --headless=new (#23440)

https://github.com/microsoft/playwright/issues/23389

.github/workflows/tests_secondary.yml

@@ -665,3 +665,24 @@ jobs:
     - run: npx playwright install-deps
     - run: utils/build/build-playwright-driver.sh
 
+  test_linux_chromium_headless_new:
+    name: Linux Chromium Headless New
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+    - run: npm run build
+    - run: npx playwright install --with-deps chromium
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium
+      env:
+        PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW: 1
+    - run: node tests/config/checkCoverage.js chromium
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: always()
+      shell: bash

packages/playwright-core/src/server/chromium/chromium.ts

@@ -295,8 +295,12 @@ export class Chromium extends BrowserType {
     if (options.devtools)
       chromeArguments.push('--auto-open-devtools-for-tabs');
     if (options.headless) {
+      if (process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW)
+        chromeArguments.push('--headless=new');
+      else
+        chromeArguments.push('--headless');
+
       chromeArguments.push(
-          '--headless',
           '--hide-scrollbars',
           '--mute-audio',
           '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',

tests/library/browsercontext-credentials.spec.ts

@@ -15,10 +15,17 @@
  * limitations under the License.
  */
 
-import { browserTest as it, expect } from '../config/browserTest';
+import { browserTest as base, expect } from '../config/browserTest';
 
-it('should fail without credentials', async ({ browser, server, browserName, headless }) => {
-  it.fail(browserName === 'chromium' && !headless);
+const it = base.extend<{ isChromiumHeadedLike: boolean }>({
+  isChromiumHeadedLike: async ({ browserName, headless }, use) => {
+    const isChromiumHeadedLike = browserName === 'chromium' && (!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW);
+    await use(isChromiumHeadedLike);
+  },
+});
+
+it('should fail without credentials', async ({ browser, server, isChromiumHeadedLike }) => {
+  it.fail(isChromiumHeadedLike);
 
   server.setAuth('/empty.html', 'user', 'pass');
   const context = await browser.newContext();
@@ -28,8 +35,8 @@ it('should fail without credentials', async ({ browser, server, browserName, hea
   await context.close();
 });
 
-it('should work with setHTTPCredentials', async ({ browser, server, browserName, headless }) => {
-  it.fail(browserName === 'chromium' && !headless);
+it('should work with setHTTPCredentials', async ({ browser, server, isChromiumHeadedLike }) => {
+  it.fail(isChromiumHeadedLike);
 
   server.setAuth('/empty.html', 'user', 'pass');
   const context = await browser.newContext();
@@ -99,8 +106,8 @@ it('should work with correct credentials and matching origin case insensitive',
   await context.close();
 });
 
-it('should fail with correct credentials and mismatching scheme', async ({ browser, server, browserName, headless }) => {
-  it.fail(browserName === 'chromium' && !headless);
+it('should fail with correct credentials and mismatching scheme', async ({ browser, server, isChromiumHeadedLike }) => {
+  it.fail(isChromiumHeadedLike);
   server.setAuth('/empty.html', 'user', 'pass');
   const context = await browser.newContext({
     httpCredentials: { username: 'user', password: 'pass', origin: server.PREFIX.replace('http://', 'https://') }
@@ -111,8 +118,8 @@ it('should fail with correct credentials and mismatching scheme', async ({ brows
   await context.close();
 });
 
-it('should fail with correct credentials and mismatching hostname', async ({ browser, server, browserName, headless }) => {
-  it.fail(browserName === 'chromium' && !headless);
+it('should fail with correct credentials and mismatching hostname', async ({ browser, server, isChromiumHeadedLike }) => {
+  it.fail(isChromiumHeadedLike);
   server.setAuth('/empty.html', 'user', 'pass');
   const hostname = new URL(server.PREFIX).hostname;
   const origin = server.PREFIX.replace(hostname, 'mismatching-hostname');
@@ -125,8 +132,8 @@ it('should fail with correct credentials and mismatching hostname', async ({ bro
   await context.close();
 });
 
-it('should fail with correct credentials and mismatching port', async ({ browser, server, browserName, headless }) => {
-  it.fail(browserName === 'chromium' && !headless);
+it('should fail with correct credentials and mismatching port', async ({ browser, server, isChromiumHeadedLike }) => {
+  it.fail(isChromiumHeadedLike);
   server.setAuth('/empty.html', 'user', 'pass');
   const origin = server.PREFIX.replace(server.PORT.toString(), (server.PORT + 1).toString());
   const context = await browser.newContext({

tests/library/download.spec.ts

@@ -635,7 +635,7 @@ it('should be able to download a inline PDF file via response interception', asy
 });
 
 it('should be able to download a inline PDF file via navigation', async ({ browser, server, asset, browserName, headless }) => {
-  it.fixme((!headless && browserName === 'chromium'));
+  it.fixme(((!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW) && browserName === 'chromium'));
   const page = await browser.newPage();
   await page.goto(server.EMPTY_PAGE);
   await page.setContent(`

tests/library/permissions.spec.ts

@@ -152,7 +152,7 @@ it.describe('permissions', () => {
   it('should support clipboard read', async ({ page, context, server, browserName, headless }) => {
     it.fail(browserName === 'webkit');
     it.fail(browserName === 'firefox', 'No such permissions (requires flag) in Firefox');
-    it.fixme(browserName === 'chromium' && !headless);
+    it.fixme(browserName === 'chromium' && (!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW));
 
     await page.goto(server.EMPTY_PAGE);
     expect(await getPermission(page, 'clipboard-read')).toBe('prompt');

tests/library/tracing.spec.ts

@@ -354,7 +354,7 @@ for (const params of [
 ]) {
   browserTest(`should produce screencast frames ${params.id}`, async ({ video, contextFactory, browserName, platform, headless }, testInfo) => {
     browserTest.skip(browserName === 'chromium' && video === 'on', 'Same screencast resolution conflicts');
-    browserTest.fixme(browserName === 'chromium' && !headless, 'Chromium screencast on headed has a min width issue');
+    browserTest.fixme(browserName === 'chromium' && (!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW), 'Chromium screencast on headed has a min width issue');
     browserTest.fixme(params.id === 'fit' && browserName === 'chromium' && platform === 'darwin', 'High DPI maxes image at 600x600');
     browserTest.fixme(params.id === 'fit' && browserName === 'webkit' && platform === 'linux', 'Image size is flaky');
     browserTest.fixme(browserName === 'firefox' && !headless, 'Image size is different');

tests/library/video.spec.ts

@@ -676,7 +676,7 @@ it.describe('screencast', () => {
 
   it('should capture full viewport', async ({ browserType, browserName, headless, isWindows }, testInfo) => {
     it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22411' });
-    it.fixme(browserName === 'chromium' && !headless, 'The square is not on the video');
+    it.fixme(browserName === 'chromium' && (!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW), 'The square is not on the video');
     it.fixme(browserName === 'firefox' && isWindows, 'https://github.com/microsoft/playwright/issues/14405');
     const size = { width: 600, height: 400 };
     const browser = await browserType.launch();
@@ -711,7 +711,7 @@ it.describe('screencast', () => {
 
   it('should capture full viewport on hidpi', async ({ browserType, browserName, headless, isWindows, isLinux }, testInfo) => {
     it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22411' });
-    it.fixme(browserName === 'chromium' && !headless, 'The square is not on the video');
+    it.fixme(browserName === 'chromium' && (!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW), 'The square is not on the video');
     it.fixme(browserName === 'firefox' && isWindows, 'https://github.com/microsoft/playwright/issues/14405');
     it.fixme(browserName === 'webkit' && isLinux, 'https://github.com/microsoft/playwright/issues/22617');
     const size = { width: 600, height: 400 };
@@ -748,7 +748,7 @@ it.describe('screencast', () => {
 
   it('should work with video+trace', async ({ browser, trace, headless }, testInfo) => {
     it.skip(trace === 'on');
-    it.fixme(!headless, 'different trace screencast image size on all browsers');
+    it.fixme(!headless || !!process.env.PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW, 'different trace screencast image size on all browsers');
 
     const size = { width: 500, height: 400 };
     const traceFile = testInfo.outputPath('trace.zip');