devops(docker): add docker integration smoke tests (#17267)

.github/workflows/tests_primary.yml

@@ -184,3 +184,4 @@ jobs:
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
+

.github/workflows/tests_secondary.yml

@@ -801,3 +801,30 @@ jobs:
     - run: npm run build
     - run: npx playwright install-deps
     - run: utils/build/build-playwright-driver.sh
+
+  smoke_test_docker_integration:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 14
+    - run: npm i -g npm@8
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+    - run: npm run build
+    - run: npx playwright install --with-deps
+    - run: ./utils/docker/build.sh --amd64 focal playwright:localbuild
+    - run: PWTEST_DOCKER_BASE_IMAGE=playwright:localbuild npx playwright docker build
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run dtest
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: always()
+      shell: bash
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: docker-integration-test-results
+        path: test-results
+

package.json

@@ -13,6 +13,7 @@
   },
   "license": "Apache-2.0",
   "scripts": {
+    "dtest": "cross-env PLAYWRIGHT_DOCKER=1 playwright docker test --config=tests/library/playwright.config.ts --grep '@smoke'",
     "ctest": "playwright test --config=tests/library/playwright.config.ts --project=chromium",
     "ftest": "playwright test --config=tests/library/playwright.config.ts --project=firefox",
     "wtest": "playwright test --config=tests/library/playwright.config.ts --project=webkit",
@@ -24,7 +25,6 @@
     "test-html-reporter": "playwright test --config=packages/html-reporter",
     "test-web": "playwright test --config=packages/web",
     "ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts",
-    "vtest": "cross-env PLAYWRIGHT_DOCKER=1 node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts",
     "ct": "playwright test tests/components/test-all.spec.js --reporter=list",
     "test": "playwright test --config=tests/library/playwright.config.ts",
     "eslint": "eslint --ext ts,tsx .",

tests/config/platformFixtures.ts

@@ -24,7 +24,7 @@ export type PlatformWorkerFixtures = {
 };
 
 export const platformTest = test.extend<{}, PlatformWorkerFixtures>({
-  platform: [process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }],
+  platform: [process.env.PLAYWRIGHT_DOCKER ? 'linux' : process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }],
   isWindows: [process.platform === 'win32', { scope: 'worker' }],
   isMac: [process.platform === 'darwin', { scope: 'worker' }],
   isLinux: [process.platform === 'linux', { scope: 'worker' }],

tests/config/testMode.ts

@@ -17,7 +17,7 @@
 import { start } from '../../packages/playwright-core/lib/outofprocess';
 import type { Playwright } from '../../packages/playwright-core/lib/client/playwright';
 
-export type TestModeName = 'default' | 'driver' | 'service' | 'service2';
+export type TestModeName = 'default' | 'driver' | 'service' | 'service2' | 'docker';
 
 interface TestMode {
   setup(): Promise<Playwright>;

tests/config/testModeFixtures.ts

@@ -37,6 +37,7 @@ export const testModeTest = test.extend<TestModeTestFixtures, TestModeWorkerOpti
   playwright: [async ({ mode }, run) => {
     const testMode = {
       default: new DefaultTestMode(),
+      docker: new DefaultTestMode(),
       service: new DefaultTestMode(),
       driver: new DriverTestMode(),
       service2: new DefaultTestMode(),

tests/library/capabilities.spec.ts

@@ -65,7 +65,8 @@ it('should respect CSP @smoke', async ({ page, server }) => {
   expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS');
 });
 
-it('should play video @smoke', async ({ page, asset, browserName, platform }) => {
+it('should play video @smoke', async ({ page, asset, browserName, platform, mode }) => {
+  it.skip(mode === 'docker', 'local paths do not work with remote setup');
   // TODO: the test passes on Windows locally but fails on GitHub Action bot,
   // apparently due to a Media Pack issue in the Windows Server.
   // Also the test is very flaky on Linux WebKit.

tests/library/download.spec.ts

@@ -50,7 +50,8 @@ it.describe('download event', () => {
     });
   });
 
-  it('should report download when navigation turns into download @smoke', async ({ browser, server, browserName }) => {
+  it('should report download when navigation turns into download @smoke', async ({ browser, server, browserName, mode }) => {
+    it.skip(mode === 'docker', 'local paths do not work remote connection');
     const page = await browser.newPage();
     const [download, responseOrError] = await Promise.all([
       page.waitForEvent('download'),

tests/library/logger.spec.ts

@@ -16,7 +16,8 @@
 
 import { playwrightTest as it, expect } from '../config/browserTest';
 
-it('should log @smoke', async ({ browserType }) => {
+it('should log @smoke', async ({ browserType, mode }) => {
+  it.fixme(mode === 'docker', 'logger is not plumbed into the remote connection');
   const log = [];
   const browser = await browserType.launch({ logger: {
     log: (name, severity, message) => log.push({ name, severity, message }),

tests/library/playwright.config.ts

@@ -20,6 +20,7 @@ loadEnv({ path: path.join(__dirname, '..', '..', '.env') });
 import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test';
 import * as path from 'path';
 import type { TestModeWorkerOptions } from '../config/testModeFixtures';
+import type { TestModeName } from '../config/testMode';
 import type { CoverageWorkerOptions } from '../config/coverageFixtures';
 
 type BrowserName = 'chromium' | 'firefox' | 'webkit';
@@ -33,9 +34,13 @@ const getExecutablePath = (browserName: BrowserName) => {
     return process.env.WKPATH;
 };
 
-const mode = process.env.PW_OUT_OF_PROCESS_DRIVER ?
-  'driver' :
-  (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service' | 'service2');
+let mode: TestModeName = 'default';
+if (process.env.PW_OUT_OF_PROCESS_DRIVER)
+  mode = 'driver';
+else if (process.env.PLAYWRIGHT_DOCKER)
+  mode = 'docker';
+else
+  mode = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service' | 'service2');
 const headed = process.argv.includes('--headed');
 const channel = process.env.PWTEST_CHANNEL as any;
 const video = !!process.env.PWTEST_VIDEO;
@@ -129,6 +134,7 @@ for (const browserName of browserNames) {
       metadata: {
         platform: process.platform,
         docker: !!process.env.INSIDE_DOCKER,
+        dockerIntegration: !!process.env.PLAYWRIGHT_DOCKER,
         headful: !!headed,
         browserName,
         channel,

tests/library/proxy.spec.ts

@@ -28,7 +28,8 @@ it('should throw for bad server value', async ({ browserType }) => {
   expect(error.message).toContain('proxy.server: expected string, got number');
 });
 
-it('should use proxy @smoke', async ({ browserType, server }) => {
+it('should use proxy @smoke', async ({ browserType, server, mode }) => {
+  it.skip(mode === 'docker', 'proxy is not supported for remote connection');
   server.setRoute('/target.html', async (req, res) => {
     res.end('<html><title>Served by the proxy</title></html>');
   });

tests/page/page-event-network.spec.ts

@@ -42,7 +42,7 @@ it('Page.Events.Response @smoke', async ({ page, server }) => {
   expect(responses[0].request()).toBeTruthy();
 });
 
-it('Page.Events.RequestFailed @smoke', async ({ page, server, browserName, isMac, isWindows }) => {
+it('Page.Events.RequestFailed @smoke', async ({ page, server, browserName, platform }) => {
   server.setRoute('/one-style.css', (req, res) => {
     res.setHeader('Content-Type', 'text/css');
     res.connection.destroy();
@@ -57,12 +57,12 @@ it('Page.Events.RequestFailed @smoke', async ({ page, server, browserName, isMac
   if (browserName === 'chromium') {
     expect(failedRequests[0].failure().errorText).toBe('net::ERR_EMPTY_RESPONSE');
   } else if (browserName === 'webkit') {
-    if (isMac)
-      expect(failedRequests[0].failure().errorText).toBe('The network connection was lost.');
-    else if (isWindows)
-      expect(failedRequests[0].failure().errorText).toBe('Server returned nothing (no headers, no data)');
-    else
+    if (platform === 'linux')
       expect(failedRequests[0].failure().errorText).toBe('Message Corrupt');
+    else if (platform === 'darwin')
+      expect(failedRequests[0].failure().errorText).toBe('The network connection was lost.');
+    else if (platform === 'win32')
+      expect(failedRequests[0].failure().errorText).toBe('Server returned nothing (no headers, no data)');
   } else {
     expect(failedRequests[0].failure().errorText).toBe('NS_ERROR_NET_RESET');
   }