name: "tests 2" on: push: branches: - main - release-* pull_request: paths-ignore: - 'browser_patches/**' - 'docs/**' types: [ labeled ] branches: - main - release-* env: # Force terminal colors. @see https://www.npmjs.com/package/colors FORCE_COLOR: 1 FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }} jobs: test_linux: name: ${{ matrix.os }} (${{ matrix.browser }}) strategy: fail-fast: false matrix: browser: [chromium, firefox, webkit] os: [ubuntu-18.04] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: DEBUG: pw:install PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} - run: node tests/config/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: always() with: name: ${{ matrix.browser }}-${{ matrix.os }}-test-results path: test-results test_mac: name: ${{ matrix.os }} (${{ matrix.browser }}) strategy: fail-fast: false matrix: os: [macos-10.15, macos-11.0] browser: [chromium, firefox, webkit] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: DEBUG: pw:install PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: npm run test -- --project=${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: ${{ matrix.browser }}-${{ matrix.os }}-test-results path: test-results test_win: name: "Windows" strategy: fail-fast: false matrix: browser: [chromium, firefox, webkit] runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: DEBUG: pw:install PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: npm run test -- --project=${{ matrix.browser }} shell: bash - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: ${{ matrix.browser }}-win-test-results path: test-results test-package-installations: runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: node_version: - "12.0.0" - "^12.0.0" - "^14.1.0" # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988) - "^16.0.0" timeout-minutes: 20 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: ${{ matrix.node_version }} # NPM 7 is the latest version which supports Node.js 12.0.0 - run: npm i -g npm@7 - run: npm ci env: DEBUG: pw:install - run: npm run build - run: npx playwright install-deps - name: INSTALLATION TESTS run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash installation-tests/run_all_tests.sh headful_linux: name: "Headful Linux" strategy: fail-fast: false matrix: browser: [chromium, firefox, webkit] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: DEBUG: pw:install PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} if: ${{ always() }} env: HEADFUL: 1 - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: headful-${{ matrix.browser }}-linux-test-results path: test-results transport_linux: name: "Transport" strategy: fail-fast: false matrix: mode: [driver, service] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: DEBUG: pw:install PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_MODE: ${{ matrix.mode }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: mode-${{ matrix.mode }}-linux-test-results path: test-results tracing_linux: name: "Tracing" strategy: fail-fast: false matrix: browser: [chromium, firefox, webkit] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: DEBUG: pw:install PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} env: PWTEST_TRACE: 1 - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() chrome_stable_linux: name: "Chrome Stable (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chrome - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: chrome - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: chrome-stable-linux-test-results path: test-results chrome_stable_win: name: "Chrome Stable (Win)" runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chrome - run: npm run ctest shell: bash env: PWTEST_CHANNEL: chrome - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: chrome-stable-win-test-results path: test-results chrome_stable_mac: name: "Chrome Stable (Mac)" runs-on: macos-10.15 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chrome - run: npm run ctest env: PWTEST_CHANNEL: chrome - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: chrome-stable-mac-test-results path: test-results firefox_beta_linux: name: "Firefox Beta (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps firefox-beta chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ftest env: PWTEST_CHANNEL: firefox-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: firefox-beta-linux-test-results path: test-results firefox_beta_win: name: "Firefox Beta (Win)" runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps firefox-beta chromium - run: npm run ftest shell: bash env: PWTEST_CHANNEL: firefox-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: firefox-beta-win-test-results path: test-results firefox_beta_mac: name: "Firefox Beta (Mac)" runs-on: macos-10.15 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps firefox-beta chromium - run: npm run ftest env: PWTEST_CHANNEL: firefox-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: firefox-beta-mac-test-results path: test-results edge_stable_mac: name: "Edge Stable (Mac)" runs-on: macos-10.15 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge - run: npm run ctest env: PWTEST_CHANNEL: msedge - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: msedge-stable-mac-test-results path: test-results edge_stable_win: name: "Edge Stable (Win)" runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge - run: npm run ctest shell: bash env: PWTEST_CHANNEL: msedge - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: edge-stable-win-test-results path: test-results edge_stable_linux: name: "Edge Stable (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: msedge - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: edge-stable-linux-test-results path: test-results edge_beta_mac: name: "Edge Beta (Mac)" runs-on: macos-10.15 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge-beta - run: npm run ctest env: PWTEST_CHANNEL: msedge-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: msedge-beta-mac-test-results path: test-results edge_beta_win: name: "Edge Beta (Win)" runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge-beta - run: npm run ctest shell: bash env: PWTEST_CHANNEL: msedge-beta - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: edge-beta-win-test-results path: test-results edge_beta_linux: name: "Edge Beta (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge-beta - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: msedge-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: edge-beta-linux-test-results path: test-results edge_dev_mac: name: "Edge Dev (Mac)" runs-on: macos-10.15 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge-dev - run: npm run ctest env: PWTEST_CHANNEL: msedge-dev - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: msedge-dev-mac-test-results path: test-results edge_dev_win: name: "Edge Dev (Win)" runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge-dev - run: npm run ctest shell: bash env: PWTEST_CHANNEL: msedge-dev - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: edge-dev-win-test-results path: test-results edge_dev_linux: name: "Edge Dev (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps msedge-dev - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: msedge-dev - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: edge-dev-linux-test-results path: test-results chrome_beta_linux: name: "Chrome Beta (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chrome-beta - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: chrome-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: chrome-beta-linux-test-results path: test-results chrome_beta_win: name: "Chrome Beta (Win)" runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chrome-beta - run: npm run ctest shell: bash env: PWTEST_CHANNEL: chrome-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: chrome-beta-win-test-results path: test-results chrome_beta_mac: name: "Chrome Beta (Mac)" runs-on: macos-10.15 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chrome-beta - run: npm run ctest env: PWTEST_CHANNEL: chrome-beta - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: chrome-beta-mac-test-results path: test-results test_electron: name: "Electron Linux" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run etest - run: node tests/config/checkCoverage.js electron - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() - uses: actions/upload-artifact@v1 if: ${{ always() }} with: name: electron-linux-test-results path: test-results build-playwright-driver: name: "build-playwright-driver" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: 12 - run: npm i -g npm@8.3 - run: npm ci - run: npm run build - run: npx playwright install-deps - run: utils/build/build-playwright-driver.sh