playwright/.github/workflows/tests.yml

name: tests

on:
  push:
    branches:
      - master
      - release-*
  pull_request:
    branches:
      - master
      - release-*

env:
  # Force terminal colors. @see https://www.npmjs.com/package/colors
  FORCE_COLOR: 1
  FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }}

jobs:
  test_linux:
    name: ${{ matrix.os }} (${{ matrix.browser }})
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
        os: [ubuntu-18.04, ubuntu-20.04]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 10
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: mkdir -p coredumps
    # Set core dump file name pattern
    - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    # Enable core dumps in the subshell.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json && npm run coverage"
      env:
        BROWSER: ${{ matrix.browser }}
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
    - uses: actions/upload-artifact@v1
      if: always()
      with:
        name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
        path: test-results

  test_mac:
    name: ${{ matrix.os }} (${{ matrix.browser }})
    strategy:
      fail-fast: false
      matrix:
        os: [macos-10.15, macos-11.0]
        browser: [chromium, firefox, webkit]
        shard: ['1', '2']
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 10
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json --shard=${{ matrix.shard }}/2
      env:
        BROWSER: ${{ matrix.browser }}
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
        path: test-results

  test_win:
    name: "Windows"
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: windows-latest
    steps:
    - name: Install Media Pack
      shell: powershell
      run: Install-WindowsFeature Server-Media-Foundation
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 10
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
      shell: bash
      env:
        BROWSER: ${{ matrix.browser }}
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
      shell: bash
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: ${{ matrix.browser }}-win-test-results
        path: test-results

  test-package-installations:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        node_version:
        - "^10.17.0" # pre 10.17, --unhandled-rejections=strict was not an option (https://github.com/nodejs/node/pull/26599) which we need in our tests
        - "^12.0.0"
        - "^14.1.0"  # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988)
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: ${{ matrix.node_version }}
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: bash packages/installation-tests/installation-tests.sh

  headful_linux:
    name: "Headful Linux"
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: ubuntu-18.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 10
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: mkdir -p coredumps
    # Set core dump file name pattern
    - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    # Enable core dumps in the subshell.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
      if: ${{ always() }}
      env:
        BROWSER: ${{ matrix.browser }}
        HEADFUL: 1
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: headful-${{ matrix.browser }}-linux-test-results
        path: test-results

  transport_linux:
    name: "Transport"
    strategy:
      fail-fast: false
      matrix:
        mode: [driver, service]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 10
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: mkdir -p coredumps
    # Set core dump file name pattern
    - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    # Enable core dumps in the subshell.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
      env:
        BROWSER: "chromium"
        PWMODE: "${{ matrix.mode }}"
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: mode-${{ matrix.mode }}-linux-test-results
        path: test-results

  video_linux:
    name: "Video Linux"
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: ubuntu-18.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 10
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - run: mkdir -p coredumps
    # Set core dump file name pattern
    - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    # Enable core dumps in the subshell.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json -p video"
      env:
        BROWSER: ${{ matrix.browser }}
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: video-${{ matrix.browser }}-linux-test-results
        path: test-results
  test_android:
    name: Android Emulator
    runs-on: macos-10.15
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v1
      with:
        node-version: 14
    - uses: microsoft/playwright-github-action@v1
    - run: npm ci
    - run: npm run build
    - name: Create Android Emulator
      run: utils/avd_recreate.sh
    - name: Start Android Emulator
      run: utils/avd_start.sh
    - run: npx folio test/android -p browserName=chromium --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json
      env:
        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
        PW_ANDROID_TESTS: 1
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always() && github.ref == 'refs/heads/master'
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: android-test-results
        path: test-results