AFFiNE/.github/workflows/build-test.yml
2024-11-01 07:05:12 +00:00

750 lines
23 KiB
YAML
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

name: Build & Test
on:
push:
branches:
- canary
- beta
- stable
- v[0-9]+.[0-9]+.x-staging
- v[0-9]+.[0-9]+.x
paths-ignore:
- README.md
pull_request:
env:
DEBUG: napi:*
BUILD_TYPE: canary
APP_NAME: affine
AFFINE_ENV: dev
COVERAGE: true
MACOSX_DEPLOYMENT_TARGET: '10.13'
NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
DEPLOYMENT_TYPE: affine
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: ['javascript', 'typescript']
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
steps:
- name: Checkout repository
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v3
# Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
# If the Autobuild fails above, remove it and uncomment the following three lines.
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
# - run: |
# echo "Run, Build Application using script"
# ./location_of_script_within_repo/buildscript.sh
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Run oxlint
# oxlint is fast, so wrong code will fail quickly
run: yarn dlx $(node -e "console.log(require('./package.json').scripts['lint:ox'].replace('oxlint', 'oxlint@' + require('./package.json').devDependencies.oxlint))")
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
electron-install: false
full-cache: true
- name: Run i18n codegen
run: yarn workspace @affine/i18n build
- name: Run ESLint
run: yarn lint:eslint --max-warnings=0
- name: Run Prettier
# Set nmMode in `actions/setup-node` will modify the .yarnrc.yml
run: |
git checkout .yarnrc.yml
yarn lint:prettier
- name: Yarn Dedupe
run: yarn dedupe --check
- name: Run Type Check
run: yarn typecheck
check-yarn-binary:
name: Check yarn binary
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Run check
run: |
yarn set version $(node -e "console.log(require('./package.json').packageManager.split('@')[1])")
git diff --exit-code
e2e-test:
name: E2E Test
runs-on: ubuntu-latest
env:
DISTRIBUTION: web
IN_CI_TEST: true
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5]
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
playwright-install: true
electron-install: false
full-cache: true
- name: Run playwright tests
run: yarn workspace @affine-test/affine-local e2e --forbid-only --shard=${{ matrix.shard }}/${{ strategy.job-total }}
- name: Upload test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: test-results-e2e-${{ matrix.shard }}
path: ./test-results
if-no-files-found: ignore
e2e-mobile-test:
name: E2E Mobile Test
runs-on: ubuntu-latest
env:
DISTRIBUTION: mobile
IN_CI_TEST: true
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5]
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
playwright-install: true
electron-install: false
full-cache: true
- name: Run playwright tests
run: yarn workspace @affine-test/affine-mobile e2e --forbid-only --shard=${{ matrix.shard }}/${{ strategy.job-total }}
- name: Upload test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: test-results-e2e-mobile-${{ matrix.shard }}
path: ./test-results
if-no-files-found: ignore
e2e-migration-test:
name: E2E Migration Test
runs-on: ubuntu-latest
env:
DISTRIBUTION: web
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
playwright-install: true
electron-install: false
full-cache: true
- name: Run playwright tests
run: yarn workspace @affine-test/affine-migration e2e --forbid-only
- name: Upload test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: test-results-e2e-migration
path: ./test-results
if-no-files-found: ignore
unit-test:
name: Unit Test
runs-on: ubuntu-latest
needs:
- build-native
env:
DISTRIBUTION: web
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
electron-install: true
full-cache: true
- name: Download affine.linux-x64-gnu.node
uses: actions/download-artifact@v4
with:
name: affine.linux-x64-gnu.node
path: ./packages/frontend/native
- name: Unit Test
run: yarn nx test:coverage @affine/monorepo
- name: Upload unit test coverage results
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./.coverage/store/lcov.info
flags: unittest
name: affine
fail_ci_if_error: false
build-native:
name: Build AFFiNE native (${{ matrix.spec.target }})
runs-on: ${{ matrix.spec.os }}
env:
CARGO_PROFILE_RELEASE_DEBUG: '1'
strategy:
fail-fast: false
matrix:
spec:
- { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
- { os: windows-latest, target: x86_64-pc-windows-msvc }
- { os: macos-14, target: x86_64-apple-darwin }
- { os: macos-14, target: aarch64-apple-darwin }
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
extra-flags: workspaces focus @affine/native
electron-install: false
- name: Setup filename
id: filename
shell: bash
run: |
export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)")
echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT"
- name: Build AFFiNE native
uses: ./.github/actions/build-rust
with:
target: ${{ matrix.spec.target }}
package: '@affine/native'
nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
- name: Upload ${{ steps.filename.outputs.filename }}
uses: actions/upload-artifact@v4
with:
name: ${{ steps.filename.outputs.filename }}
path: ./packages/frontend/native/${{ steps.filename.outputs.filename }}
if-no-files-found: error
build-server-native:
name: Build Server native
runs-on: ubuntu-latest
env:
CARGO_PROFILE_RELEASE_DEBUG: '1'
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
extra-flags: workspaces focus @affine/server-native
electron-install: false
- name: Build Rust
uses: ./.github/actions/build-rust
with:
target: 'x86_64-unknown-linux-gnu'
package: '@affine/server-native'
nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
- name: Upload server-native.node
uses: actions/upload-artifact@v4
with:
name: server-native.node
path: ./packages/backend/native/server-native.node
if-no-files-found: error
build-electron-renderer:
name: Build @affine/electron renderer
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
electron-install: false
full-cache: true
- name: Build Electron renderer
# always skip cache because its fast, and cache configuration is always changing
run: yarn build
env:
DISTRIBUTION: desktop
- name: zip web
run: tar -czf dist.tar.gz --directory=packages/frontend/apps/electron/renderer/dist .
- name: Upload web artifact
uses: actions/upload-artifact@v4
with:
name: web
path: dist.tar.gz
if-no-files-found: error
server-test:
name: Server Test
runs-on: ubuntu-latest
needs: build-server-native
env:
NODE_ENV: test
DISTRIBUTION: web
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: affine
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
mailer:
image: mailhog/mailhog
ports:
- 1025:1025
- 8025:8025
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
electron-install: false
full-cache: true
- name: Download server-native.node
uses: actions/download-artifact@v4
with:
name: server-native.node
path: ./packages/backend/server
- name: Prepare Server Test Environment
uses: ./.github/actions/server-test-env
- name: Run server tests
run: yarn workspace @affine/server test:coverage
env:
CARGO_TARGET_DIR: '${{ github.workspace }}/target'
COPILOT_OPENAI_API_KEY: 'use_fake_openai_api_key'
- name: Upload server test coverage results
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./packages/backend/server/.coverage/lcov.info
flags: server-test
name: affine
fail_ci_if_error: false
copilot-api-test:
name: Server Copilot Api Test
runs-on: ubuntu-latest
needs:
- build-server-native
env:
NODE_ENV: test
DISTRIBUTION: web
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: affine
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
mailer:
image: mailhog/mailhog
ports:
- 1025:1025
- 8025:8025
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
electron-install: false
full-cache: true
- name: Download server-native.node
uses: actions/download-artifact@v4
with:
name: server-native.node
path: ./packages/backend/server
- name: Prepare Server Test Environment
uses: ./.github/actions/server-test-env
- name: Run server tests
run: yarn workspace @affine/server test:copilot:coverage --forbid-only
env:
CARGO_TARGET_DIR: '${{ github.workspace }}/target'
COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
- name: Upload server test coverage results
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./packages/backend/server/.coverage/lcov.info
flags: server-test
name: affine
fail_ci_if_error: false
copilot-e2e-test:
name: Server Copilot E2E Test
runs-on: ubuntu-latest
env:
DISTRIBUTION: web
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
IN_CI_TEST: true
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2, 3]
shardTotal: [3]
needs:
- build-server-native
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: affine
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- uses: actions/checkout@v4
- name: Check blocksuite update
id: check-blocksuite-update
env:
BASE_REF: ${{ github.base_ref }}
run: |
if [ -n "$BASE_REF" ]; then
if node ./scripts/detect-blocksuite-update.mjs "$BASE_REF"; then
echo "skip=false" >> $GITHUB_OUTPUT
else
echo "skip=true" >> $GITHUB_OUTPUT
fi
else
echo "skip=true" >> $GITHUB_OUTPUT
fi
- name: Setup Node.js
if: ${{ steps.check-blocksuite-update.outputs.skip == 'true' }}
uses: ./.github/actions/setup-node
with:
playwright-install: true
electron-install: false
hard-link-nm: false
- name: Download server-native.node
if: ${{ steps.check-blocksuite-update.outputs.skip == 'true' }}
uses: actions/download-artifact@v4
with:
name: server-native.node
path: ./packages/backend/server
- name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
if: ${{ steps.check-blocksuite-update.outputs.skip == 'true' }}
uses: ./.github/actions/copilot-test
with:
script: yarn workspace @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
server-e2e-test:
name: ${{ matrix.tests.name }}
runs-on: ubuntu-latest
env:
DISTRIBUTION: web
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
IN_CI_TEST: true
strategy:
fail-fast: false
matrix:
tests:
- name: 'Server E2E Test 1/3'
script: yarn workspace @affine-test/affine-cloud e2e --forbid-only --shard=1/3
- name: 'Server E2E Test 2/3'
script: yarn workspace @affine-test/affine-cloud e2e --forbid-only --shard=2/3
- name: 'Server E2E Test 3/3'
script: yarn workspace @affine-test/affine-cloud e2e --forbid-only --shard=3/3
- name: 'Server Desktop E2E Test'
script: |
yarn workspace @affine/electron build:dev
# Workaround for Electron apps failing to initialize on Ubuntu 24.04 due to AppArmor restrictions
# Disables unprivileged user namespaces restriction to allow Electron apps to run
# Reference: https://github.com/electron/electron/issues/42510
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop-cloud e2e
needs:
- build-server-native
- build-native
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: affine
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
mailer:
image: mailhog/mailhog
ports:
- 1025:1025
- 8025:8025
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
playwright-install: true
hard-link-nm: false
- name: Download server-native.node
uses: actions/download-artifact@v4
with:
name: server-native.node
path: ./packages/backend/server
- name: Download affine.linux-x64-gnu.node
uses: actions/download-artifact@v4
with:
name: affine.linux-x64-gnu.node
path: ./packages/frontend/native
- name: Prepare Server Test Environment
uses: ./.github/actions/server-test-env
- name: ${{ matrix.tests.name }}
run: |
${{ matrix.tests.script }}
env:
DEV_SERVER_URL: http://localhost:8080
COPILOT_OPENAI_API_KEY: 1
COPILOT_FAL_API_KEY: 1
- name: Upload test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: test-results-e2e-server
path: ./test-results
if-no-files-found: ignore
desktop-test:
name: Desktop Test (${{ matrix.spec.os }}, ${{ matrix.spec.platform }}, ${{ matrix.spec.arch }}, ${{ matrix.spec.target }}, ${{ matrix.spec.test }})
runs-on: ${{ matrix.spec.os }}
strategy:
fail-fast: false
matrix:
spec:
- {
os: macos-14,
platform: macos,
arch: x64,
target: x86_64-apple-darwin,
test: false,
}
- {
os: macos-14,
platform: macos,
arch: arm64,
target: aarch64-apple-darwin,
test: true,
}
- {
os: ubuntu-latest,
platform: linux,
arch: x64,
target: x86_64-unknown-linux-gnu,
test: true,
}
- {
os: windows-latest,
platform: windows,
arch: x64,
target: x86_64-pc-windows-msvc,
test: true,
}
needs:
- build-electron-renderer
- build-native
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
timeout-minutes: 10
with:
extra-flags: workspaces focus @affine/electron @affine/monorepo @affine-test/affine-desktop
playwright-install: true
hard-link-nm: false
enableScripts: false
- name: Setup filename
id: filename
shell: bash
run: |
export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)")
echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT"
- name: Download ${{ steps.filename.outputs.filename }}
uses: actions/download-artifact@v4
with:
name: ${{ steps.filename.outputs.filename }}
path: ./packages/frontend/native
- name: Run unit tests
if: ${{ matrix.spec.test }}
shell: bash
run: yarn workspace @affine/electron vitest
- name: Download web artifact
uses: ./.github/actions/download-web
with:
path: packages/frontend/apps/electron/resources/web-static
- name: Build Desktop Layers
run: yarn workspace @affine/electron build
- name: Run desktop tests
if: ${{ matrix.spec.os == 'ubuntu-latest' }}
run: |
# Workaround for Electron apps failing to initialize on Ubuntu 24.04 due to AppArmor restrictions
# Disables unprivileged user namespaces restriction to allow Electron apps to run
# Reference: https://github.com/electron/electron/issues/42510
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop e2e
- name: Run desktop tests
if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-latest' }}
run: yarn workspace @affine-test/affine-desktop e2e
- name: Make bundle (macOS)
if: ${{ matrix.spec.target == 'aarch64-apple-darwin' }}
env:
SKIP_BUNDLE: true
SKIP_WEB_BUILD: true
HOIST_NODE_MODULES: 1
run: yarn workspace @affine/electron package --platform=darwin --arch=arm64
- name: Make Bundle (Linux)
run: |
sudo add-apt-repository universe
sudo apt install -y libfuse2 elfutils flatpak flatpak-builder
flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
flatpak update
# some flatpak deps need git protocol.file.allow
git config --global protocol.file.allow always
yarn workspace @affine/electron make --platform=linux --arch=x64
if: ${{ matrix.spec.target == 'x86_64-unknown-linux-gnu' }}
env:
SKIP_WEB_BUILD: 1
HOIST_NODE_MODULES: 1
- name: Output check
if: ${{ matrix.spec.os == 'macos-14' && matrix.spec.arch == 'arm64' }}
run: |
yarn workspace @affine/electron exec node --loader ts-node/esm/transpile-only ./scripts/macos-arm64-output-check.ts
- name: Upload test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: test-results-e2e-${{ matrix.spec.os }}-${{ matrix.spec.arch }}
path: ./test-results
if-no-files-found: ignore
test-build-mobile-app:
uses: ./.github/workflows/release-mobile.yml
with:
build-type: canary
build-target: development
secrets: inherit
permissions:
id-token: 'write'
test-done:
needs:
- analyze
- lint
- check-yarn-binary
- e2e-test
- e2e-mobile-test
- e2e-migration-test
- unit-test
- build-native
- build-server-native
- build-electron-renderer
- server-test
- copilot-e2e-test
- server-e2e-test
- desktop-test
- test-build-mobile-app
if: always()
runs-on: ubuntu-latest
name: 3, 2, 1 Launch
steps:
- run: exit 1
# Thank you, next https://github.com/vercel/next.js/blob/canary/.github/workflows/build_and_test.yml#L379
if: ${{ always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}