From f69649c9224fb79aa81b5381c960ce09557e2a5d Mon Sep 17 00:00:00 2001 From: forehalo Date: Thu, 28 Mar 2024 04:02:13 +0000 Subject: [PATCH] ci: separate image build to a standalone workflow (#6167) --- .github/workflows/build-selfhost-image.yml | 25 +++ .github/workflows/build-server-image.yml | 191 +++++++++++++++++++++ .github/workflows/deploy.yml | 179 ++----------------- 3 files changed, 232 insertions(+), 163 deletions(-) create mode 100644 .github/workflows/build-selfhost-image.yml create mode 100644 .github/workflows/build-server-image.yml diff --git a/.github/workflows/build-selfhost-image.yml b/.github/workflows/build-selfhost-image.yml new file mode 100644 index 0000000000..b8fc83446d --- /dev/null +++ b/.github/workflows/build-selfhost-image.yml @@ -0,0 +1,25 @@ +name: Build Selfhost Image + +on: + workflow_dispatch: + inputs: + flavor: + description: 'Select distribution to build' + type: choice + default: canary + options: + - canary + - beta + - stable + +permissions: + contents: 'write' + id-token: 'write' + packages: 'write' + +jobs: + build-image: + name: Build Image + uses: ./.github/workflows/build-server-image.yml + with: + flavor: ${{ github.event.inputs.flavor }} diff --git a/.github/workflows/build-server-image.yml b/.github/workflows/build-server-image.yml new file mode 100644 index 0000000000..6eeeec4fbd --- /dev/null +++ b/.github/workflows/build-server-image.yml @@ -0,0 +1,191 @@ +name: Build Images + +on: + workflow_call: + inputs: + flavor: + type: string + required: true + +env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + +permissions: + contents: 'write' + id-token: 'write' + packages: 'write' + +jobs: + build-server: + name: Build Server + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + electron-install: false + extra-flags: workspaces focus @affine/server + - name: Build Server + run: yarn workspace @affine/server build + - name: Upload server dist + uses: actions/upload-artifact@v4 + with: + name: server-dist + path: ./packages/backend/server/dist + if-no-files-found: error + + build-web-selfhost: + name: Build @affine/web selfhost + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.flavor }} + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + - name: Build Core + run: yarn nx build @affine/web --skip-nx-cache + env: + BUILD_TYPE: ${{ github.event.inputs.flavor }} + SHOULD_REPORT_TRACE: false + PUBLIC_PATH: '/' + SELF_HOSTED: true + - name: Download selfhost fonts + run: node ./scripts/download-blocksuite-fonts.mjs + - name: Upload web artifact + uses: actions/upload-artifact@v4 + with: + name: selfhost-web + path: ./packages/frontend/web/dist + if-no-files-found: error + + build-storage: + name: Build Storage - ${{ matrix.targets.name }} + runs-on: ubuntu-latest + strategy: + matrix: + targets: + - name: x86_64-unknown-linux-gnu + file: storage.node + - name: aarch64-unknown-linux-gnu + file: storage.arm64.node + - name: armv7-unknown-linux-gnueabihf + file: storage.armv7.node + + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + electron-install: false + extra-flags: workspaces focus @affine/storage + - name: Build Rust + uses: ./.github/actions/build-rust + with: + target: ${{ matrix.targets.name }} + package: '@affine/storage' + nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + - name: Upload ${{ matrix.targets.file }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.targets.file }} + path: ./packages/backend/storage/storage.node + if-no-files-found: error + + build-docker: + name: Build Docker + runs-on: ubuntu-latest + needs: + - build-server + - build-web-selfhost + - build-storage + steps: + - uses: actions/checkout@v4 + - name: Download server dist + uses: actions/download-artifact@v4 + with: + name: server-dist + path: ./packages/backend/server/dist + - name: Download storage.node + uses: actions/download-artifact@v4 + with: + name: storage.node + path: ./packages/backend/server + - name: Download storage.node arm64 + uses: actions/download-artifact@v4 + with: + name: storage.arm64.node + path: ./packages/backend/storage + - name: Download storage.node arm64 + uses: actions/download-artifact@v4 + with: + name: storage.armv7.node + path: . + - name: move storage files + run: | + mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node + mv storage.node ./packages/backend/server/storage.armv7.node + - name: Setup env + run: | + echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + if [ -z "${{ inputs.flavor }}" ] + then + echo "RELEASE_FLAVOR=canary" >> "$GITHUB_ENV" + else + echo "RELEASE_FLAVOR=${{ inputs.flavor }}" >> "$GITHUB_ENV" + fi + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + logout: false + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # setup node without cache configuration + # Prisma cache is not compatible with docker build cache + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + registry-url: https://npm.pkg.github.com + scope: '@toeverything' + + - name: Download selfhost web artifact + uses: actions/download-artifact@v4 + with: + name: selfhost-web + path: ./packages/frontend/web/dist + + - name: Install Node.js dependencies + run: | + yarn config set --json supportedArchitectures.cpu '["x64", "arm64", "arm"]' + yarn config set --json supportedArchitectures.libc '["glibc"]' + yarn workspaces focus @affine/server --production + + - name: Generate Prisma client + run: yarn workspace @affine/server prisma generate + + - name: Build graphql Dockerfile + uses: docker/build-push-action@v5 + with: + context: . + push: true + pull: true + platforms: linux/amd64,linux/arm64,linux/arm/v7 + provenance: true + file: .github/deployment/node/Dockerfile + tags: ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fb47d43e88..74c287d51e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,32 +13,21 @@ on: - stable - internal env: - APP_NAME: affine NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} MIXPANEL_TOKEN: '389c0615a69b57cca7d3fa0a4824c930' +permissions: + contents: 'write' + id-token: 'write' + packages: 'write' + jobs: - build-server: - name: Build Server - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Version - id: version - uses: ./.github/actions/setup-version - - name: Setup Node.js - uses: ./.github/actions/setup-node - with: - electron-install: false - extra-flags: workspaces focus @affine/server - - name: Build Server - run: yarn workspace @affine/server build - - name: Upload server dist - uses: actions/upload-artifact@v4 - with: - name: server-dist - path: ./packages/backend/server/dist - if-no-files-found: error + build-server-image: + name: Build Server Image + uses: ./.github/workflows/build-server-image.yml + with: + flavor: ${{ github.event.inputs.flavor }} + build-web: name: Build @affine/web runs-on: ubuntu-latest @@ -72,112 +61,17 @@ jobs: path: ./packages/frontend/web/dist if-no-files-found: error - build-web-selfhost: - name: Build @affine/web selfhost + build-frontend-image: + name: Build Frontend Image runs-on: ubuntu-latest - environment: ${{ github.event.inputs.flavor }} - steps: - - uses: actions/checkout@v4 - - name: Setup Version - id: version - uses: ./.github/actions/setup-version - - name: Setup Node.js - uses: ./.github/actions/setup-node - - name: Build Core - run: yarn nx build @affine/web --skip-nx-cache - env: - BUILD_TYPE: ${{ github.event.inputs.flavor }} - SHOULD_REPORT_TRACE: false - PUBLIC_PATH: '/' - SELF_HOSTED: true - - name: Download selfhost fonts - run: node ./scripts/download-blocksuite-fonts.mjs - - name: Upload web artifact - uses: actions/upload-artifact@v4 - with: - name: selfhost-web - path: ./packages/frontend/web/dist - if-no-files-found: error - - build-storage: - name: Build Storage - ${{ matrix.targets.name }} - runs-on: ubuntu-latest - strategy: - matrix: - targets: - - name: x86_64-unknown-linux-gnu - file: storage.node - - name: aarch64-unknown-linux-gnu - file: storage.arm64.node - - name: armv7-unknown-linux-gnueabihf - file: storage.armv7.node - - steps: - - uses: actions/checkout@v4 - - name: Setup Version - id: version - uses: ./.github/actions/setup-version - - name: Setup Node.js - uses: ./.github/actions/setup-node - with: - electron-install: false - extra-flags: workspaces focus @affine/storage - - name: Build Rust - uses: ./.github/actions/build-rust - with: - target: ${{ matrix.targets.name }} - package: '@affine/storage' - nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} - - name: Upload ${{ matrix.targets.file }} - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.targets.file }} - path: ./packages/backend/storage/storage.node - if-no-files-found: error - - build-docker: - name: Build Docker - runs-on: ubuntu-latest - permissions: - contents: 'write' - id-token: 'write' - packages: 'write' needs: - - build-server - build-web - - build-web-selfhost - - build-storage steps: - - uses: actions/checkout@v4 - - name: Download core artifact + - name: Download web artifact uses: actions/download-artifact@v4 with: name: web path: ./packages/frontend/web/dist - - name: Download server dist - uses: actions/download-artifact@v4 - with: - name: server-dist - path: ./packages/backend/server/dist - - name: Download storage.node - uses: actions/download-artifact@v4 - with: - name: storage.node - path: ./packages/backend/server - - name: Download storage.node arm64 - uses: actions/download-artifact@v4 - with: - name: storage.arm64.node - path: ./packages/backend/storage - - name: Download storage.node arm64 - uses: actions/download-artifact@v4 - with: - name: storage.armv7.node - path: . - - name: move storage files - run: | - mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node - mv storage.node ./packages/backend/server/storage.armv7.node - name: Setup env run: | echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" @@ -187,7 +81,6 @@ jobs: else echo "RELEASE_FLAVOR=${{ inputs.flavor }}" >> "$GITHUB_ENV" fi - - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -210,53 +103,13 @@ jobs: file: .github/deployment/front/Dockerfile tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}} - # setup node without cache configuration - # Prisma cache is not compatible with docker build cache - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version-file: '.nvmrc' - registry-url: https://npm.pkg.github.com - scope: '@toeverything' - - - name: Remove web dist - run: rm -rf ./packages/frontend/web/dist - - - name: Download selfhost web artifact - uses: actions/download-artifact@v4 - with: - name: selfhost-web - path: ./packages/frontend/web/dist - - - name: Install Node.js dependencies - run: | - yarn config set --json supportedArchitectures.cpu '["x64", "arm64", "arm"]' - yarn config set --json supportedArchitectures.libc '["glibc"]' - yarn workspaces focus @affine/server --production - - - name: Generate Prisma client - run: yarn workspace @affine/server prisma generate - - - name: Build graphql Dockerfile - uses: docker/build-push-action@v5 - with: - context: . - push: true - pull: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 - provenance: true - file: .github/deployment/node/Dockerfile - tags: ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}} - deploy: name: Deploy to cluster if: ${{ github.event_name == 'workflow_dispatch' }} environment: ${{ github.event.inputs.flavor }} - permissions: - contents: 'write' - id-token: 'write' needs: - - build-docker + - build-frontend-image + - build-server-image runs-on: ubuntu-latest steps: - uses: actions/checkout@v4