refactor(core): Rebuild docker infrastructure (#8451)

.dockerignore

@@ -7,7 +7,12 @@ packages/node-dev
 packages/**/node_modules
 packages/**/dist
 packages/**/.turbo
+packages/**/*.test.*
 .git
 .github
 *.tsbuildinfo
 packages/cli/dist/**/e2e.*
+docker/compose
+docker/**/Dockerfile
+.vscode
+cypress

.github/workflows/docker-base-image.yml

@@ -9,7 +9,6 @@ on:
         required: true
         default: '18'
         options:
-          - '16'
           - '18'
           - '20'
 
@@ -25,6 +24,13 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.0.0
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Login to DockerHub
         uses: docker/login-action@v3.0.0
         with:
@@ -34,11 +40,13 @@ jobs:
       - name: Build
         uses: docker/build-push-action@v5.1.0
         with:
-          context: ./docker/images/n8n-base
+          context: .
+          file: ./docker/images/n8n-base/Dockerfile
           build-args: |
             NODE_VERSION=${{github.event.inputs.node_version}}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64
           provenance: false
           push: true
           tags: |
             ${{ secrets.DOCKER_USERNAME }}/base:${{ github.event.inputs.node_version }}
+            ghcr.io/${{ github.repository_owner }}/base:${{ github.event.inputs.node_version }}

.github/workflows/docker-image-beta.yml

@@ -1,44 +0,0 @@
-name: Docker Image - Beta
-
-on:
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: 'Branch to create image off.'
-        required: true
-        default: 'ai-beta'
-      tag:
-        description: 'Name of the docker tag to create.'
-        required: true
-        default: 'ai-beta'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4.1.1
-        with:
-          fetch-depth: 0
-          ref: ${{ github.event.inputs.branch || 'ai-beta' }}
-
-      - uses: docker/setup-qemu-action@v3.0.0
-      - uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v3.0.0
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5.1.0
-        with:
-          context: .
-          file: ./docker/images/n8n-custom/Dockerfile
-          build-args: |
-            N8N_RELEASE_TYPE=beta
-          platforms: linux/amd64,linux/arm64
-          provenance: false
-          push: true
-          tags: ${{ secrets.DOCKER_USERNAME }}/n8n:${{ github.event.inputs.tag || 'ai-beta' }}
-          no-cache: true

.github/workflows/docker-images-nightly.yml

@@ -35,11 +35,6 @@ on:
         description: 'URL to call after Docker Image got built successfully.'
         required: false
         default: ''
-      include-arm64:
-        description: 'Include ARM64 support'
-        type: boolean
-        required: true
-        default: false
 
 jobs:
   build:
@@ -78,14 +73,15 @@ jobs:
         uses: docker/build-push-action@v5.1.0
         with:
           context: .
+          file: ./docker/images/n8n-custom/Dockerfile
           build-args: |
             N8N_RELEASE_TYPE=nightly
-          file: ./docker/images/n8n-custom/Dockerfile
-          platforms: ${{ github.event.inputs.include-arm64 == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+          platforms: linux/amd64,linux/arm64
           provenance: false
           push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           tags: ${{ secrets.DOCKER_USERNAME }}/n8n:${{ github.event.inputs.tag || 'nightly' }}
-          no-cache: true
 
       - name: Call Success URL - optionally
         run: |

.github/workflows/docker-images.yml

@@ -40,7 +40,7 @@ jobs:
           context: ./docker/images/n8n
           build-args: |
             N8N_VERSION=${{ steps.vars.outputs.tag }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64
           provenance: false
           push: true
           tags: |

.npmrc

@@ -1,3 +1,5 @@
+audit = false
+fund = false
 update-notifier = false
 auto-install-peers = true
 strict-peer-dependencies = false

docker/images/n8n-base/.npmrc

@@ -1,4 +0,0 @@
-audit = false
-fund = false
-loglevel = warn
-update-notifier = false

docker/images/n8n-base/Dockerfile

@@ -1,20 +1,38 @@
 ARG NODE_VERSION=18
-FROM node:${NODE_VERSION}-alpine
 
-WORKDIR /home/node
-COPY .npmrc /usr/local/etc/npmrc
+# 1. Use a builder step to download various dependencies
+FROM node:${NODE_VERSION}-alpine as builder
 
-RUN \
-	apk add --update git openssh graphicsmagick tini tzdata ca-certificates libc6-compat && \
-	npm install -g npm@9.5.1 full-icu && \
-	rm -rf /var/cache/apk/* /root/.npm /tmp/* && \
-	# Install fonts
+# Install fonts
+RUN	\
 	apk --no-cache add --virtual fonts msttcorefonts-installer fontconfig && \
 	update-ms-fonts && \
 	fc-cache -f && \
 	apk del fonts && \
-	find  /usr/share/fonts/truetype/msttcorefonts/ -type l -exec unlink {} \; && \
-	rm -rf /var/cache/apk/* /tmp/*
+	find  /usr/share/fonts/truetype/msttcorefonts/ -type l -exec unlink {} \;
 
+# Install git and other OS dependencies
+RUN apk add --update git openssh graphicsmagick tini tzdata ca-certificates libc6-compat jq
+
+# Update npm and install full-uci
+COPY .npmrc /usr/local/etc/npmrc
+RUN npm install -g npm@9.9.2 full-icu@1.5.0
+
+# Activate corepack, and install pnpm
+WORKDIR /tmp
+COPY package.json ./
+RUN corepack enable && corepack prepare --activate
+
+# Cleanup
+RUN	rm -rf /lib/apk/db /var/cache/apk/ /tmp/* /root/.npm /root/.cache/node /opt/yarn*
+
+# 2. Start with a new clean image and copy over the added files into a single layer
+FROM node:${NODE_VERSION}-alpine
+COPY --from=builder / /
+
+# Delete this folder to make the base image backward compatible to be able to build older version images
+RUN rm -rf /tmp/v8-compile-cache*
+
+WORKDIR /home/node
 ENV NODE_ICU_DATA /usr/local/lib/node_modules/full-icu
 EXPOSE 5678/tcp

docker/images/n8n-custom/Dockerfile

@@ -1,40 +1,41 @@
 ARG NODE_VERSION=18
 
 # 1. Create an image to build n8n
-FROM n8nio/base:${NODE_VERSION} as builder
+FROM --platform=linux/amd64 n8nio/base:${NODE_VERSION} as builder
 
-COPY --chown=node:node turbo.json package.json .npmrc pnpm-lock.yaml pnpm-workspace.yaml jest.config.js tsconfig*.json ./
-COPY --chown=node:node scripts ./scripts
-COPY --chown=node:node packages ./packages
-COPY --chown=node:node patches ./patches
-
-RUN apk add --update jq
-RUN corepack enable && corepack prepare --activate
-USER node
-
-RUN pnpm install --frozen-lockfile
+# Build the application from source
+WORKDIR /src
+COPY . /src
+RUN --mount=type=cache,id=pnpm-store,target=/root/.local/share/pnpm/store --mount=type=cache,id=pnpm-metadata,target=/root/.cache/pnpm/metadata pnpm install --frozen-lockfile
 RUN pnpm build
-RUN rm -rf node_modules
+
+# Delete all dev dependencies
 RUN jq 'del(.pnpm.patchedDependencies)' package.json > package.json.tmp; mv package.json.tmp package.json
 RUN node scripts/trim-fe-packageJson.js
-RUN NODE_ENV=production pnpm install --prod --no-optional
-RUN find . -type f -name "*.ts" -o -name "*.js.map" -o -name "*.vue" -o -name "tsconfig.json" -o -name "*.tsbuildinfo" | xargs rm -rf
-RUN rm -rf packages/@n8n_io/eslint-config packages/editor-ui/src packages/editor-ui/node_modules packages/design-system
-RUN rm -rf patches .npmrc *.yaml node_modules/.cache packages/**/node_modules/.cache packages/**/.turbo .config .cache .local .node /tmp/*
 
+# Delete any source code, source-mapping, or typings
+RUN find . -type f -name "*.ts" -o -name "*.js.map" -o -name "*.vue" -o -name "tsconfig.json" -o -name "*.tsbuildinfo" | xargs rm -rf
+
+# Deploy the `n8n` package into /compiled
+RUN mkdir /compiled
+RUN NODE_ENV=production pnpm --filter=n8n --prod --no-optional deploy /compiled
 
 # 2. Start with a new clean image with just the code that is needed to run n8n
 FROM n8nio/base:${NODE_VERSION}
-ARG N8N_RELEASE_TYPE=dev
-COPY --from=builder /home/node /usr/local/lib/node_modules/n8n
-RUN ln -s /usr/local/lib/node_modules/n8n/packages/cli/bin/n8n /usr/local/bin/n8n
+ENV NODE_ENV=production
 
+ARG N8N_RELEASE_TYPE=dev
+ENV N8N_RELEASE_TYPE=${N8N_RELEASE_TYPE}
+
+WORKDIR /home/node
+COPY --from=builder /compiled /usr/local/lib/node_modules/n8n
 COPY docker/images/n8n/docker-entrypoint.sh /
 
 RUN \
+	pnpm rebuild --dir /usr/local/lib/node_modules/n8n sqlite3 && \
+	ln -s /usr/local/lib/node_modules/n8n/bin/n8n /usr/local/bin/n8n && \
 	mkdir .n8n && \
 	chown node:node .n8n
+
 USER node
-ENV NODE_ENV=production
-ENV N8N_RELEASE_TYPE=${N8N_RELEASE_TYPE}
 ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]

docker/images/n8n/Dockerfile

@@ -8,14 +8,7 @@ ENV N8N_VERSION=${N8N_VERSION}
 ENV NODE_ENV=production
 ENV N8N_RELEASE_TYPE=stable
 RUN set -eux; \
-	apkArch="$(apk --print-arch)"; \
-	case "$apkArch" in \
-	'armv7') apk --no-cache add --virtual build-dependencies python3 build-base;; \
-	esac && \
 	npm install -g --omit=dev n8n@${N8N_VERSION} && \
-	case "$apkArch" in \
-	'armv7') apk del build-dependencies;; \
-	esac && \
 	rm -rf /usr/local/lib/node_modules/n8n/node_modules/@n8n/chat && \
 	rm -rf /usr/local/lib/node_modules/n8n/node_modules/n8n-design-system && \
 	rm -rf /usr/local/lib/node_modules/n8n/node_modules/n8n-editor-ui/node_modules && \

docker/images/n8n/README.md

@@ -219,10 +219,10 @@ docker run -it --rm \
 ## Build Docker-Image
 
 ```bash
-docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --build-arg N8N_VERSION=<VERSION> -t n8n:<VERSION> .
+docker buildx build --platform linux/amd64,linux/arm64 --build-arg N8N_VERSION=<VERSION> -t n8n:<VERSION> .
 
 # For example:
-docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --build-arg N8N_VERSION=0.114.0 -t n8n:0.114.0 .
+docker buildx build --platform linux/amd64,linux/arm64 --build-arg N8N_VERSION=0.114.0 -t n8n:0.114.0 .
 ```
 
 ## What does n8n mean and how do you pronounce it?