From 736635a94b4e3e088a4f7b8d99a2479247f618f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Fri, 15 Nov 2024 09:38:30 +0100 Subject: [PATCH] Begin moving to postgres spilo + adding pgvector (#8309) We will remove the `twenty-postgres` image that was used for local development and only use `twenty-postgres-pilo` (which we use in prod), bringing the development environment closer to prod and avoiding having to maintain 2 images. Instead of provisioning the super user after the db initialization, we directly rely on the superuser provided by Spilo for simplicity. We also introduce a change that tries to create the right database (`default` or `test`) based on the context. How to test: ``` docker build -t twentycrm/twenty-postgres-spilo:latest -f ./packages/twenty-docker/twenty-postgres-spilo/Dockerfile . docker images --no-trunc | grep twenty-postgres-spilo postgres-on-docker: docker run \ --name twenty_pg \ -e PGUSER_SUPERUSER=twenty \ -e PGPASSWORD_SUPERUSER=twenty \ -e ALLOW_NOSSL=true \ -v twenty_db_data:/home/postgres/pgdata \ -p 5432:5432 \ REPLACE_WITH_IMAGE_ID ``` --- .github/workflows/ci-server.yaml | 31 ++++++++++--- .github/workflows/ci-test-docker-compose.yaml | 37 ++++++++++++--- .github/workflows/ci-website.yaml | 21 ++++++--- .vscode/twenty.code-workspace | 4 -- Makefile | 22 ++++++--- install.sh | 2 +- packages/twenty-docker/.env.example | 3 +- packages/twenty-docker/Makefile | 11 ----- packages/twenty-docker/docker-compose.yml | 19 ++++---- .../k8s/manifests/deployment-db.yaml | 8 ++-- .../k8s/manifests/deployment-server.yaml | 2 +- .../k8s/manifests/deployment-worker.yaml | 2 +- .../twenty-docker/k8s/terraform/README.md | 2 +- .../twenty-docker/k8s/terraform/variables.tf | 2 +- .../twenty-postgres-spilo/Dockerfile | 10 ++--- .../twenty-docker/twenty-postgres/Dockerfile | 45 ------------------- .../twenty-docker/twenty-postgres/init.sql | 4 -- packages/twenty-docker/twenty/Dockerfile | 3 +- packages/twenty-docker/twenty/entrypoint.sh | 8 ++++ .../drivers/env_variables.ts | 2 +- packages/twenty-server/.env.example | 2 +- packages/twenty-server/.env.test | 2 +- packages/twenty-server/scripts/render-run.sh | 2 +- .../twenty-server/scripts/render-worker.sh | 2 +- packages/twenty-server/scripts/setup-db.ts | 7 ++- packages/twenty-server/scripts/truncate-db.ts | 8 ++++ .../self-hosting/cloud-providers.mdx | 12 ++--- .../self-hosting/docker-compose.mdx | 4 +- render.yaml | 2 - 29 files changed, 149 insertions(+), 130 deletions(-) delete mode 100644 packages/twenty-docker/twenty-postgres/Dockerfile delete mode 100644 packages/twenty-docker/twenty-postgres/init.sql diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yaml index 5d7a35244f..06b1413035 100644 --- a/.github/workflows/ci-server.yaml +++ b/.github/workflows/ci-server.yaml @@ -18,12 +18,19 @@ jobs: NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 services: postgres: - image: twentycrm/twenty-postgres + image: twentycrm/twenty-postgres-spilo env: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres + PGUSER_SUPERUSER: postgres + PGPASSWORD_SUPERUSER: twenty + ALLOW_NOSSL: "true" + SPILO_PROVIDER: "local" ports: - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 redis: image: redis ports: @@ -63,6 +70,11 @@ jobs: - name: Server / Write .env if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-server + - name: Server / Create DB + if: steps.changed-files.outputs.any_changed == 'true' + run: | + PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";' + PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";' - name: Worker / Run if: steps.changed-files.outputs.any_changed == 'true' run: npx nx run twenty-server:worker:ci @@ -109,12 +121,19 @@ jobs: needs: server-setup services: postgres: - image: twentycrm/twenty-postgres + image: twentycrm/twenty-postgres-spilo env: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres + PGUSER_SUPERUSER: postgres + PGPASSWORD_SUPERUSER: twenty + ALLOW_NOSSL: "true" + SPILO_PROVIDER: "local" ports: - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 redis: image: redis ports: diff --git a/.github/workflows/ci-test-docker-compose.yaml b/.github/workflows/ci-test-docker-compose.yaml index d7202857a7..a9a076da33 100644 --- a/.github/workflows/ci-test-docker-compose.yaml +++ b/.github/workflows/ci-test-docker-compose.yaml @@ -8,7 +8,7 @@ concurrency: jobs: test: - timeout-minutes: 10 + timeout-minutes: 30 runs-on: ubuntu-latest steps: - name: Checkout @@ -36,20 +36,42 @@ jobs: yq eval 'del(.services.db.image)' -i docker-compose.yml yq eval '.services.db.build.context = "../../"' -i docker-compose.yml - yq eval '.services.db.build.dockerfile = "./packages/twenty-docker/twenty-postgres/Dockerfile"' -i docker-compose.yml + yq eval '.services.db.build.dockerfile = "./packages/twenty-docker/twenty-postgres-spilo/Dockerfile"' -i docker-compose.yml echo "Setting up .env file..." cp .env.example .env echo "Generating secrets..." echo "# === Randomly generated secrets ===" >>.env echo "APP_SECRET=$(openssl rand -base64 32)" >>.env - echo "POSTGRES_ADMIN_PASSWORD=$(openssl rand -base64 32)" >>.env + echo "PGPASSWORD_SUPERUSER=$(openssl rand -base64 32)" >>.env - echo "Starting server..." - docker compose up -d + echo "Docker compose up..." + docker compose up -d || { + echo "Docker compose failed to start" + docker compose logs + exit 1 + } docker compose logs db server -f & pid=$! + echo "Waiting for database to start..." + count=0 + while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-db-1) = "healthy" ]; do + sleep 1; + count=$((count+1)); + if [ $(docker inspect --format='{{.State.Status}}' twenty-db-1) = "exited" ]; then + echo "Database exited" + docker compose logs db + exit 1 + fi + if [ $count -gt 300 ]; then + echo "Failed to start database after 5 minutes" + docker compose logs db + exit 1 + fi + echo "Still waiting for database... (${count}/60)" + done + echo "Waiting for server to start..." count=0 while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-server-1) = "healthy" ]; do @@ -57,11 +79,14 @@ jobs: count=$((count+1)); if [ $(docker inspect --format='{{.State.Status}}' twenty-server-1) = "exited" ]; then echo "Server exited" + docker compose logs server exit 1 fi if [ $count -gt 300 ]; then - echo "Failed to start server" + echo "Failed to start server after 5 minutes" + docker compose logs server exit 1 fi + echo "Still waiting for server... (${count}/300s)" done working-directory: ./packages/twenty-docker/ diff --git a/.github/workflows/ci-website.yaml b/.github/workflows/ci-website.yaml index 770381855b..0f87078849 100644 --- a/.github/workflows/ci-website.yaml +++ b/.github/workflows/ci-website.yaml @@ -17,12 +17,19 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: twentycrm/twenty-postgres + image: twentycrm/twenty-postgres-spilo env: - POSTGRES_PASSWORD: twenty - POSTGRES_USER: twenty + PGUSER_SUPERUSER: postgres + PGPASSWORD_SUPERUSER: twenty + ALLOW_NOSSL: "true" + SPILO_PROVIDER: "local" ports: - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 with: @@ -37,16 +44,20 @@ jobs: if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install + - name: Server / Create DB + if: steps.changed-files.outputs.any_changed == 'true' + run: PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";' + - name: Website / Run migrations if: steps.changed-files.outputs.changed == 'true' run: npx nx database:migrate twenty-website env: - DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default + DATABASE_PG_URL: postgres://postgres:twenty@localhost:5432/default - name: Website / Build Website if: steps.changed-files.outputs.changed == 'true' run: npx nx build twenty-website env: - DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default + DATABASE_PG_URL: postgres://postgres:twenty@localhost:5432/default - name: Mark as VALID if: steps.changed-files.outputs.changed != 'true' # If no changes, mark as valid diff --git a/.vscode/twenty.code-workspace b/.vscode/twenty.code-workspace index ce2498959c..38d8b34543 100644 --- a/.vscode/twenty.code-workspace +++ b/.vscode/twenty.code-workspace @@ -24,10 +24,6 @@ "name": "packages/twenty-emails", "path": "../packages/twenty-emails" }, - { - "name": "packages/twenty-postgres", - "path": "../packages/twenty-postgres" - }, { "name": "packages/twenty-server", "path": "../packages/twenty-server" diff --git a/Makefile b/Makefile index af42f19d9f..3b115523d1 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,20 @@ postgres-on-docker: - docker run \ - --name twenty_postgres \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=default \ - -v twenty_db_data:/var/lib/postgresql/data \ + docker run -d \ + --name twenty_pg \ + -e PGUSER_SUPERUSER=postgres \ + -e PGPASSWORD_SUPERUSER=twenty \ + -e ALLOW_NOSSL=true \ + -v twenty_db_data:/home/postgres/pgdata \ -p 5432:5432 \ - twentycrm/twenty-postgres:latest + twentycrm/twenty-postgres-spilo:latest + @echo "Waiting for PostgreSQL to be ready..." + @until PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres \ + -c 'SELECT pg_is_in_recovery();' 2>/dev/null | grep -q 'f'; do \ + sleep 1; \ + done + PGPASSWORD=twenty psql -h localhost -p 5432 -U postgres -d postgres \ + -c "CREATE DATABASE \"default\" WITH OWNER postgres;" \ + -c "CREATE DATABASE \"test\" WITH OWNER postgres;" redis-on-docker: docker run -d --name twenty_redis -p 6379:6379 redis/redis-stack-server:latest \ No newline at end of file diff --git a/install.sh b/install.sh index 34741b89b9..1f80b57506 100755 --- a/install.sh +++ b/install.sh @@ -93,7 +93,7 @@ fi echo "# === Randomly generated secrets ===" >>.env echo "APP_SECRET=$(openssl rand -base64 32)" >>.env echo "" >>.env -echo "POSTGRES_ADMIN_PASSWORD=$(openssl rand -base64 32)" >>.env +echo "PGPASSWORD_SUPERUSER=$(openssl rand -hex 16)" >>.env echo -e "\t• .env configuration completed" diff --git a/packages/twenty-docker/.env.example b/packages/twenty-docker/.env.example index b10e09876e..5a3ab70a71 100644 --- a/packages/twenty-docker/.env.example +++ b/packages/twenty-docker/.env.example @@ -1,6 +1,7 @@ TAG=latest -# POSTGRES_ADMIN_PASSWORD=replace_me_with_a_strong_password +#PGUSER_SUPERUSER=postgres +#PGPASSWORD_SUPERUSER=replace_me_with_a_strong_password PG_DATABASE_HOST=db:5432 REDIS_URL=redis://redis:6379 diff --git a/packages/twenty-docker/Makefile b/packages/twenty-docker/Makefile index e8d77b60bf..2d89c4c4ad 100644 --- a/packages/twenty-docker/Makefile +++ b/packages/twenty-docker/Makefile @@ -4,9 +4,6 @@ prod-build: prod-run: @docker run -d -p 3000:3000 --name twenty twenty -prod-postgres-build: - @cd ../.. && docker build -f ./packages/twenty-docker/twenty-postgres/Dockerfile --tag twenty-postgres . && cd - - prod-postgres-run: @docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres --name twenty-postgres twenty-postgres @@ -15,11 +12,3 @@ prod-website-build: prod-website-run: @docker run -d -p 3000:3000 --name twenty-website twenty-website - -release-postgres: - @cd ../.. && docker buildx build \ - --push \ - --no-cache \ - --platform linux/amd64,linux/arm64 \ - -f ./packages/twenty-docker/twenty-postgres/Dockerfile -t twentycrm/twenty-postgres:$(version) -t twentycrm/twenty-postgres:latest . \ - && cd - diff --git a/packages/twenty-docker/docker-compose.yml b/packages/twenty-docker/docker-compose.yml index 41d80dabc3..fcd485a28d 100644 --- a/packages/twenty-docker/docker-compose.yml +++ b/packages/twenty-docker/docker-compose.yml @@ -22,10 +22,10 @@ services: - "3000:3000" environment: PORT: 3000 - PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default + PG_DATABASE_URL: postgres://${PGUSER_SUPERUSER:-postgres}:${PGPASSWORD_SUPERUSER:-twenty}@${PG_DATABASE_HOST:-db:5432}/default SERVER_URL: ${SERVER_URL} FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL} - REDIS_URL: ${REDIS_URL:-redis://localhost:6379} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} ENABLE_DB_MIGRATIONS: "true" @@ -52,10 +52,10 @@ services: image: twentycrm/twenty:${TAG} command: ["yarn", "worker:prod"] environment: - PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default + PG_DATABASE_URL: postgres://${PGUSER_SUPERUSER:-postgres}:${PGPASSWORD_SUPERUSER:-twenty}@${PG_DATABASE_HOST:-db:5432}/default SERVER_URL: ${SERVER_URL} FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL} - REDIS_URL: ${REDIS_URL:-redis://localhost:6379} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} ENABLE_DB_MIGRATIONS: "false" # it already runs on the server @@ -73,13 +73,16 @@ services: restart: always db: - image: twentycrm/twenty-postgres:${TAG} + image: twentycrm/twenty-postgres-spilo:${TAG} volumes: - - db-data:/bitnami/postgresql + - db-data:/home/postgres/pgdata environment: - POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD} + PGUSER_SUPERUSER: ${PGUSER_SUPERUSER:-postgres} + PGPASSWORD_SUPERUSER: ${PGPASSWORD_SUPERUSER:-twenty} + ALLOW_NOSSL: "true" + SPILO_PROVIDER: "local" healthcheck: - test: pg_isready -U twenty -d default + test: pg_isready -U ${PGUSER_SUPERUSER:-postgres} -h localhost -d postgres interval: 5s timeout: 5s retries: 10 diff --git a/packages/twenty-docker/k8s/manifests/deployment-db.yaml b/packages/twenty-docker/k8s/manifests/deployment-db.yaml index 31a3361774..4cfe13912b 100644 --- a/packages/twenty-docker/k8s/manifests/deployment-db.yaml +++ b/packages/twenty-docker/k8s/manifests/deployment-db.yaml @@ -30,10 +30,10 @@ spec: image: twentycrm/twenty-postgres:latest imagePullPolicy: Always env: - - name: POSTGRES_PASSWORD + - name: PGUSER_SUPERUSER + value: "postgres" + - name: PGPASSWORD_SUPERUSER value: "twenty" - - name: BITNAMI_DEBUG - value: "true" ports: - containerPort: 5432 name: tcp @@ -48,7 +48,7 @@ spec: stdin: true tty: true volumeMounts: - - mountPath: /bitnami/postgresql + - mountPath: /home/postgres/pgdata name: twentycrm-db-data dnsPolicy: ClusterFirst restartPolicy: Always diff --git a/packages/twenty-docker/k8s/manifests/deployment-server.yaml b/packages/twenty-docker/k8s/manifests/deployment-server.yaml index 8577297886..cd3ad4aa7f 100644 --- a/packages/twenty-docker/k8s/manifests/deployment-server.yaml +++ b/packages/twenty-docker/k8s/manifests/deployment-server.yaml @@ -40,7 +40,7 @@ spec: - name: FRONT_BASE_URL value: "https://crm.example.com:443" - name: "PG_DATABASE_URL" - value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default" + value: "postgres://postgres:twenty@twenty-db.twentycrm.svc.cluster.local/default" - name: "REDIS_URL" value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379" - name: ENABLE_DB_MIGRATIONS diff --git a/packages/twenty-docker/k8s/manifests/deployment-worker.yaml b/packages/twenty-docker/k8s/manifests/deployment-worker.yaml index eb1938ba6d..281652dbbc 100644 --- a/packages/twenty-docker/k8s/manifests/deployment-worker.yaml +++ b/packages/twenty-docker/k8s/manifests/deployment-worker.yaml @@ -31,7 +31,7 @@ spec: - name: FRONT_BASE_URL value: "https://crm.example.com:443" - name: PG_DATABASE_URL - value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default" + value: "postgres://postgres:twenty@twenty-db.twentycrm.svc.cluster.local/default" - name: ENABLE_DB_MIGRATIONS value: "false" # it already runs on the server - name: STORAGE_TYPE diff --git a/packages/twenty-docker/k8s/terraform/README.md b/packages/twenty-docker/k8s/terraform/README.md index f6955300a6..32facfd180 100644 --- a/packages/twenty-docker/k8s/terraform/README.md +++ b/packages/twenty-docker/k8s/terraform/README.md @@ -51,7 +51,7 @@ To make configuration changes to how this doc is generated, see `./.terraform-do | [twentycrm\_app\_hostname](#input\_twentycrm\_app\_hostname) | The protocol, DNS fully qualified hostname, and port used to access TwentyCRM in your environment. Ex: https://crm.example.com:443 | `string` | n/a | yes | | [twentycrm\_pgdb\_admin\_password](#input\_twentycrm\_pgdb\_admin\_password) | TwentyCRM password for postgres database. | `string` | n/a | yes | | [twentycrm\_app\_name](#input\_twentycrm\_app\_name) | A friendly name prefix to use for every component deployed. | `string` | `"twentycrm"` | no | -| [twentycrm\_db\_image](#input\_twentycrm\_db\_image) | TwentyCRM image for database deployment. This defaults to latest. | `string` | `"twentycrm/twenty-postgres:latest"` | no | +| [twentycrm\_db\_image](#input\_twentycrm\_db\_image) | TwentyCRM image for database deployment. This defaults to latest. | `string` | `"twentycrm/twenty-postgres-spilo:latest"` | no | | [twentycrm\_db\_pv\_capacity](#input\_twentycrm\_db\_pv\_capacity) | Storage capacity provisioned for database persistent volume. | `string` | `"10Gi"` | no | | [twentycrm\_db\_pv\_path](#input\_twentycrm\_db\_pv\_path) | Local path to use to store the physical volume if using local storage on nodes. | `string` | `""` | no | | [twentycrm\_db\_pvc\_requests](#input\_twentycrm\_db\_pvc\_requests) | Storage capacity reservation for database persistent volume claim. | `string` | `"10Gi"` | no | diff --git a/packages/twenty-docker/k8s/terraform/variables.tf b/packages/twenty-docker/k8s/terraform/variables.tf index 7b682db79a..3ce7394ab2 100644 --- a/packages/twenty-docker/k8s/terraform/variables.tf +++ b/packages/twenty-docker/k8s/terraform/variables.tf @@ -29,7 +29,7 @@ variable "twentycrm_server_image" { variable "twentycrm_db_image" { type = string - default = "twentycrm/twenty-postgres:latest" + default = "twentycrm/twenty-postgres-spilo:latest" description = "TwentyCRM image for database deployment. This defaults to latest." } diff --git a/packages/twenty-docker/twenty-postgres-spilo/Dockerfile b/packages/twenty-docker/twenty-postgres-spilo/Dockerfile index a87a8a97ec..9a84c120d5 100644 --- a/packages/twenty-docker/twenty-postgres-spilo/Dockerfile +++ b/packages/twenty-docker/twenty-postgres-spilo/Dockerfile @@ -3,10 +3,10 @@ ARG SPILO_VERSION=3.2-p1 ARG WRAPPERS_VERSION=0.2.0 # Build the mysql_fdw extension -FROM debian:bookworm as build-mysql_fdw +FROM debian:bookworm AS build-mysql_fdw ARG POSTGRES_VERSION -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt install -y \ build-essential \ @@ -17,14 +17,14 @@ RUN apt update && \ # Install mysql_fdw RUN git clone https://github.com/EnterpriseDB/mysql_fdw.git -WORKDIR mysql_fdw +WORKDIR /mysql_fdw RUN make USE_PGXS=1 # Build libssl for wrappers -FROM ubuntu:22.04 as build-libssl +FROM ubuntu:22.04 AS build-libssl -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt install -y \ build-essential \ diff --git a/packages/twenty-docker/twenty-postgres/Dockerfile b/packages/twenty-docker/twenty-postgres/Dockerfile deleted file mode 100644 index 9c9b96398e..0000000000 --- a/packages/twenty-docker/twenty-postgres/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -ARG IMAGE_TAG='15.5.0-debian-11-r15' - -FROM bitnami/postgresql:${IMAGE_TAG} - -ARG PG_MAIN_VERSION=15 -ARG WRAPPERS_VERSION=0.2.0 -ARG TARGETARCH - -USER root - -RUN set -eux; \ - ARCH="$(dpkg --print-architecture)"; \ - case "${ARCH}" in \ - aarch64|arm64) \ - TARGETARCH='arm64'; \ - ;; \ - amd64|x86_64) \ - TARGETARCH='amd64'; \ - ;; \ - *) \ - echo "Unsupported arch: ${ARCH}"; \ - exit 1; \ - ;; \ - esac; - -RUN apt update && apt install build-essential git curl default-libmysqlclient-dev -y - -# Install precompiled supabase wrappers extensions -RUN curl -L "https://github.com/supabase/wrappers/releases/download/v${WRAPPERS_VERSION}/wrappers-v${WRAPPERS_VERSION}-pg${PG_MAIN_VERSION}-${TARGETARCH}-linux-gnu.deb" -o wrappers.deb -RUN dpkg --install wrappers.deb -RUN cp /usr/share/postgresql/${PG_MAIN_VERSION}/extension/wrappers* /opt/bitnami/postgresql/share/extension/ -RUN cp /usr/lib/postgresql/${PG_MAIN_VERSION}/lib/wrappers* /opt/bitnami/postgresql/lib/ - -RUN export PATH=/usr/local/pgsql/bin/:$PATH -RUN export PATH=/usr/local/mysql/bin/:$PATH -RUN git clone https://github.com/EnterpriseDB/mysql_fdw.git -WORKDIR mysql_fdw -RUN make USE_PGXS=1 -RUN make USE_PGXS=1 install - -COPY ./packages/twenty-docker/twenty-postgres/init.sql /docker-entrypoint-initdb.d/ - -USER 1001 -ENTRYPOINT ["/opt/bitnami/scripts/postgresql/entrypoint.sh"] -CMD ["/opt/bitnami/scripts/postgresql/run.sh"] diff --git a/packages/twenty-docker/twenty-postgres/init.sql b/packages/twenty-docker/twenty-postgres/init.sql deleted file mode 100644 index 16a4788af6..0000000000 --- a/packages/twenty-docker/twenty-postgres/init.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE DATABASE "default"; -CREATE DATABASE "test"; -CREATE USER twenty PASSWORD 'twenty'; -ALTER ROLE twenty superuser; diff --git a/packages/twenty-docker/twenty/Dockerfile b/packages/twenty-docker/twenty/Dockerfile index e57cf30d14..b1d5d4dff9 100644 --- a/packages/twenty-docker/twenty/Dockerfile +++ b/packages/twenty-docker/twenty/Dockerfile @@ -52,9 +52,10 @@ RUN apk add --no-cache curl jq RUN npm install -g tsx +RUN apk add --no-cache postgresql-client + COPY ./packages/twenty-docker/twenty/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh - WORKDIR /app/packages/twenty-server ARG REACT_APP_SERVER_BASE_URL diff --git a/packages/twenty-docker/twenty/entrypoint.sh b/packages/twenty-docker/twenty/entrypoint.sh index 9733781fd5..c6844ec739 100755 --- a/packages/twenty-docker/twenty/entrypoint.sh +++ b/packages/twenty-docker/twenty/entrypoint.sh @@ -4,6 +4,14 @@ if [ "${ENABLE_DB_MIGRATIONS}" = "true" ] && [ ! -f /app/docker-data/db_status ]; then echo "Running database setup and migrations..." + # Creating the database if it doesn't exist + PGUSER=$(echo $PG_DATABASE_URL | awk -F '//' '{print $2}' | awk -F ':' '{print $1}') + PGPASS=$(echo $PG_DATABASE_URL | awk -F ':' '{print $3}' | awk -F '@' '{print $1}') + PGHOST=$(echo $PG_DATABASE_URL | awk -F '@' '{print $2}' | awk -F ':' '{print $1}') + PGPORT=$(echo $PG_DATABASE_URL | awk -F ':' '{print $4}' | awk -F '/' '{print $1}') + PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'default'" | grep -q 1 || \ + PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -c "CREATE DATABASE \"default\"" + # Run setup and migration scripts NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts yarn database:migrate:prod diff --git a/packages/twenty-e2e-testing/drivers/env_variables.ts b/packages/twenty-e2e-testing/drivers/env_variables.ts index 2bb7f57d88..07ba4f6334 100644 --- a/packages/twenty-e2e-testing/drivers/env_variables.ts +++ b/packages/twenty-e2e-testing/drivers/env_variables.ts @@ -3,7 +3,7 @@ import path from 'path'; export const envVariables = (variables: string) => { let payload = ` - PG_DATABASE_URL=postgres://twenty:twenty@localhost:5432/default + PG_DATABASE_URL=postgres://postgres:twenty@localhost:5432/default FRONT_BASE_URL=http://localhost:3001 ACCESS_TOKEN_SECRET=replace_me_with_a_random_string_access LOGIN_TOKEN_SECRET=replace_me_with_a_random_string_login diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index 6054688dc7..bddacf4a08 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -1,5 +1,5 @@ # Use this for local setup -PG_DATABASE_URL=postgres://twenty:twenty@localhost:5432/default +PG_DATABASE_URL=postgres://postgres:twenty@localhost:5432/default REDIS_URL=redis://localhost:6379 FRONT_BASE_URL=http://localhost:3001 diff --git a/packages/twenty-server/.env.test b/packages/twenty-server/.env.test index 659411dc21..ac0225eacd 100644 --- a/packages/twenty-server/.env.test +++ b/packages/twenty-server/.env.test @@ -1,4 +1,4 @@ -PG_DATABASE_URL=postgres://twenty:twenty@localhost:5432/test +PG_DATABASE_URL=postgres://postgres:twenty@localhost:5432/test REDIS_URL=redis://localhost:6379 DEBUG_MODE=true diff --git a/packages/twenty-server/scripts/render-run.sh b/packages/twenty-server/scripts/render-run.sh index 056a3c919e..813f9e20ff 100755 --- a/packages/twenty-server/scripts/render-run.sh +++ b/packages/twenty-server/scripts/render-run.sh @@ -1,4 +1,4 @@ #!/bin/sh -export PG_DATABASE_URL=postgres://twenty:twenty@$PG_DATABASE_HOST:$PG_DATABASE_PORT/default +export PG_DATABASE_URL=postgres://postgres:twenty@$PG_DATABASE_HOST:$PG_DATABASE_PORT/default yarn database:init:prod node dist/src/main diff --git a/packages/twenty-server/scripts/render-worker.sh b/packages/twenty-server/scripts/render-worker.sh index 1a34c6f7c3..25f259e417 100755 --- a/packages/twenty-server/scripts/render-worker.sh +++ b/packages/twenty-server/scripts/render-worker.sh @@ -1,3 +1,3 @@ #!/bin/sh -export PG_DATABASE_URL=postgres://twenty:twenty@$PG_DATABASE_HOST:$PG_DATABASE_PORT/default +export PG_DATABASE_URL=postgres://postgres:twenty@$PG_DATABASE_HOST:$PG_DATABASE_PORT/default node dist/src/queue-worker/queue-worker diff --git a/packages/twenty-server/scripts/setup-db.ts b/packages/twenty-server/scripts/setup-db.ts index a562a31149..f5baadd8ef 100644 --- a/packages/twenty-server/scripts/setup-db.ts +++ b/packages/twenty-server/scripts/setup-db.ts @@ -7,6 +7,11 @@ import { camelToSnakeCase, performQuery } from './utils'; rawDataSource .initialize() .then(async () => { + await performQuery( + 'CREATE EXTENSION IF NOT EXISTS "vector"', + 'create extension "vector (pgvector)"', + ); + await performQuery( 'CREATE SCHEMA IF NOT EXISTS "public"', 'create schema "public"', @@ -53,7 +58,7 @@ rawDataSource for (const wrapper of supabaseWrappers) { await performQuery( ` - CREATE FOREIGN DATA WRAPPER "${wrapper.toLowerCase()}_fdw" + CREATE FOREIGN DATA WRAPPER IF NOT EXISTS "${wrapper.toLowerCase()}_fdw" HANDLER "${camelToSnakeCase(wrapper)}_fdw_handler" VALIDATOR "${camelToSnakeCase(wrapper)}_fdw_validator"; `, diff --git a/packages/twenty-server/scripts/truncate-db.ts b/packages/twenty-server/scripts/truncate-db.ts index bc5b4feeac..c484752f5d 100644 --- a/packages/twenty-server/scripts/truncate-db.ts +++ b/packages/twenty-server/scripts/truncate-db.ts @@ -21,6 +21,14 @@ async function dropSchemasSequentially() { // Iterate over each schema and drop it // This is to avoid dropping all schemas at once, which would cause an out of shared memory error for (const schema of schemas) { + if ( + schema.schema_name === 'metric_helpers' || + schema.schema_name === 'user_management' || + schema.schema_name === 'public' + ) { + continue; + } + await performQuery( ` DROP SCHEMA IF EXISTS "${schema.schema_name}" CASCADE; diff --git a/packages/twenty-website/src/content/developers/self-hosting/cloud-providers.mdx b/packages/twenty-website/src/content/developers/self-hosting/cloud-providers.mdx index 8c40c0b313..df3bb2b254 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/cloud-providers.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/cloud-providers.mdx @@ -392,7 +392,7 @@ resource "azurerm_container_app" "twenty_db" { max_replicas = 1 container { name = local.db_app_name - image = "docker.io/twentycrm/twenty-postgres:${local.db_tag}" + image = "docker.io/twentycrm/twenty-postgres-spilo:${local.db_tag}" cpu = local.cpu memory = local.memory @@ -402,16 +402,12 @@ resource "azurerm_container_app" "twenty_db" { } env { - name = "POSTGRES_USER" + name = "PGUSER_SUPERUSER" value = "postgres" } env { - name = "POSTGRES_PASSWORD" - value = "postgres" - } - env { - name = "POSTGRES_DB" - value = "default" + name = "PGPASSWORD_SUPERUSER" + value = "twenty" } } diff --git a/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx b/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx index a6d79bbb2e..7ff62a44c0 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx @@ -66,10 +66,10 @@ Follow these steps for a manual setup. 4. **Set the Postgres Password** - Update the `POSTGRES_ADMIN_PASSWORD` value in the .env file with a strong password. + Update the `PGPASSWORD_SUPERUSER` value in the .env file with a strong password. ```ini - POSTGRES_ADMIN_PASSWORD=my_strong_password + PGPASSWORD_SUPERUSER=my_strong_password ``` ### Step 2: Obtain the Docker Compose File diff --git a/render.yaml b/render.yaml index 3d31f47f5b..b7850996ff 100644 --- a/render.yaml +++ b/render.yaml @@ -81,5 +81,3 @@ services: value: postgres - key: POSTGRES_PASSWORD value: postgres - - key: POSTGRES_DB - value: default