From 22e121b246411cb32205171c2f2a3c02193f05e5 Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Mon, 25 Dec 2023 00:35:03 +0530 Subject: [PATCH] Using psql command instead of postgres in initialScript (#66) * Use psql instead of postgres, as a result, use `pg_ctl` to temporarily start the postgres server in `${name}-init` process. * Avoid creating a custom `postgresPkg` with extensions by using `apply` attribute provided by `mkOption` * Refactor setup script, use `writeShellApplication` and resolve shell-check errors --------- Co-authored-by: shivaraj-bh --- nix/default.nix | 2 +- nix/{postgres.nix => postgres/default.nix} | 141 ++++----------------- nix/{ => postgres}/postgres_test.nix | 0 nix/postgres/setup-script.nix | 124 ++++++++++++++++++ test/flake.nix | 2 +- 5 files changed, 150 insertions(+), 119 deletions(-) rename nix/{postgres.nix => postgres/default.nix} (64%) rename nix/{ => postgres}/postgres_test.nix (100%) create mode 100644 nix/postgres/setup-script.nix diff --git a/nix/default.nix b/nix/default.nix index 6d14365..4ce0d8d 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -8,7 +8,7 @@ in ./elasticsearch.nix ./mysql.nix ./nginx.nix - ./postgres.nix + ./postgres ./redis-cluster.nix ./redis.nix ./zookeeper.nix diff --git a/nix/postgres.nix b/nix/postgres/default.nix similarity index 64% rename from nix/postgres.nix rename to nix/postgres/default.nix index 618a43a..72d39c1 100644 --- a/nix/postgres.nix +++ b/nix/postgres/default.nix @@ -7,7 +7,24 @@ in options = { enable = lib.mkEnableOption name; - package = lib.mkPackageOption pkgs "postgresql" { }; + package = lib.mkOption { + type = types.package; + description = "Which package of postgresql to use"; + default = pkgs.postgresql; + defaultText = lib.literalExpression "pkgs.postgresql"; + apply = postgresPkg: + if config.extensions != null then + if builtins.hasAttr "withPackages" postgresPkg + then postgresPkg.withPackages config.extensions + else + builtins.throw '' + Cannot add extensions to the PostgreSQL package. + `services.postgres.package` is missing the `withPackages` attribute. Did you already add extensions to the package? + '' + else postgresPkg; + + }; + extensions = lib.mkOption { type = with types; nullOr (functionTo (listOf package)); default = null; @@ -254,125 +271,15 @@ in internal = true; readOnly = true; default = - let - postgresPkg = - if config.extensions != null then - if builtins.hasAttr "withPackages" config.package - then config.package.withPackages config.extensions - else - builtins.throw '' - Cannot add extensions to the PostgreSQL package. - `services.postgres.package` is missing the `withPackages` attribute. Did you already add extensions to the package? - '' - else config.package; - in { processes = { # DB initialization "${name}-init" = let - setupInitialDatabases = - if config.initialDatabases != [ ] then - (lib.concatMapStrings - (database: '' - echo "Checking presence of database: ${database.name}" - # Create initial databases - dbAlreadyExists="$( - echo "SELECT 1 as exists FROM pg_database WHERE datname = '${database.name}';" | \ - postgres --single -E postgres | \ - ${pkgs.gnugrep}/bin/grep -c 'exists = "1"' || true - )" - echo $dbAlreadyExists - if [ 1 -ne "$dbAlreadyExists" ]; then - echo "Creating database: ${database.name}" - echo 'create database "${database.name}";' | postgres --single -E postgres - - - ${lib.optionalString (database.schema != null) '' - echo "Applying database schema on ${database.name}" - if [ -f "${database.schema}" ] - then - echo "Running file ${database.schema}" - ${pkgs.gawk}/bin/awk 'NF' "${database.schema}" | postgres --single -j -E ${database.name} - elif [ -d "${database.schema}" ] - then - # Read sql files in version order. Apply one file - # at a time to handle files where the last statement - # doesn't end in a ;. - ls -1v "${database.schema}"/*.sql | while read f ; do - echo "Applying sql file: $f" - ${pkgs.gawk}/bin/awk 'NF' "$f" | postgres --single -j -E ${database.name} - done - else - echo "ERROR: Could not determine how to apply schema with ${database.schema}" - exit 1 - fi - ''} - fi - '') - config.initialDatabases) - else - lib.optionalString config.createDatabase '' - echo "CREATE DATABASE ''${USER:-$(id -nu)};" | postgres --single -E postgres ''; - - runInitialScript = - let - scriptCmd = sqlScript: '' - echo "${sqlScript}" | postgres --single -E postgres - ''; - in - { - before = with config.initialScript; - lib.optionalString (before != null) (scriptCmd before); - after = with config.initialScript; - lib.optionalString (after != null) (scriptCmd after); - }; - - toStr = value: - if true == value then - "yes" - else if false == value then - "no" - else if lib.isString value then - "'${lib.replaceStrings [ "'" ] [ "''" ] value}'" - else - toString value; - - configFile = pkgs.writeText "postgresql.conf" (lib.concatStringsSep "\n" - (lib.mapAttrsToList (n: v: "${n} = ${toStr v}") (config.defaultSettings // config.settings))); - - initdbArgs = - config.initdbArgs - ++ (lib.optionals (config.superuser != null) [ "-U" config.superuser ]) - ++ [ "-D" config.dataDir ]; - - setupScript = pkgs.writeShellScriptBin "setup-postgres" '' - set -euo pipefail - export PATH=${postgresPkg}/bin:${pkgs.coreutils}/bin - - if [[ ! -d "$PGDATA" ]]; then - set -x - mkdir -p ${config.dataDir} - initdb ${lib.concatStringsSep " " initdbArgs} - set +x - - ${runInitialScript.before} - ${setupInitialDatabases} - ${runInitialScript.after} - else - echo "Postgres data directory already exists. Skipping initialization." - fi - - # Setup config - set -x - cp ${configFile} "$PGDATA/postgresql.conf" - ''; + setupScript = import ./setup-script.nix { inherit config pkgs lib; }; in { - command = '' - export PGDATA="${config.dataDir}" - ${lib.getExe setupScript} - ''; + command = setupScript; namespace = name; }; @@ -381,9 +288,9 @@ in let startScript = pkgs.writeShellApplication { name = "start-postgres"; + runtimeInputs = [ config.package pkgs.coreutils ]; text = '' - export PATH="${postgresPkg}"/bin:$PATH - set -x + set -euo pipefail PGDATA=$(readlink -f "${config.dataDir}") export PGDATA postgres -k "$PGDATA" @@ -397,11 +304,10 @@ in in { command = startScript; - depends_on."${name}-init".condition = "process_completed_successfully"; # SIGINT (= 2) for faster shutdown: https://www.postgresql.org/docs/current/server-shutdown.html shutdown.signal = 2; readiness_probe = { - exec.command = "${postgresPkg}/bin/pg_isready ${lib.concatStringsSep " " pg_isreadyArgs}"; + exec.command = "${config.package}/bin/pg_isready ${lib.concatStringsSep " " pg_isreadyArgs}"; initial_delay_seconds = 2; period_seconds = 10; timeout_seconds = 4; @@ -409,6 +315,7 @@ in failure_threshold = 5; }; namespace = name; + depends_on."${name}-init".condition = "process_completed_successfully"; # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy availability.restart = "on_failure"; }; diff --git a/nix/postgres_test.nix b/nix/postgres/postgres_test.nix similarity index 100% rename from nix/postgres_test.nix rename to nix/postgres/postgres_test.nix diff --git a/nix/postgres/setup-script.nix b/nix/postgres/setup-script.nix new file mode 100644 index 0000000..227a974 --- /dev/null +++ b/nix/postgres/setup-script.nix @@ -0,0 +1,124 @@ +{ config, pkgs, lib }: +let + setupInitialDatabases = + if config.initialDatabases != [ ] then + (lib.concatMapStrings + (database: '' + echo "Checking presence of database: ${database.name}" + # Create initial databases + dbAlreadyExists=$( + echo "SELECT 1 as exists FROM pg_database WHERE datname = '${database.name}';" | \ + psql -d postgres | \ + grep -c 'exists = "1"' || true + ) + echo "$dbAlreadyExists" + if [ 1 -ne "$dbAlreadyExists" ]; then + echo "Creating database: ${database.name}" + echo 'create database "${database.name}";' | psql -d postgres + + + ${lib.optionalString (database.schema != null) '' + echo "Applying database schema on ${database.name}" + if [ -f "${database.schema}" ] + then + echo "Running file ${database.schema}" + awk 'NF' "${database.schema}" | psql -d ${database.name} + elif [ -d "${database.schema}" ] + then + # Read sql files in version order. Apply one file + # at a time to handle files where the last statement + # doesn't end in a ;. + find "${database.schema}"/*.sql | while read -r f ; do + echo "Applying sql file: $f" + awk 'NF' "$f" | psql -d ${database.name} + done + else + echo "ERROR: Could not determine how to apply schema with ${database.schema}" + exit 1 + fi + ''} + fi + '') + config.initialDatabases) + else + lib.optionalString config.createDatabase '' + echo "CREATE DATABASE ''${USER:-$(id -nu)};" | psql -d postgres ''; + runInitialScript = + let + scriptCmd = sqlScript: '' + echo "${sqlScript}" | psql -d postgres + ''; + in + { + before = with config.initialScript; + lib.optionalString (before != null) (scriptCmd before); + after = with config.initialScript; + lib.optionalString (after != null) (scriptCmd after); + }; + toStr = value: + if true == value then + "yes" + else if false == value then + "no" + else if lib.isString value then + "'${lib.replaceStrings [ "'" ] [ "''" ] value}'" + else + toString value; + configFile = pkgs.writeText "postgresql.conf" (lib.concatStringsSep "\n" + (lib.mapAttrsToList (n: v: "${n} = ${toStr v}") (config.defaultSettings // config.settings))); + + initdbArgs = + config.initdbArgs + ++ (lib.optionals (config.superuser != null) [ "-U" config.superuser ]) + ++ [ "-D" config.dataDir ]; +in +(pkgs.writeShellApplication { + name = "setup-postgres"; + runtimeInputs = with pkgs; [ config.package coreutils gnugrep gawk ]; + text = '' + set -euo pipefail + # Setup postgres ENVs + export PGDATA="${config.dataDir}" + export PGPORT="${toString config.port}" + POSTGRES_RUN_INITIAL_SCRIPT="false" + + + if [[ ! -d "$PGDATA" ]]; then + initdb ${lib.concatStringsSep " " initdbArgs} + POSTGRES_RUN_INITIAL_SCRIPT="true" + echo + echo "PostgreSQL initdb process complete." + echo + fi + + # Setup config + cp ${configFile} "$PGDATA/postgresql.conf" + + if [[ "$POSTGRES_RUN_INITIAL_SCRIPT" = "true" ]]; then + echo + echo "PostgreSQL is setting up the initial database." + echo + PGHOST=$(mktemp -d "$(readlink -f ${config.dataDir})/pg-init-XXXXXX") + export PGHOST + + function remove_tmp_pg_init_sock_dir() { + if [[ -d "$1" ]]; then + rm -rf "$1" + fi + } + trap 'remove_tmp_pg_init_sock_dir "$PGHOST"' EXIT + + pg_ctl -D "$PGDATA" -w start -o "-c unix_socket_directories=$PGHOST -c listen_addresses= -p ${toString config.port}" + ${runInitialScript.before} + ${setupInitialDatabases} + ${runInitialScript.after} + pg_ctl -D "$PGDATA" -m fast -w stop + remove_tmp_pg_init_sock_dir "$PGHOST" + else + echo + echo "PostgreSQL database directory appears to contain a database; Skipping initialization" + echo + fi + unset POSTGRES_RUN_INITIAL_SCRIPT + ''; +}) diff --git a/test/flake.nix b/test/flake.nix index 4691143..23a2a3c 100644 --- a/test/flake.nix +++ b/test/flake.nix @@ -41,7 +41,7 @@ ../nix/elasticsearch_test.nix ../nix/mysql_test.nix ../nix/nginx_test.nix - ../nix/postgres_test.nix + ../nix/postgres/postgres_test.nix ../nix/redis_test.nix ../nix/redis-cluster_test.nix ../nix/zookeeper_test.nix