juspay/services-flake
1
1
Fork 0
mirror of https://github.com/juspay/services-flake.git synced 2024-09-11 04:15:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor: writeShellScriptBin -> writeShellApplication (#155)

Browse Source 
Provides us with following benefits:
- use `runtimeInputs` instead of manually exporting `PATH`
- get shellcheck by default

---------

Co-authored-by: Sridhar Ratnakumar <3998+srid@users.noreply.github.com>
This commit is contained in:
Shivaraj B H 2024-03-19 01:47:25 +05:30 committed by GitHub
parent f158353b59
commit 819ab20e8e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 205 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
nix/apache-kafka.nix
View File

@ -161,17 +161,21 @@ with lib;
processes = {
"${name}" =
let
startScript = pkgs.writeShellScriptBin "start-kafka" ''
${config.jre}/bin/java \
-cp "${config.package}/libs/*" \
-Dlog4j.configuration=file:${config.configFiles.log4jProperties} \
${toString config.jvmOptions} \
kafka.Kafka \
${config.configFiles.serverProperties}
'';
startScript = pkgs.writeShellApplication {
name = "start-kafka";
runtimeInputs = [ config.jre ];
text = ''
java \
-cp "${config.package}/libs/*" \
-Dlog4j.configuration=file:${config.configFiles.log4jProperties} \
${toString config.jvmOptions} \
kafka.Kafka \
${config.configFiles.serverProperties}
'';
};
in
{
command = "${startScript}/bin/start-kafka";
command = startScript;
readiness_probe = {
# TODO: need to find a better way to check if kafka is ready. Maybe use one of the scripts in bin?

40
nix/cassandra.nix
View File

@ -119,30 +119,34 @@ in
'';
};
startScript = pkgs.writeShellScriptBin "start-cassandra" ''
set -euo pipefail
startScript = pkgs.writeShellApplication {
name = "start-cassandra";
runtimeInputs = [ pkgs.coreutils config.package ];
text = ''
set -euo pipefail
DATA_DIR="$(readlink -m ${config.dataDir})"
if [[ ! -d "$DATA_DIR" ]]; then
mkdir -p "$DATA_DIR"
fi
DATA_DIR="$(readlink -m ${config.dataDir})"
if [[ ! -d "$DATA_DIR" ]]; then
mkdir -p "$DATA_DIR"
fi
CASSANDRA_CONF="${cassandraConfig}"
export CASSANDRA_CONF
CASSANDRA_CONF="${cassandraConfig}"
export CASSANDRA_CONF
CASSANDRA_LOG_DIR="$DATA_DIR/log/"
mkdir -p "$CASSANDRA_LOG_DIR"
export CASSANDRA_LOG_DIR
CASSANDRA_LOG_DIR="$DATA_DIR/log/"
mkdir -p "$CASSANDRA_LOG_DIR"
export CASSANDRA_LOG_DIR
CASSANDRA_HOME="${config.package}"
export CASSANDRA_HOME
CASSANDRA_HOME="${config.package}"
export CASSANDRA_HOME
CLASSPATH="${config.package}/lib"
export CLASSPATH
CLASSPATH="${config.package}/lib"
export CLASSPATH
export LOCAL_JMX="yes"
exec ${config.package}/bin/cassandra -f
'';
export LOCAL_JMX="yes"
exec cassandra -f
'';
};
in
{
command = startScript;

2
nix/clickhouse/default.nix
View File

@ -154,7 +154,7 @@ in
};
in
{
command = "${lib.getExe startScript}";
command = startScript;
readiness_probe = {
# FIXME: revert back to clickhouse-client readiness_probe once CI is moved out of github public runners

67
nix/elasticsearch.nix
View File

@ -141,44 +141,53 @@ in
postBuild = "${pkgs.coreutils}/bin/mkdir -p $out/plugins";
};
startScript = pkgs.writeShellScript "es-startup" ''
set -e
startScript = pkgs.writeShellApplication {
mkdir -m 0700 -p "${config.dataDir}"
export ES_HOME=$(${pkgs.coreutils}/bin/realpath ${config.dataDir})
export ES_JAVA_OPTS="${toString config.extraJavaOptions}"
export ES_PATH_CONF="${config.dataDir}/config"
# Install plugins
rm -f "${config.dataDir}/plugins"
ln -sf ${esPlugins}/plugins "${config.dataDir}/plugins"
rm -f "${config.dataDir}/lib"
ln -sf ${config.package}/lib "${config.dataDir}/lib"
rm -f "${config.dataDir}/modules"
ln -sf ${config.package}/modules "${config.dataDir}/modules"
name = "es-startup";
runtimeInputs = [ pkgs.coreutils config.package ];
text = ''
set -e
# Create config dir
mkdir -m 0700 -p "${config.dataDir}/config"
rm -f "${config.dataDir}/config/elasticsearch.yml"
cp ${elasticsearchYml} "${config.dataDir}/config/elasticsearch.yml"
rm -f "${config.dataDir}/logging.yml"
rm -f "${config.dataDir}/config/${loggingConfigFilename}"
cp ${loggingConfigFile} "${config.dataDir}/config/${loggingConfigFilename}"
mkdir -p "${config.dataDir}"
chmod 0700 "${config.dataDir}"
ES_HOME=$(${pkgs.coreutils}/bin/realpath ${config.dataDir})
ES_JAVA_OPTS="${toString config.extraJavaOptions}"
ES_PATH_CONF="${config.dataDir}/config"
export ES_HOME ES_JAVA_OPTS ES_PATH_CONF
mkdir -p "${config.dataDir}/scripts"
rm -f "${config.dataDir}/config/jvm.options"
# Install plugins
rm -f "${config.dataDir}/plugins"
ln -sf ${esPlugins}/plugins "${config.dataDir}/plugins"
rm -f "${config.dataDir}/lib"
ln -sf ${config.package}/lib "${config.dataDir}/lib"
rm -f "${config.dataDir}/modules"
ln -sf ${config.package}/modules "${config.dataDir}/modules"
cp ${config.package}/config/jvm.options "${config.dataDir}/config/jvm.options"
# Create config dir
mkdir -p "${config.dataDir}/config"
chmod 0700 "${config.dataDir}/config"
rm -f "${config.dataDir}/config/elasticsearch.yml"
cp ${elasticsearchYml} "${config.dataDir}/config/elasticsearch.yml"
rm -f "${config.dataDir}/logging.yml"
rm -f "${config.dataDir}/config/${loggingConfigFilename}"
cp ${loggingConfigFile} "${config.dataDir}/config/${loggingConfigFilename}"
# Create log dir
mkdir -m 0700 -p "${config.dataDir}/logs"
mkdir -p "${config.dataDir}/scripts"
rm -f "${config.dataDir}/config/jvm.options"
# Start it
exec ${config.package}/bin/elasticsearch ${toString config.extraCmdLineOptions}
'';
cp ${config.package}/config/jvm.options "${config.dataDir}/config/jvm.options"
# Create log dir
mkdir -p "${config.dataDir}/logs"
chmod 0700 "${config.dataDir}/logs"
# Start it
exec elasticsearch ${toString config.extraCmdLineOptions}
'';
};
in
{
command = "${startScript}";
command = startScript;
readiness_probe = {
exec.command = "${pkgs.curl}/bin/curl -f -k http://${config.listenAddress}:${toString config.port}";

144
nix/mysql/default.nix
View File

@ -169,16 +169,21 @@ in
mysqlOptions = "--defaults-file=${configFile}";
mysqldOptions = "${mysqlOptions} --datadir=${config.dataDir} --basedir=${config.package}";
envs = ''
export MYSQL_HOME=$(${pkgs.coreutils}/bin/realpath ${config.dataDir})
export MYSQL_UNIX_PORT=$(${pkgs.coreutils}/bin/realpath ${config.dataDir + "/mysql.sock"})
export MYSQLX_UNIX_PORT=$(${pkgs.coreutils}/bin/realpath ${config.dataDir + "/mysqlx.sock"})
MYSQL_HOME=$(${pkgs.coreutils}/bin/realpath ${config.dataDir})
MYSQL_UNIX_PORT=$(${pkgs.coreutils}/bin/realpath ${config.dataDir + "/mysql.sock"})
MYSQLX_UNIX_PORT=$(${pkgs.coreutils}/bin/realpath ${config.dataDir + "/mysqlx.sock"})
export MYSQL_HOME
export MYSQL_UNIX_PORT
export MYSQLX_UNIX_PORT
${lib.optionalString (lib.hasAttrByPath [ "mysqld" "port" ] config.settings) "export MYSQL_TCP_PORT=${toString config.settings.mysqld.port}"}
'';
initDatabaseCmd =
if isMariaDB
then "${config.package}/bin/mysql_install_db ${mysqldOptions} --auth-root-authentication-method=normal"
else "${config.package}/bin/mysqld ${mysqldOptions} --default-time-zone=SYSTEM --initialize-insecure";
then "mysql_install_db ${mysqldOptions} --auth-root-authentication-method=normal"
else "mysqld ${mysqldOptions} --default-time-zone=SYSTEM --initialize-insecure";
importTimeZones =
if (config.importTimeZones != null)
@ -188,85 +193,92 @@ in
configureTimezones = ''
# Start a temp database with the default-time-zone to import tz data
# and hide the temp database from the configureScript by setting a custom socket
nohup ${config.package}/bin/mysqld ${mysqldOptions} --socket="${config.dataDir}/config.sock" --skip-networking --default-time-zone=SYSTEM &
nohup mysqld ${mysqldOptions} --socket="${config.dataDir}/config.sock" --skip-networking --default-time-zone=SYSTEM &
while ! MYSQL_PWD="" ${config.package}/bin/mysqladmin --socket="${config.dataDir}/config.sock" ping -u root --silent; do
while ! MYSQL_PWD="" mysqladmin --socket="${config.dataDir}/config.sock" ping -u root --silent; do
sleep 1
done
${config.package}/bin/mysql_tzinfo_to_sql ${pkgs.tzdata}/share/zoneinfo/ | MYSQL_PWD="" ${config.package}/bin/mysql --socket="${config.dataDir}/config.sock" -u root mysql
mysql_tzinfo_to_sql ${pkgs.tzdata}/share/zoneinfo/ | MYSQL_PWD="" mysql --socket="${config.dataDir}/config.sock" -u root mysql
# Shutdown the temp database
MYSQL_PWD="" ${config.package}/bin/mysqladmin --socket="${config.dataDir}/config.sock" shutdown -u root
MYSQL_PWD="" mysqladmin --socket="${config.dataDir}/config.sock" shutdown -u root
'';
startScript = pkgs.writeShellScriptBin "start-mysql" ''
set -euo pipefail
startScript = pkgs.writeShellApplication {
name = "start-mysql";
runtimeInputs = [ config.package pkgs.coreutils ];
text = ''
set -euo pipefail
if [[ ! -d ${config.dataDir} || ! -f ${config.dataDir}/ibdata1 ]]; then
mkdir -p ${config.dataDir}
${initDatabaseCmd}
${lib.optionalString importTimeZones configureTimezones}
fi
${envs}
exec ${config.package}/bin/mysqld ${mysqldOptions}
'';
if [[ ! -d ${config.dataDir} || ! -f ${config.dataDir}/ibdata1 ]]; then
mkdir -p ${config.dataDir}
${initDatabaseCmd}
${lib.optionalString importTimeZones configureTimezones}
fi
${envs}
exec mysqld ${mysqldOptions}
'';
};
runInitialScript = lib.optionalString (config.initialScript != null) ''echo ${lib.escapeShellArg config.initialScript} | MYSQL_PWD="" ${config.package}/bin/mysql -u root -N
runInitialScript = lib.optionalString (config.initialScript != null) ''echo ${lib.escapeShellArg config.initialScript} | MYSQL_PWD="" mysql -u root -N
'';
configureScript = pkgs.writeShellScriptBin "configure-mysql" ''
PATH="${lib.makeBinPath [config.package pkgs.coreutils pkgs.findutils]}:$PATH"
set -euo pipefail
${envs}
${lib.concatMapStrings (database: ''
# Create initial databases
exists="$(
MYSQL_PWD="" ${config.package}/bin/mysql -u root -sB information_schema \
<<< 'select count(*) from schemata where schema_name = "${database.name}"'
)"
if [[ "$exists" -eq 0 ]]; then
echo "Creating initial database: ${database.name}"
( echo 'create database `${database.name}`;'
${lib.optionalString (database.schema != null) ''
echo 'use `${database.name}`;'
# TODO: this silently falls through if database.schema does not exist,
# we should catch this somehow and exit, but can't do it here because we're in a subshell.
if [ -f "${database.schema}" ]
then
cat ${database.schema}
elif [ -d "${database.schema}" ]
then
find ${database.schema} -type f -name '*.sql' | xargs cat
configureScript = pkgs.writeShellApplication {
name = "configure-mysql";
runtimeInputs = with pkgs; [ config.package coreutils findutils ];
text = ''
set -euo pipefail
${envs}
${lib.concatMapStrings (database: ''
# Create initial databases
exists="$(
MYSQL_PWD="" mysql -u root -sB information_schema \
<<< 'select count(*) from schemata where schema_name = "${database.name}"'
)"
if [[ "$exists" -eq 0 ]]; then
echo "Creating initial database: ${database.name}"
( echo "create database ${database.name};"
${lib.optionalString (database.schema != null) ''
echo "use ${database.name};"
# TODO: this silently falls through if database.schema does not exist,
# we should catch this somehow and exit, but can't do it here because we're in a subshell.
if [ -f "${database.schema}" ]
then
cat ${database.schema}
elif [ -d "${database.schema}" ]
then
# -print0/-0 is used because of: https://www.shellcheck.net/wiki/SC2038
find ${database.schema} -type f -name '*.sql' -print0 | xargs -0 cat
fi
''}
) | MYSQL_PWD="" mysql -u root -N
else
echo "Database ${database.name} exists, skipping creation."
fi
''}
) | MYSQL_PWD="" ${config.package}/bin/mysql -u root -N
else
echo "Database ${database.name} exists, skipping creation."
fi
'')
config.initialDatabases}
'')
config.initialDatabases}
${lib.concatMapStrings (user: ''
echo "Adding user: ${user.name}"
${lib.optionalString (user.password != null) "password='${user.password}'"}
( echo "CREATE USER IF NOT EXISTS '${user.name}'@'localhost' ${lib.optionalString (user.password != null) "IDENTIFIED BY '$password'"};"
${lib.concatStringsSep "\n" (lib.mapAttrsToList (database: permission: ''
echo 'GRANT ${permission} ON ${database} TO `${user.name}`@`localhost`;'
'')
user.ensurePermissions)}
) | MYSQL_PWD="" ${config.package}/bin/mysql -u root -N
'')
config.ensureUsers}
${runInitialScript}
'';
${lib.concatMapStrings (user: ''
echo "Adding user: ${user.name}"
${lib.optionalString (user.password != null) "password='${user.password}'"}
( echo "CREATE USER IF NOT EXISTS '${user.name}'@'localhost' ${lib.optionalString (user.password != null) "IDENTIFIED BY '$password'"};"
${lib.concatStringsSep "\n" (lib.mapAttrsToList (database: permission: ''
echo "GRANT ${permission} ON ${database} TO '${user.name}'@'localhost';"
'')
user.ensurePermissions)}
) | MYSQL_PWD="" mysql -u root -N
'')
config.ensureUsers}
${runInitialScript}
'';
};
in
{
"${name}" =
{
command = "${startScript}/bin/start-mysql";
command = startScript;
readiness_probe = {
# Turns out using `--defaults-file` alone doesn't make the readiness_probe work unless `MYSQL_UNIX_PORT` is set.
@ -284,7 +296,7 @@ in
availability.restart = "on_failure";
};
"${name}-configure" = {
command = "${configureScript}/bin/configure-mysql";
command = configureScript;
namespace = name;
depends_on."${name}".condition = "process_healthy";
};

20
nix/nginx.nix
View File

@ -92,17 +92,21 @@ in
readOnly = true;
default =
let
startScript = pkgs.writeShellScriptBin "start-nginx" ''
set -euo pipefail
if [[ ! -d "${config.dataDir}" ]]; then
mkdir -p "${config.dataDir}"
fi
${config.package}/bin/nginx -p $(pwd) -c ${config.configFile} -e /dev/stderr
'';
startScript = pkgs.writeShellApplication {
name = "start-nginx";
runtimeInputs = [ pkgs.coreutils config.package ];
text = ''
set -euo pipefail
if [[ ! -d "${config.dataDir}" ]]; then
mkdir -p "${config.dataDir}"
fi
nginx -p "$(pwd)" -c ${config.configFile} -e /dev/stderr
'';
};
in
{
processes."${name}" = {
command = "${startScript}/bin/start-nginx";
command = startScript;
readiness_probe = {
# FIXME need a better health check
exec.command = "[ -e ${config.dataDir}/nginx/nginx.pid ]";

2
nix/prometheus.nix
View File

@ -84,7 +84,7 @@ in
};
in
{
command = "${startScript}/bin/start-prometheus";
command = startScript;
readiness_probe = {
http_get = {
host = config.listenAddress;

22
nix/redis-cluster.nix
View File

@ -90,20 +90,24 @@ in
${cfg.extraConfig}
'';
startScript = pkgs.writeShellScriptBin "start-redis" ''
set -euo pipefail
startScript = pkgs.writeShellApplication {
name = "start-redis";
runtimeInputs = [ pkgs.coreutils config.package ];
text = ''
set -euo pipefail
export REDISDATA=${config.dataDir}
export REDISDATA=${config.dataDir}
if [[ ! -d "$REDISDATA" ]]; then
mkdir -p "$REDISDATA"
fi
if [[ ! -d "$REDISDATA" ]]; then
mkdir -p "$REDISDATA"
fi
exec ${config.package}/bin/redis-server ${redisConfig} --dir "$REDISDATA"
'';
exec redis-server ${redisConfig} --dir "$REDISDATA"
'';
};
in
lib.nameValuePair "${name}-${nodeName}" {
command = "${startScript}/bin/start-redis";
command = startScript;
shutdown.command = "${config.package}/bin/redis-cli -p ${port} shutdown nosave";
readiness_probe = {

22
nix/redis.nix
View File

@ -55,20 +55,24 @@ in
${config.extraConfig}
'';
startScript = pkgs.writeShellScriptBin "start-redis" ''
set -euo pipefail
startScript = pkgs.writeShellApplication {
name = "start-redis";
runtimeInputs = [ pkgs.coreutils config.package ];
text = ''
set -euo pipefail
export REDISDATA=${config.dataDir}
export REDISDATA=${config.dataDir}
if [[ ! -d "$REDISDATA" ]]; then
mkdir -p "$REDISDATA"
fi
if [[ ! -d "$REDISDATA" ]]; then
mkdir -p "$REDISDATA"
fi
exec ${config.package}/bin/redis-server ${redisConfig} --dir "$REDISDATA"
'';
exec redis-server ${redisConfig} --dir "$REDISDATA"
'';
};
in
{
command = "${startScript}/bin/start-redis";
command = startScript;
readiness_probe = {
exec.command = "${config.package}/bin/redis-cli -p ${toString config.port} ping";

24
nix/zookeeper.nix
View File

@ -123,18 +123,22 @@ with lib;
];
};
startScript = pkgs.writeShellScriptBin "start-zookeeper" ''
${config.jre}/bin/java \
-cp "${config.package}/lib/*:${configDir}" \
${escapeShellArgs config.extraCmdLineOptions} \
-Dzookeeper.datadir.autocreate=true \
${optionalString config.preferIPv4 "-Djava.net.preferIPv4Stack=true"} \
org.apache.zookeeper.server.quorum.QuorumPeerMain \
${configDir}/zoo.cfg
'';
startScript = pkgs.writeShellApplication {
name = "start-zookeeper";
runtimeInputs = [ config.jre ];
text = ''
java \
-cp "${config.package}/lib/*:${configDir}" \
${escapeShellArgs config.extraCmdLineOptions} \
-Dzookeeper.datadir.autocreate=true \
${optionalString config.preferIPv4 "-Djava.net.preferIPv4Stack=true"} \
org.apache.zookeeper.server.quorum.QuorumPeerMain \
${configDir}/zoo.cfg
'';
};
in
{
command = "${startScript}/bin/start-zookeeper";
command = startScript;
readiness_probe = {
exec.command = "echo stat | ${pkgs.netcat.nc}/bin/nc localhost ${toString config.port}";