Skip to content

Commit

Permalink
Using psql command instead of postgres in initialScript (#66)
Browse files Browse the repository at this point in the history
* Use psql instead of postgres, as a result, use `pg_ctl` to temporarily start the postgres server in `${name}-init` process.
* Avoid creating a custom `postgresPkg` with extensions by using `apply` attribute provided by `mkOption`
* Refactor setup script, use `writeShellApplication` and resolve shell-check errors

---------

Co-authored-by: shivaraj-bh <[email protected]>
  • Loading branch information
rsrohitsingh682 and shivaraj-bh authored Dec 24, 2023
1 parent 8cd80d7 commit 22e121b
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 119 deletions.
2 changes: 1 addition & 1 deletion nix/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ in
./elasticsearch.nix
./mysql.nix
./nginx.nix
./postgres.nix
./postgres
./redis-cluster.nix
./redis.nix
./zookeeper.nix
Expand Down
141 changes: 24 additions & 117 deletions nix/postgres.nix → nix/postgres/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,24 @@ in
options = {
enable = lib.mkEnableOption name;

package = lib.mkPackageOption pkgs "postgresql" { };
package = lib.mkOption {
type = types.package;
description = "Which package of postgresql to use";
default = pkgs.postgresql;
defaultText = lib.literalExpression "pkgs.postgresql";
apply = postgresPkg:
if config.extensions != null then
if builtins.hasAttr "withPackages" postgresPkg
then postgresPkg.withPackages config.extensions
else
builtins.throw ''
Cannot add extensions to the PostgreSQL package.
`services.postgres.package` is missing the `withPackages` attribute. Did you already add extensions to the package?
''
else postgresPkg;

};

extensions = lib.mkOption {
type = with types; nullOr (functionTo (listOf package));
default = null;
Expand Down Expand Up @@ -254,125 +271,15 @@ in
internal = true;
readOnly = true;
default =
let
postgresPkg =
if config.extensions != null then
if builtins.hasAttr "withPackages" config.package
then config.package.withPackages config.extensions
else
builtins.throw ''
Cannot add extensions to the PostgreSQL package.
`services.postgres.package` is missing the `withPackages` attribute. Did you already add extensions to the package?
''
else config.package;
in
{
processes = {
# DB initialization
"${name}-init" =
let
setupInitialDatabases =
if config.initialDatabases != [ ] then
(lib.concatMapStrings
(database: ''
echo "Checking presence of database: ${database.name}"
# Create initial databases
dbAlreadyExists="$(
echo "SELECT 1 as exists FROM pg_database WHERE datname = '${database.name}';" | \
postgres --single -E postgres | \
${pkgs.gnugrep}/bin/grep -c 'exists = "1"' || true
)"
echo $dbAlreadyExists
if [ 1 -ne "$dbAlreadyExists" ]; then
echo "Creating database: ${database.name}"
echo 'create database "${database.name}";' | postgres --single -E postgres
${lib.optionalString (database.schema != null) ''
echo "Applying database schema on ${database.name}"
if [ -f "${database.schema}" ]
then
echo "Running file ${database.schema}"
${pkgs.gawk}/bin/awk 'NF' "${database.schema}" | postgres --single -j -E ${database.name}
elif [ -d "${database.schema}" ]
then
# Read sql files in version order. Apply one file
# at a time to handle files where the last statement
# doesn't end in a ;.
ls -1v "${database.schema}"/*.sql | while read f ; do
echo "Applying sql file: $f"
${pkgs.gawk}/bin/awk 'NF' "$f" | postgres --single -j -E ${database.name}
done
else
echo "ERROR: Could not determine how to apply schema with ${database.schema}"
exit 1
fi
''}
fi
'')
config.initialDatabases)
else
lib.optionalString config.createDatabase ''
echo "CREATE DATABASE ''${USER:-$(id -nu)};" | postgres --single -E postgres '';

runInitialScript =
let
scriptCmd = sqlScript: ''
echo "${sqlScript}" | postgres --single -E postgres
'';
in
{
before = with config.initialScript;
lib.optionalString (before != null) (scriptCmd before);
after = with config.initialScript;
lib.optionalString (after != null) (scriptCmd after);
};

toStr = value:
if true == value then
"yes"
else if false == value then
"no"
else if lib.isString value then
"'${lib.replaceStrings [ "'" ] [ "''" ] value}'"
else
toString value;

configFile = pkgs.writeText "postgresql.conf" (lib.concatStringsSep "\n"
(lib.mapAttrsToList (n: v: "${n} = ${toStr v}") (config.defaultSettings // config.settings)));

initdbArgs =
config.initdbArgs
++ (lib.optionals (config.superuser != null) [ "-U" config.superuser ])
++ [ "-D" config.dataDir ];

setupScript = pkgs.writeShellScriptBin "setup-postgres" ''
set -euo pipefail
export PATH=${postgresPkg}/bin:${pkgs.coreutils}/bin
if [[ ! -d "$PGDATA" ]]; then
set -x
mkdir -p ${config.dataDir}
initdb ${lib.concatStringsSep " " initdbArgs}
set +x
${runInitialScript.before}
${setupInitialDatabases}
${runInitialScript.after}
else
echo "Postgres data directory already exists. Skipping initialization."
fi
# Setup config
set -x
cp ${configFile} "$PGDATA/postgresql.conf"
'';
setupScript = import ./setup-script.nix { inherit config pkgs lib; };
in
{
command = ''
export PGDATA="${config.dataDir}"
${lib.getExe setupScript}
'';
command = setupScript;
namespace = name;
};

Expand All @@ -381,9 +288,9 @@ in
let
startScript = pkgs.writeShellApplication {
name = "start-postgres";
runtimeInputs = [ config.package pkgs.coreutils ];
text = ''
export PATH="${postgresPkg}"/bin:$PATH
set -x
set -euo pipefail
PGDATA=$(readlink -f "${config.dataDir}")
export PGDATA
postgres -k "$PGDATA"
Expand All @@ -397,18 +304,18 @@ in
in
{
command = startScript;
depends_on."${name}-init".condition = "process_completed_successfully";
# SIGINT (= 2) for faster shutdown: https://www.postgresql.org/docs/current/server-shutdown.html
shutdown.signal = 2;
readiness_probe = {
exec.command = "${postgresPkg}/bin/pg_isready ${lib.concatStringsSep " " pg_isreadyArgs}";
exec.command = "${config.package}/bin/pg_isready ${lib.concatStringsSep " " pg_isreadyArgs}";
initial_delay_seconds = 2;
period_seconds = 10;
timeout_seconds = 4;
success_threshold = 1;
failure_threshold = 5;
};
namespace = name;
depends_on."${name}-init".condition = "process_completed_successfully";
# https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
availability.restart = "on_failure";
};
Expand Down
File renamed without changes.
124 changes: 124 additions & 0 deletions nix/postgres/setup-script.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
{ config, pkgs, lib }:
let
setupInitialDatabases =
if config.initialDatabases != [ ] then
(lib.concatMapStrings
(database: ''
echo "Checking presence of database: ${database.name}"
# Create initial databases
dbAlreadyExists=$(
echo "SELECT 1 as exists FROM pg_database WHERE datname = '${database.name}';" | \
psql -d postgres | \
grep -c 'exists = "1"' || true
)
echo "$dbAlreadyExists"
if [ 1 -ne "$dbAlreadyExists" ]; then
echo "Creating database: ${database.name}"
echo 'create database "${database.name}";' | psql -d postgres
${lib.optionalString (database.schema != null) ''
echo "Applying database schema on ${database.name}"
if [ -f "${database.schema}" ]
then
echo "Running file ${database.schema}"
awk 'NF' "${database.schema}" | psql -d ${database.name}
elif [ -d "${database.schema}" ]
then
# Read sql files in version order. Apply one file
# at a time to handle files where the last statement
# doesn't end in a ;.
find "${database.schema}"/*.sql | while read -r f ; do
echo "Applying sql file: $f"
awk 'NF' "$f" | psql -d ${database.name}
done
else
echo "ERROR: Could not determine how to apply schema with ${database.schema}"
exit 1
fi
''}
fi
'')
config.initialDatabases)
else
lib.optionalString config.createDatabase ''
echo "CREATE DATABASE ''${USER:-$(id -nu)};" | psql -d postgres '';
runInitialScript =
let
scriptCmd = sqlScript: ''
echo "${sqlScript}" | psql -d postgres
'';
in
{
before = with config.initialScript;
lib.optionalString (before != null) (scriptCmd before);
after = with config.initialScript;
lib.optionalString (after != null) (scriptCmd after);
};
toStr = value:
if true == value then
"yes"
else if false == value then
"no"
else if lib.isString value then
"'${lib.replaceStrings [ "'" ] [ "''" ] value}'"
else
toString value;
configFile = pkgs.writeText "postgresql.conf" (lib.concatStringsSep "\n"
(lib.mapAttrsToList (n: v: "${n} = ${toStr v}") (config.defaultSettings // config.settings)));

initdbArgs =
config.initdbArgs
++ (lib.optionals (config.superuser != null) [ "-U" config.superuser ])
++ [ "-D" config.dataDir ];
in
(pkgs.writeShellApplication {
name = "setup-postgres";
runtimeInputs = with pkgs; [ config.package coreutils gnugrep gawk ];
text = ''
set -euo pipefail
# Setup postgres ENVs
export PGDATA="${config.dataDir}"
export PGPORT="${toString config.port}"
POSTGRES_RUN_INITIAL_SCRIPT="false"
if [[ ! -d "$PGDATA" ]]; then
initdb ${lib.concatStringsSep " " initdbArgs}
POSTGRES_RUN_INITIAL_SCRIPT="true"
echo
echo "PostgreSQL initdb process complete."
echo
fi
# Setup config
cp ${configFile} "$PGDATA/postgresql.conf"
if [[ "$POSTGRES_RUN_INITIAL_SCRIPT" = "true" ]]; then
echo
echo "PostgreSQL is setting up the initial database."
echo
PGHOST=$(mktemp -d "$(readlink -f ${config.dataDir})/pg-init-XXXXXX")
export PGHOST
function remove_tmp_pg_init_sock_dir() {
if [[ -d "$1" ]]; then
rm -rf "$1"
fi
}
trap 'remove_tmp_pg_init_sock_dir "$PGHOST"' EXIT
pg_ctl -D "$PGDATA" -w start -o "-c unix_socket_directories=$PGHOST -c listen_addresses= -p ${toString config.port}"
${runInitialScript.before}
${setupInitialDatabases}
${runInitialScript.after}
pg_ctl -D "$PGDATA" -m fast -w stop
remove_tmp_pg_init_sock_dir "$PGHOST"
else
echo
echo "PostgreSQL database directory appears to contain a database; Skipping initialization"
echo
fi
unset POSTGRES_RUN_INITIAL_SCRIPT
'';
})
2 changes: 1 addition & 1 deletion test/flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
../nix/elasticsearch_test.nix
../nix/mysql_test.nix
../nix/nginx_test.nix
../nix/postgres_test.nix
../nix/postgres/postgres_test.nix
../nix/redis_test.nix
../nix/redis-cluster_test.nix
../nix/zookeeper_test.nix
Expand Down

0 comments on commit 22e121b

Please sign in to comment.